ai语音输入

chenxt
1 parent 8acabc8a
Showing 1 changed file with 310 additions and 31 deletions
src/mobile/Ai/newAi.jsx
-import React, { useState, useEffect, useRef } from 'react';
+import React, { useState, useEffect, useRef, useCallback } from 'react';
 import ReactMarkdown from 'react-markdown';
 import remarkGfm from 'remark-gfm';
-import { AudioOutline } from "antd-mobile-icons";
-import './AiChatStyles.less'; // 引入外部样式文件
+import { AudioOutline, AudioFill } from "antd-mobile-icons";
+import './AiChatStyles.less';
 const ChatInterface = () => {
   // ==================== 状态管理 ====================
@@ -15,13 +15,27 @@ const ChatInterface = () =&gt; {
   const [currentModel, setCurrentModel] = useState('general');
   const [chatHistory, setChatHistory] = useState([]);
   const [welcomeContent, setWelcomeContent] = useState('');
-  
+
+  // 语音输入状态
+  const [isRecording, setIsRecording] = useState(false);
+  const [isWsConnected, setIsWsConnected] = useState(false);
+  const [isVoiceMode, setIsVoiceMode] = useState(false);
+  const [recordingDuration, setRecordingDuration] = useState(0);
+
   const messagesEndRef = useRef(null);
   const inputRef = useRef(null);
+  const wsRef = useRef(null);
+  const audioContextRef = useRef(null);
+  const scriptProcessorRef = useRef(null);
+  const inputNodeRef = useRef(null);
+  const recordingTimerRef = useRef(null);
+  const isRecordingRef = useRef(false);
   // ==================== 配置 ====================
   const CONFIG = {
     backendUrl: 'http://localhost:8099/xlyAi',
+    wsUrl: 'ws://121.43.128.225:10096', // 语音识别WebSocket地址
+    sampleRate: 16000,
     endpoints: {
       chat: '/api/v1/chat/query',
       process: '/api/v1/chat/query',
@@ -52,6 +66,212 @@ const ChatInterface = () =&gt; {
     messagesEndRef.current?.scrollIntoView({ behavior: 'smooth' });
   };
+  // ==================== WebSocket 语音识别 ====================
+
+  // 连接语音识别WebSocket
+  const connectWebSocket = useCallback(() => {
+    if (wsRef.current && wsRef.current.readyState === WebSocket.OPEN) {
+      return;
+    }
+
+    const ws = new WebSocket(CONFIG.wsUrl);
+    ws.binaryType = "arraybuffer";
+
+    ws.onopen = () => {
+      console.log("语音识别WebSocket连接成功");
+      setIsWsConnected(true);
+    };
+
+    ws.onmessage = (event) => {
+      try {
+        const res = JSON.parse(event.data);
+        if (res.code === 0 && (res.msg === "success" || res.msg === "flush_success")) {
+          if (res.text && res.text.trim()) {
+            setInputValue(prev => {
+              const separator = prev && !prev.endsWith(' ') ? ' ' : '';
+              const newValue = prev ? `${prev}${separator}${res.text}` : res.text;
+              console.log('语音识别结果:', res.text, '更新后:', newValue);
+              return newValue;
+            });
+          }
+        }
+      } catch (e) {
+        console.error("WebSocket消息解析失败:", e);
+      }
+    };
+
+    ws.onclose = () => {
+      console.log("语音识别WebSocket连接断开");
+      setIsWsConnected(false);
+      if (isRecordingRef.current) {
+        stopRecording();
+      }
+    };
+
+    ws.onerror = (err) => {
+      console.error("WebSocket错误:", err);
+      setIsWsConnected(false);
+    };
+
+    wsRef.current = ws;
+  }, []);
+
+  // 断开WebSocket
+  const disconnectWebSocket = useCallback(() => {
+    if (wsRef.current) {
+      wsRef.current.close();
+      wsRef.current = null;
+    }
+    setIsWsConnected(false);
+  }, []);
+
+  // 发送指令
+  const sendCommand = useCallback((action) => {
+    if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) {
+      return;
+    }
+    const cmd = JSON.stringify({ action });
+    wsRef.current.send(cmd);
+  }, []);
+
+  // Float32 转 Int16 PCM
+  const float32ToInt16 = (float32Array) => {
+    const int16Array = new Int16Array(float32Array.length);
+    for (let i = 0; i < float32Array.length; i++) {
+      let s = Math.max(-1, Math.min(1, float32Array[i]));
+      int16Array[i] = s < 0 ? s * 0x8000 : s * 0x7FFF;
+    }
+    return new Uint8Array(int16Array.buffer);
+  };
+
+  // 音频重采样
+  const resampleAudio = (data, originalRate, targetRate) => {
+    if (originalRate === targetRate) return data;
+    const ratio = targetRate / originalRate;
+    const newLength = Math.round(data.length * ratio);
+    const result = new Float32Array(newLength);
+    for (let i = 0; i < newLength; i++) {
+      result[i] = data[Math.round(i / ratio)] || 0;
+    }
+    return result;
+  };
+
+  // 开始录音
+  const startRecording = async () => {
+    if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
+      alert("浏览器不支持麦克风采集");
+      return;
+    }
+
+    try {
+      // 先连接WebSocket
+      if (!isWsConnected) {
+        connectWebSocket();
+        await new Promise(resolve => setTimeout(resolve, 1000));
+      }
+
+      // 检查 WebSocket 是否已连接
+      if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) {
+        alert("语音识别服务未连接，请重试");
+        return;
+      }
+
+      const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+      audioContextRef.current = new (window.AudioContext || window.webkitAudioContext)();
+      inputNodeRef.current = audioContextRef.current.createMediaStreamSource(stream);
+      scriptProcessorRef.current = audioContextRef.current.createScriptProcessor(2048, 1, 1);
+
+      // 使用 ref 来检查录音状态，避免闭包问题
+      scriptProcessorRef.current.onaudioprocess = (event) => {
+        if (!isRecordingRef.current) return;
+        const inputData = event.inputBuffer.getChannelData(0);
+        const resampledData = resampleAudio(inputData, audioContextRef.current.sampleRate, CONFIG.sampleRate);
+        const pcmData = float32ToInt16(resampledData);
+        if (wsRef.current && wsRef.current.readyState === WebSocket.OPEN) {
+          wsRef.current.send(pcmData);
+        }
+      };
+
+      inputNodeRef.current.connect(scriptProcessorRef.current);
+      scriptProcessorRef.current.connect(audioContextRef.current.destination);
+
+      // 关键：先设置 ref，再设置 state
+      isRecordingRef.current = true;
+      setIsRecording(true);
+      setIsVoiceMode(true);
+      setRecordingDuration(0);
+
+      // 开始计时
+      recordingTimerRef.current = setInterval(() => {
+        setRecordingDuration(prev => prev + 1);
+      }, 1000);
+
+      console.log("录音已开始");
+
+    } catch (e) {
+      console.error("录音启动失败:", e);
+      alert("录音启动失败：" + e.message);
+      isRecordingRef.current = false;
+      setIsRecording(false);
+    }
+  };
+
+  // 停止录音
+  const stopRecording = useCallback(() => {
+    console.log("停止录音");
+    isRecordingRef.current = false;
+    setIsRecording(false);
+    setIsVoiceMode(false);
+
+    if (recordingTimerRef.current) {
+      clearInterval(recordingTimerRef.current);
+      recordingTimerRef.current = null;
+    }
+
+    if (inputNodeRef.current) {
+      inputNodeRef.current.disconnect();
+      inputNodeRef.current = null;
+    }
+    if (scriptProcessorRef.current) {
+      scriptProcessorRef.current.disconnect();
+      scriptProcessorRef.current = null;
+    }
+    if (audioContextRef.current) {
+      audioContextRef.current.close();
+      audioContextRef.current = null;
+    }
+
+    // 发送刷新指令获取最终结果
+    sendCommand("flush");
+  }, [sendCommand]);
+
+  // 切换录音状态（点击按钮）
+  const toggleRecording = useCallback(() => {
+    if (isRecordingRef.current) {
+      // 正在录音，停止
+      stopRecording();
+    } else {
+      // 未录音，开始
+      startRecording();
+    }
+  }, [stopRecording]);
+
+  // 取消录音并清空
+  const cancelRecording = useCallback(() => {
+    if (isRecordingRef.current) {
+      stopRecording();
+    }
+    setInputValue('');
+    setIsVoiceMode(false);
+  }, [stopRecording]);
+
+  // 格式化录音时长
+  const formatDuration = (seconds) => {
+    const mins = Math.floor(seconds / 60).toString().padStart(2, '0');
+    const secs = (seconds % 60).toString().padStart(2, '0');
+    return `${mins}:${secs}`;
+  };
+
   // ==================== 初始化 ====================
   useEffect(() => {
     setMessages([
@@ -75,7 +295,7 @@ const ChatInterface = () =&gt; {
         const data = await response.json();
         if (data.data) {
           setWelcomeContent(data.data);
-          setMessages(prev => prev.map(msg => 
+          setMessages(prev => prev.map(msg =>
             msg.id === 'welcome' ? { ...msg, content: data.data } : msg
           ));
         }
@@ -93,20 +313,22 @@ const ChatInterface = () =&gt; {
       }
       if (e.key === 'Escape') {
         setInputValue('');
-      }
-      if (e.key === 'ArrowUp' && inputValue === '') {
-        const lastUserMessage = chatHistory
-          .filter(item => item.role === 'user')
-          .pop();
-        if (lastUserMessage) {
-          setInputValue(lastUserMessage.content);
-          e.preventDefault();
+        if (isRecordingRef.current) {
+          stopRecording();
+          setIsVoiceMode(false);
         }
       }
     };
     document.addEventListener('keydown', handleKeyDown);
-    return () => document.removeEventListener('keydown', handleKeyDown);
+
+    return () => {
+      document.removeEventListener('keydown', handleKeyDown);
+      disconnectWebSocket();
+      if (recordingTimerRef.current) {
+        clearInterval(recordingTimerRef.current);
+      }
+    };
   }, []);
   useEffect(() => {
@@ -130,6 +352,12 @@ const ChatInterface = () =&gt; {
     const message = inputValue.trim();
     if (!message || isLoading) return;
+    // 如果正在录音，先停止
+    if (isRecordingRef.current) {
+      stopRecording();
+      setIsVoiceMode(false);
+    }
+
     let currentSessionId = sessionId;
     if (!currentSessionId) {
       currentSessionId = generateRandomString(20);
@@ -168,7 +396,7 @@ const ChatInterface = () =&gt; {
       }
       const data = await response.json();
-      
+
       if (data.data) {
         addMessage(data.data, 'ai');
         setChatHistory(prev => {
@@ -210,6 +438,11 @@ const ChatInterface = () =&gt; {
       ]);
       setChatHistory([]);
       setSessionId('');
+      setInputValue('');
+      if (isRecordingRef.current) {
+        stopRecording();
+        setIsVoiceMode(false);
+      }
     }
   };
@@ -220,12 +453,12 @@ const ChatInterface = () =&gt; {
   };
   const handleRegenerateMessage = (messageId, content) => {
-    setChatHistory(prev => prev.filter(item => 
+    setChatHistory(prev => prev.filter(item =>
       !(item.role === 'assistant' && item.content === content)
     ));
-    
+
     setMessages(prev => prev.filter(msg => msg.id !== messageId));
-    
+
     setInputValue(content);
     setTimeout(() => handleSendMessage(), 100);
   };
@@ -244,7 +477,7 @@ const ChatInterface = () =&gt; {
           <p>AI 印刷助手</p>
         </div>
         <div className="header-right">
-          <select 
+          <select
             className="model-selector"
             value={currentModel}
             onChange={handleModelChange}
@@ -252,7 +485,7 @@ const ChatInterface = () =&gt; {
             <option value="process">小羚羊印刷行业大模型</option>
             <option value="general">qwen2.5:14b</option>
           </select>
-          <button 
+          <button
             className="model-selectors"
             onClick={handleClearChat}
           >
@@ -267,14 +500,14 @@ const ChatInterface = () =&gt; {
           {/* 消息区域 */}
           <div className="messages-container">
             {messages.map((msg) => (
-              <div 
+              <div
                 key={msg.id}
                 className={`message ${msg.type}-message`}
               >
                 <div className={`message-bubble ${msg.type}-bubble`}>
                   <div className="message-content">
                     {msg.type === 'ai' ? (
-                      <ReactMarkdown 
+                      <ReactMarkdown
                         remarkPlugins={[remarkGfm]}
                         components={{
                           code: ({ node, inline, className, children, ...props }) => (
@@ -300,13 +533,13 @@ const ChatInterface = () =&gt; {
                     <span className="message-time">{msg.time}</span>
                     {msg.type === 'ai' && !msg.isWelcome && (
                       <div className="message-actions">
-                        <button 
+                        <button
                           className="action-btn"
                           onClick={() => handleCopyMessage(msg.content)}
                         >
                           复制
                         </button>
-                        <button 
+                        <button
                           className="action-btn"
                           onClick={() => handleRegenerateMessage(msg.id, msg.content)}
                         >
@@ -318,7 +551,7 @@ const ChatInterface = () =&gt; {
                 </div>
               </div>
             ))}
-            
+
             {/* 打字机效果 */}
             {isLoading && (
               <div className="message ai-message">
@@ -330,20 +563,46 @@ const ChatInterface = () =&gt; {
                 </div>
               </div>
             )}
-            
+
             <div ref={messagesEndRef} className="bottom-spacer" />
           </div>
           {/* 输入区域 */}
           <div className="input-section">
+            {/* 语音模式提示 - 仅在录音时显示 */}
+            {isRecording && (
+              <div className="voice-mode-indicator">
+                <div className="voice-wave">
+                  <span></span>
+                  <span></span>
+                  <span></span>
+                  <span></span>
+                  <span></span>
+                </div>
+                <span className="voice-text">
+                  正在录音 {formatDuration(recordingDuration)}
+                </span>
+                <button
+                  className="voice-cancel-btn"
+                  onClick={cancelRecording}
+                >
+                  取消
+                </button>
+              </div>
+            )}
+
             <div className="input-wrapper">
               <input
                 ref={inputRef}
                 type="text"
                 className="message-input"
-                placeholder="输入您的问题..."
+                placeholder={isRecording ? "正在听您说话..." : "输入您的问题..."}
                 value={inputValue}
-                onChange={(e) => setInputValue(e.target.value)}
+                onChange={(e) => {
+                  if (!isRecording) {
+                    setInputValue(e.target.value);
+                  }
+                }}
                 onKeyPress={(e) => {
                   if (e.key === 'Enter' && !e.shiftKey) {
                     e.preventDefault();
@@ -351,12 +610,32 @@ const ChatInterface = () =&gt; {
                   }
                 }}
                 disabled={isLoading}
+                readOnly={isRecording}
               />
-              <AudioOutline className='message-icon'/>
-              <button 
+
+              {/* 语音按钮 - 点击切换录音状态 */}
+              <button
+                className={`voice-button ${isRecording ? 'recording' : ''}`}
+                onClick={toggleRecording}
+                disabled={isLoading}
+              >
+                {isRecording ? (
+                  <>
+                    {/* <AudioFill className="voice-icon" /> */}
+                    <span className="voice-text">结束录音</span>
+                  </>
+                ) : (
+                  <>
+                    {/* <AudioOutline className="voice-icon" /> */}
+                    <span className="voice-text">点击录音</span>
+                  </>
+                )}
+              </button>
+
+              <button
                 className={`send-button ${isLoading ? 'disabled' : ''}`}
                 onClick={handleSendMessage}
-                disabled={isLoading}
+                disabled={isLoading || isRecording}
               >
                 发送
               </button>