xlyErp / xlyUmi

Commit c6ddce2bbce26ddd61aeece6a9a30c6fc73e14b1

Authored by chenxt
1 parent 8acabc8a

ai语音输入

Inline Side-by-side
Showing 1 changed file with 310 additions and 31 deletions
src/mobile/Ai/newAi.jsx
  View file @c6ddce2
1   -import React, { useState, useEffect, useRef } from 'react';
  1 +import React, { useState, useEffect, useRef, useCallback } from 'react';
2 2 import ReactMarkdown from 'react-markdown';
3 3 import remarkGfm from 'remark-gfm';
4   -import { AudioOutline } from "antd-mobile-icons";
5   -import './AiChatStyles.less'; // 引入外部样式文件
  4 +import { AudioOutline, AudioFill } from "antd-mobile-icons";
  5 +import './AiChatStyles.less';
6 6  
7 7 const ChatInterface = () => {
8 8 // ==================== 状态管理 ====================
... ... @@ -15,13 +15,27 @@ const ChatInterface = () => {
15 15 const [currentModel, setCurrentModel] = useState('general');
16 16 const [chatHistory, setChatHistory] = useState([]);
17 17 const [welcomeContent, setWelcomeContent] = useState('');
18   -
  18 +
  19 + // 语音输入状态
  20 + const [isRecording, setIsRecording] = useState(false);
  21 + const [isWsConnected, setIsWsConnected] = useState(false);
  22 + const [isVoiceMode, setIsVoiceMode] = useState(false);
  23 + const [recordingDuration, setRecordingDuration] = useState(0);
  24 +
19 25 const messagesEndRef = useRef(null);
20 26 const inputRef = useRef(null);
  27 + const wsRef = useRef(null);
  28 + const audioContextRef = useRef(null);
  29 + const scriptProcessorRef = useRef(null);
  30 + const inputNodeRef = useRef(null);
  31 + const recordingTimerRef = useRef(null);
  32 + const isRecordingRef = useRef(false);
21 33  
22 34 // ==================== 配置 ====================
23 35 const CONFIG = {
24 36 backendUrl: 'http://localhost:8099/xlyAi',
  37 + wsUrl: 'ws://121.43.128.225:10096', // 语音识别WebSocket地址
  38 + sampleRate: 16000,
25 39 endpoints: {
26 40 chat: '/api/v1/chat/query',
27 41 process: '/api/v1/chat/query',
... ... @@ -52,6 +66,212 @@ const ChatInterface = () => {
52 66 messagesEndRef.current?.scrollIntoView({ behavior: 'smooth' });
53 67 };
54 68  
  69 + // ==================== WebSocket 语音识别 ====================
  70 +
  71 + // 连接语音识别WebSocket
  72 + const connectWebSocket = useCallback(() => {
  73 + if (wsRef.current && wsRef.current.readyState === WebSocket.OPEN) {
  74 + return;
  75 + }
  76 +
  77 + const ws = new WebSocket(CONFIG.wsUrl);
  78 + ws.binaryType = "arraybuffer";
  79 +
  80 + ws.onopen = () => {
  81 + console.log("语音识别WebSocket连接成功");
  82 + setIsWsConnected(true);
  83 + };
  84 +
  85 + ws.onmessage = (event) => {
  86 + try {
  87 + const res = JSON.parse(event.data);
  88 + if (res.code === 0 && (res.msg === "success" || res.msg === "flush_success")) {
  89 + if (res.text && res.text.trim()) {
  90 + setInputValue(prev => {
  91 + const separator = prev && !prev.endsWith(' ') ? ' ' : '';
  92 + const newValue = prev ? `${prev}${separator}${res.text}` : res.text;
  93 + console.log('语音识别结果:', res.text, '更新后:', newValue);
  94 + return newValue;
  95 + });
  96 + }
  97 + }
  98 + } catch (e) {
  99 + console.error("WebSocket消息解析失败:", e);
  100 + }
  101 + };
  102 +
  103 + ws.onclose = () => {
  104 + console.log("语音识别WebSocket连接断开");
  105 + setIsWsConnected(false);
  106 + if (isRecordingRef.current) {
  107 + stopRecording();
  108 + }
  109 + };
  110 +
  111 + ws.onerror = (err) => {
  112 + console.error("WebSocket错误:", err);
  113 + setIsWsConnected(false);
  114 + };
  115 +
  116 + wsRef.current = ws;
  117 + }, []);
  118 +
  119 + // 断开WebSocket
  120 + const disconnectWebSocket = useCallback(() => {
  121 + if (wsRef.current) {
  122 + wsRef.current.close();
  123 + wsRef.current = null;
  124 + }
  125 + setIsWsConnected(false);
  126 + }, []);
  127 +
  128 + // 发送指令
  129 + const sendCommand = useCallback((action) => {
  130 + if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) {
  131 + return;
  132 + }
  133 + const cmd = JSON.stringify({ action });
  134 + wsRef.current.send(cmd);
  135 + }, []);
  136 +
  137 + // Float32 转 Int16 PCM
  138 + const float32ToInt16 = (float32Array) => {
  139 + const int16Array = new Int16Array(float32Array.length);
  140 + for (let i = 0; i < float32Array.length; i++) {
  141 + let s = Math.max(-1, Math.min(1, float32Array[i]));
  142 + int16Array[i] = s < 0 ? s * 0x8000 : s * 0x7FFF;
  143 + }
  144 + return new Uint8Array(int16Array.buffer);
  145 + };
  146 +
  147 + // 音频重采样
  148 + const resampleAudio = (data, originalRate, targetRate) => {
  149 + if (originalRate === targetRate) return data;
  150 + const ratio = targetRate / originalRate;
  151 + const newLength = Math.round(data.length * ratio);
  152 + const result = new Float32Array(newLength);
  153 + for (let i = 0; i < newLength; i++) {
  154 + result[i] = data[Math.round(i / ratio)] || 0;
  155 + }
  156 + return result;
  157 + };
  158 +
  159 + // 开始录音
  160 + const startRecording = async () => {
  161 + if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
  162 + alert("浏览器不支持麦克风采集");
  163 + return;
  164 + }
  165 +
  166 + try {
  167 + // 先连接WebSocket
  168 + if (!isWsConnected) {
  169 + connectWebSocket();
  170 + await new Promise(resolve => setTimeout(resolve, 1000));
  171 + }
  172 +
  173 + // 检查 WebSocket 是否已连接
  174 + if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) {
  175 + alert("语音识别服务未连接，请重试");
  176 + return;
  177 + }
  178 +
  179 + const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
  180 + audioContextRef.current = new (window.AudioContext || window.webkitAudioContext)();
  181 + inputNodeRef.current = audioContextRef.current.createMediaStreamSource(stream);
  182 + scriptProcessorRef.current = audioContextRef.current.createScriptProcessor(2048, 1, 1);
  183 +
  184 + // 使用 ref 来检查录音状态，避免闭包问题
  185 + scriptProcessorRef.current.onaudioprocess = (event) => {
  186 + if (!isRecordingRef.current) return;
  187 + const inputData = event.inputBuffer.getChannelData(0);
  188 + const resampledData = resampleAudio(inputData, audioContextRef.current.sampleRate, CONFIG.sampleRate);
  189 + const pcmData = float32ToInt16(resampledData);
  190 + if (wsRef.current && wsRef.current.readyState === WebSocket.OPEN) {
  191 + wsRef.current.send(pcmData);
  192 + }
  193 + };
  194 +
  195 + inputNodeRef.current.connect(scriptProcessorRef.current);
  196 + scriptProcessorRef.current.connect(audioContextRef.current.destination);
  197 +
  198 + // 关键：先设置 ref，再设置 state
  199 + isRecordingRef.current = true;
  200 + setIsRecording(true);
  201 + setIsVoiceMode(true);
  202 + setRecordingDuration(0);
  203 +
  204 + // 开始计时
  205 + recordingTimerRef.current = setInterval(() => {
  206 + setRecordingDuration(prev => prev + 1);
  207 + }, 1000);
  208 +
  209 + console.log("录音已开始");
  210 +
  211 + } catch (e) {
  212 + console.error("录音启动失败:", e);
  213 + alert("录音启动失败：" + e.message);
  214 + isRecordingRef.current = false;
  215 + setIsRecording(false);
  216 + }
  217 + };
  218 +
  219 + // 停止录音
  220 + const stopRecording = useCallback(() => {
  221 + console.log("停止录音");
  222 + isRecordingRef.current = false;
  223 + setIsRecording(false);
  224 + setIsVoiceMode(false);
  225 +
  226 + if (recordingTimerRef.current) {
  227 + clearInterval(recordingTimerRef.current);
  228 + recordingTimerRef.current = null;
  229 + }
  230 +
  231 + if (inputNodeRef.current) {
  232 + inputNodeRef.current.disconnect();
  233 + inputNodeRef.current = null;
  234 + }
  235 + if (scriptProcessorRef.current) {
  236 + scriptProcessorRef.current.disconnect();
  237 + scriptProcessorRef.current = null;
  238 + }
  239 + if (audioContextRef.current) {
  240 + audioContextRef.current.close();
  241 + audioContextRef.current = null;
  242 + }
  243 +
  244 + // 发送刷新指令获取最终结果
  245 + sendCommand("flush");
  246 + }, [sendCommand]);
  247 +
  248 + // 切换录音状态（点击按钮）
  249 + const toggleRecording = useCallback(() => {
  250 + if (isRecordingRef.current) {
  251 + // 正在录音，停止
  252 + stopRecording();
  253 + } else {
  254 + // 未录音，开始
  255 + startRecording();
  256 + }
  257 + }, [stopRecording]);
  258 +
  259 + // 取消录音并清空
  260 + const cancelRecording = useCallback(() => {
  261 + if (isRecordingRef.current) {
  262 + stopRecording();
  263 + }
  264 + setInputValue('');
  265 + setIsVoiceMode(false);
  266 + }, [stopRecording]);
  267 +
  268 + // 格式化录音时长
  269 + const formatDuration = (seconds) => {
  270 + const mins = Math.floor(seconds / 60).toString().padStart(2, '0');
  271 + const secs = (seconds % 60).toString().padStart(2, '0');
  272 + return `${mins}:${secs}`;
  273 + };
  274 +
55 275 // ==================== 初始化 ====================
56 276 useEffect(() => {
57 277 setMessages([
... ... @@ -75,7 +295,7 @@ const ChatInterface = () =&gt; {
75 295 const data = await response.json();
76 296 if (data.data) {
77 297 setWelcomeContent(data.data);
78   - setMessages(prev => prev.map(msg =>
  298 + setMessages(prev => prev.map(msg =>
79 299 msg.id === 'welcome' ? { ...msg, content: data.data } : msg
80 300 ));
81 301 }
... ... @@ -93,20 +313,22 @@ const ChatInterface = () =&gt; {
93 313 }
94 314 if (e.key === 'Escape') {
95 315 setInputValue('');
96   - }
97   - if (e.key === 'ArrowUp' && inputValue === '') {
98   - const lastUserMessage = chatHistory
99   - .filter(item => item.role === 'user')
100   - .pop();
101   - if (lastUserMessage) {
102   - setInputValue(lastUserMessage.content);
103   - e.preventDefault();
  316 + if (isRecordingRef.current) {
  317 + stopRecording();
  318 + setIsVoiceMode(false);
104 319 }
105 320 }
106 321 };
107 322  
108 323 document.addEventListener('keydown', handleKeyDown);
109   - return () => document.removeEventListener('keydown', handleKeyDown);
  324 +
  325 + return () => {
  326 + document.removeEventListener('keydown', handleKeyDown);
  327 + disconnectWebSocket();
  328 + if (recordingTimerRef.current) {
  329 + clearInterval(recordingTimerRef.current);
  330 + }
  331 + };
110 332 }, []);
111 333  
112 334 useEffect(() => {
... ... @@ -130,6 +352,12 @@ const ChatInterface = () =&gt; {
130 352 const message = inputValue.trim();
131 353 if (!message || isLoading) return;
132 354  
  355 + // 如果正在录音，先停止
  356 + if (isRecordingRef.current) {
  357 + stopRecording();
  358 + setIsVoiceMode(false);
  359 + }
  360 +
133 361 let currentSessionId = sessionId;
134 362 if (!currentSessionId) {
135 363 currentSessionId = generateRandomString(20);
... ... @@ -168,7 +396,7 @@ const ChatInterface = () =&gt; {
168 396 }
169 397  
170 398 const data = await response.json();
171   -
  399 +
172 400 if (data.data) {
173 401 addMessage(data.data, 'ai');
174 402 setChatHistory(prev => {
... ... @@ -210,6 +438,11 @@ const ChatInterface = () =&gt; {
210 438 ]);
211 439 setChatHistory([]);
212 440 setSessionId('');
  441 + setInputValue('');
  442 + if (isRecordingRef.current) {
  443 + stopRecording();
  444 + setIsVoiceMode(false);
  445 + }
213 446 }
214 447 };
215 448  
... ... @@ -220,12 +453,12 @@ const ChatInterface = () =&gt; {
220 453 };
221 454  
222 455 const handleRegenerateMessage = (messageId, content) => {
223   - setChatHistory(prev => prev.filter(item =>
  456 + setChatHistory(prev => prev.filter(item =>
224 457 !(item.role === 'assistant' && item.content === content)
225 458 ));
226   -
  459 +
227 460 setMessages(prev => prev.filter(msg => msg.id !== messageId));
228   -
  461 +
229 462 setInputValue(content);
230 463 setTimeout(() => handleSendMessage(), 100);
231 464 };
... ... @@ -244,7 +477,7 @@ const ChatInterface = () =&gt; {
244 477 <p>AI 印刷助手</p>
245 478 </div>
246 479 <div className="header-right">
247   - <select
  480 + <select
248 481 className="model-selector"
249 482 value={currentModel}
250 483 onChange={handleModelChange}
... ... @@ -252,7 +485,7 @@ const ChatInterface = () =&gt; {
252 485 <option value="process">小羚羊印刷行业大模型</option>
253 486 <option value="general">qwen2.5:14b</option>
254 487 </select>
255   - <button
  488 + <button
256 489 className="model-selectors"
257 490 onClick={handleClearChat}
258 491 >
... ... @@ -267,14 +500,14 @@ const ChatInterface = () =&gt; {
267 500 {/* 消息区域 */}
268 501 <div className="messages-container">
269 502 {messages.map((msg) => (
270   - <div
  503 + <div
271 504 key={msg.id}
272 505 className={`message ${msg.type}-message`}
273 506 >
274 507 <div className={`message-bubble ${msg.type}-bubble`}>
275 508 <div className="message-content">
276 509 {msg.type === 'ai' ? (
277   - <ReactMarkdown
  510 + <ReactMarkdown
278 511 remarkPlugins={[remarkGfm]}
279 512 components={{
280 513 code: ({ node, inline, className, children, ...props }) => (
... ... @@ -300,13 +533,13 @@ const ChatInterface = () =&gt; {
300 533 <span className="message-time">{msg.time}</span>
301 534 {msg.type === 'ai' && !msg.isWelcome && (
302 535 <div className="message-actions">
303   - <button
  536 + <button
304 537 className="action-btn"
305 538 onClick={() => handleCopyMessage(msg.content)}
306 539 >
307 540 复制
308 541 </button>
309   - <button
  542 + <button
310 543 className="action-btn"
311 544 onClick={() => handleRegenerateMessage(msg.id, msg.content)}
312 545 >
... ... @@ -318,7 +551,7 @@ const ChatInterface = () =&gt; {
318 551 </div>
319 552 </div>
320 553 ))}
321   -
  554 +
322 555 {/* 打字机效果 */}
323 556 {isLoading && (
324 557 <div className="message ai-message">
... ... @@ -330,20 +563,46 @@ const ChatInterface = () =&gt; {
330 563 </div>
331 564 </div>
332 565 )}
333   -
  566 +
334 567 <div ref={messagesEndRef} className="bottom-spacer" />
335 568 </div>
336 569  
337 570 {/* 输入区域 */}
338 571 <div className="input-section">
  572 + {/* 语音模式提示 - 仅在录音时显示 */}
  573 + {isRecording && (
  574 + <div className="voice-mode-indicator">
  575 + <div className="voice-wave">
  576 + <span></span>
  577 + <span></span>
  578 + <span></span>
  579 + <span></span>
  580 + <span></span>
  581 + </div>
  582 + <span className="voice-text">
  583 + 正在录音 {formatDuration(recordingDuration)}
  584 + </span>
  585 + <button
  586 + className="voice-cancel-btn"
  587 + onClick={cancelRecording}
  588 + >
  589 + 取消
  590 + </button>
  591 + </div>
  592 + )}
  593 +
339 594 <div className="input-wrapper">
340 595 <input
341 596 ref={inputRef}
342 597 type="text"
343 598 className="message-input"
344   - placeholder="输入您的问题..."
  599 + placeholder={isRecording ? "正在听您说话..." : "输入您的问题..."}
345 600 value={inputValue}
346   - onChange={(e) => setInputValue(e.target.value)}
  601 + onChange={(e) => {
  602 + if (!isRecording) {
  603 + setInputValue(e.target.value);
  604 + }
  605 + }}
347 606 onKeyPress={(e) => {
348 607 if (e.key === 'Enter' && !e.shiftKey) {
349 608 e.preventDefault();
... ... @@ -351,12 +610,32 @@ const ChatInterface = () =&gt; {
351 610 }
352 611 }}
353 612 disabled={isLoading}
  613 + readOnly={isRecording}
354 614 />
355   - <AudioOutline className='message-icon'/>
356   - <button
  615 +
  616 + {/* 语音按钮 - 点击切换录音状态 */}
  617 + <button
  618 + className={`voice-button ${isRecording ? 'recording' : ''}`}
  619 + onClick={toggleRecording}
  620 + disabled={isLoading}
  621 + >
  622 + {isRecording ? (
  623 + <>
  624 + {/* <AudioFill className="voice-icon" /> */}
  625 + <span className="voice-text">结束录音</span>
  626 + </>
  627 + ) : (
  628 + <>
  629 + {/* <AudioOutline className="voice-icon" /> */}
  630 + <span className="voice-text">点击录音</span>
  631 + </>
  632 + )}
  633 + </button>
  634 +
  635 + <button
357 636 className={`send-button ${isLoading ? 'disabled' : ''}`}
358 637 onClick={handleSendMessage}
359   - disabled={isLoading}
  638 + disabled={isLoading || isRecording}
360 639 >
361 640 发送
362 641 </button>
... ...