feat(voice): 优化语音识别功能及增强可用性提示

- 新增语音服务可用性状态管理,动态检测浏览器支持情况
- 调整初始输入模式为键盘输入,避免误触启动语音识别
- 合并消息列表时保留已有调试消息,避免信息丢失
- VoiceInput组件新增调试日志回调,增强调试信息输出
- 细化语音识别状态机,区分idle、starting和listening状态
- 优化权限拒绝错误处理,针对iOS给出具体操作提示
- 改进语音识别启动流程,针对iOS增加软启动延迟避免冲突
- 支持多种指针和触摸事件管理,防止冲突导致的识别异常
- 按钮UI根据状态动态调整,显示不同提示文本和样式
- 禁用语音不可用时的按钮操作,避免误操作
- 增加语音识别启动超时保护,防止长时间挂起
- 优化用户交互体验,加入振动反馈及事件防抖处理
This commit is contained in:
2026-01-23 14:00:39 +08:00
parent ed60b8424f
commit 5367a4107e
2 changed files with 277 additions and 127 deletions

53
App.tsx
View File

@@ -121,7 +121,8 @@ const App: React.FC = () => {
const [isProcessing, setIsProcessing] = useState(false);
const [editingTransaction, setEditingTransaction] = useState<Transaction | null>(null);
const [isEditingFromChat, setIsEditingFromChat] = useState(false);
const [inputMode, setInputMode] = useState<'voice' | 'keyboard'>('voice');
const [inputMode, setInputMode] = useState<'voice' | 'keyboard'>('keyboard');
const [isVoiceServiceAvailable, setIsVoiceServiceAvailable] = useState(true);
const [token, setToken] = useState<string | null>(null);
const [userInfo, setUserInfo] = useState<any>(null);
@@ -151,6 +152,12 @@ const App: React.FC = () => {
useEffect(() => {
console.log('当前URL:', window.location.href);
// 检查浏览器是否支持语音识别
const SpeechRecognition = (window as any).SpeechRecognition || (window as any).webkitSpeechRecognition;
if (!SpeechRecognition) {
setIsVoiceServiceAvailable(false);
}
// 检查是否存在token,如果不存在则调用登录接口
checkAndLogin().then((userInfo) => {
// 登录成功后加载数据
@@ -264,33 +271,46 @@ const App: React.FC = () => {
relatedTransactionId: msg.transaction_id || msg.relatedTransactionId
}));
// 更新本地缓存
setMessages(messagesFromDB);
setMessages(prev => {
// 如果已经有消息了(比如调试消息),保留它们并合并
const existingDebugMessages = prev.filter(m => m.id.startsWith('debug-'));
return [...messagesFromDB, ...existingDebugMessages];
});
console.log('消息列表已从数据库加载:', messagesFromDB.length);
} else {
// 如果数据库中没有消息,显示初始欢迎消息
const aiName = aiSettings.name;
setMessages([{
setMessages(prev => {
const existingDebugMessages = prev.filter(m => m.id.startsWith('debug-'));
return [{
id: 'init',
role: 'assistant',
content: `我是${aiName}。✨\n想记账?快说!🎤`,
timestamp: Date.now()
}]);
}, ...existingDebugMessages];
});
}
} catch (error) {
console.error('加载消息列表失败:', error);
// 失败时尝试从本地存储加载
const storedMessages = localStorage.getItem('cui_messages');
if (storedMessages) {
setMessages(JSON.parse(storedMessages));
setMessages(prev => {
const existingDebugMessages = prev.filter(m => m.id.startsWith('debug-'));
return [...JSON.parse(storedMessages), ...existingDebugMessages];
});
} else {
// 显示初始欢迎消息
const aiName = aiSettings.name;
setMessages([{
setMessages(prev => {
const existingDebugMessages = prev.filter(m => m.id.startsWith('debug-'));
return [{
id: 'init',
role: 'assistant',
content: `我是${aiName}。✨\n想记账?快说!🎤`,
timestamp: Date.now()
}]);
}, ...existingDebugMessages];
});
}
}
@@ -482,6 +502,11 @@ const App: React.FC = () => {
}
};
const handleDebugLog = React.useCallback((content: string) => {
// 调试信息仅打印到控制台,不再添加到消息列表
console.log(`🔧 [调试信息] ${content}`);
}, []);
const handleSendMessage = React.useCallback(async (text: string, imageBase64?: string, mimeType?: string) => {
if (!text.trim() && !imageBase64) return;
@@ -799,7 +824,10 @@ const App: React.FC = () => {
<div className="absolute bottom-20 left-0 right-0 p-4 bg-gradient-to-t from-gray-100 via-gray-100 to-transparent z-40">
<div className="max-w-4xl mx-auto flex flex-col gap-3">
<div className="flex items-center gap-3">
<button onClick={() => setInputMode(inputMode === 'voice' ? 'keyboard' : 'voice')} className="p-3 bg-white text-gray-400 rounded-full shadow-lg hover:bg-gray-50 transition-all active:scale-90">
<button
onClick={() => setInputMode(inputMode === 'voice' ? 'keyboard' : 'voice')}
className="p-3 bg-white text-gray-400 rounded-full shadow-lg hover:bg-gray-50 transition-all active:scale-90"
>
{inputMode === 'voice' ? (
<svg className="w-6 h-6" fill="none" viewBox="0 0 24 24" stroke="currentColor"><path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M11 5H6a2 2 0 00-2 2v11a2 2 0 002 2h11a2 2 0 002-2v-5m-1.414-9.414a2 2 0 112.828 2.828L11.828 15H9v-2.828l8.586-8.586z" /></svg>
) : (
@@ -815,7 +843,14 @@ const App: React.FC = () => {
)}
</div>
) : (
<VoiceInput onTranscript={(text) => handleSendMessage(text)} isProcessing={isProcessing} isFullWidth />
<VoiceInput
onTranscript={(text) => handleSendMessage(text)}
isProcessing={isProcessing}
isFullWidth
onDebug={handleDebugLog}
onAvailabilityChange={(isAvailable) => setIsVoiceServiceAvailable(isAvailable)}
isServiceAvailable={isVoiceServiceAvailable}
/>
)}
<input type="file" accept="image/*" className="hidden" ref={fileInputRef} onChange={handleFileUpload} />

View File

@@ -1,47 +1,74 @@
import React, { useState, useEffect, useRef } from 'react';
import React, { useState, useEffect, useRef, useCallback } from 'react';
interface VoiceInputProps {
onTranscript: (text: string) => void;
isProcessing: boolean;
isFullWidth?: boolean;
onDebug?: (message: string) => void;
onAvailabilityChange?: (isAvailable: boolean) => void;
isServiceAvailable?: boolean;
}
const VoiceInput: React.FC<VoiceInputProps> = ({ onTranscript, isProcessing, isFullWidth = false }) => {
const [isListening, setIsListening] = useState(false);
const recognitionRef = useRef<any>(null);
const isStartingRef = useRef(false);
const onTranscriptRef = useRef(onTranscript);
const VoiceInput: React.FC<VoiceInputProps> = ({
onTranscript,
isProcessing,
isFullWidth = false,
onDebug,
onAvailabilityChange,
isServiceAvailable = true
}) => {
const [voiceState, setVoiceState] = useState<'idle' | 'starting' | 'listening'>('idle');
const [isInternalDenied, setIsInternalDenied] = useState(false);
// 始终保持 ref 为最新,避免在 useEffect 中依赖 onTranscript 导致重复初始化
const isServiceDenied = !isServiceAvailable || isInternalDenied;
const recognitionRef = useRef<any>(null);
const isHoldingRef = useRef(false);
const startTimestampRef = useRef(0);
const startTimeoutRef = useRef<NodeJS.Timeout | null>(null);
const lastEventTimeRef = useRef(0);
const activeEventRef = useRef<'pointer' | 'touch' | null>(null);
const onTranscriptRef = useRef(onTranscript);
useEffect(() => {
onTranscriptRef.current = onTranscript;
}, [onTranscript]);
const initRecognition = () => {
if (!('SpeechRecognition' in window || 'webkitSpeechRecognition' in window)) return;
// 初始化语音识别实例
const initRecognition = useCallback(() => {
if (typeof window === 'undefined') return null;
// 如果已有实例,先清理
const SpeechRecognition = (window as any).SpeechRecognition || (window as any).webkitSpeechRecognition;
if (!SpeechRecognition) {
onAvailabilityChange?.(false);
return null;
}
// 彻底清理并销毁旧实例,这是 iOS 修复 service-not-allowed 的关键
if (recognitionRef.current) {
try {
recognitionRef.current.onstart = null;
recognitionRef.current.onend = null;
recognitionRef.current.onerror = null;
recognitionRef.current.onresult = null;
recognitionRef.current.abort();
} catch (e) {}
}
const SpeechRecognition = (window as any).SpeechRecognition || (window as any).webkitSpeechRecognition;
const recognition = new SpeechRecognition();
recognition.continuous = true;
recognition.lang = 'zh-CN';
recognition.continuous = false;
// 针对 iOS 优化语言标识符
recognition.lang = 'zh-Hans-CN';
recognition.interimResults = true;
recognition.onstart = () => {
setIsListening(true);
isStartingRef.current = false;
console.log('Speech recognition started');
if (startTimeoutRef.current) clearTimeout(startTimeoutRef.current);
setVoiceState('listening');
onDebug?.('DEBUG: recognition.onstart 触发');
};
recognition.onresult = (event: any) => {
const results = event.results;
onDebug?.(`DEBUG: recognition.onresult 触发, isFinal: ${results[event.resultIndex]?.isFinal}`);
for (let i = event.resultIndex; i < results.length; i++) {
if (results[i].isFinal) {
const transcript = results[i][0].transcript;
@@ -51,55 +78,54 @@ const VoiceInput: React.FC<VoiceInputProps> = ({ onTranscript, isProcessing, isF
};
recognition.onerror = async (event: any) => {
console.error('Speech recognition error', event.error);
if (startTimeoutRef.current) clearTimeout(startTimeoutRef.current);
const errorType = event.error;
onDebug?.(`DEBUG: recognition.onerror 触发, error: ${errorType}`);
let message = '';
switch (event.error) {
case 'network':
message = "语音识别网络连接失败。请检查网络或尝试切换浏览器(推荐 Chrome/Edge)。";
break;
case 'not-allowed':
case 'service-not-allowed':
// 移动端初次授权时可能触发 race condition:用户点击“允许”但 start() 已超时失败
// 此时我们通过 getUserMedia 探测真实权限状态
setVoiceState('idle');
isHoldingRef.current = false;
activeEventRef.current = null;
if (errorType === 'service-not-allowed' || errorType === 'not-allowed') {
setIsInternalDenied(true);
onAvailabilityChange?.(false);
if (errorType === 'service-not-allowed') {
const isIOS = /iPad|iPhone|iPod/.test(navigator.userAgent);
if (isIOS) {
alert("语音识别服务不可用。\n\n解决办法:\n1. 确保您的 iPhone 已开启“听写”功能(设置 -> 通用 -> 键盘 -> 启用听写)\n2. 确保您的 Siri 已开启\n3. 检查网络连接是否正常。");
} else {
alert("语音识别服务被拒绝,请稍后重试。");
}
} else {
// not-allowed 处理
onDebug?.(`DEBUG: 检测到错误 not-allowed,尝试探测麦克风硬件...`);
try {
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
// 如果能获取到流,说明权限其实是有的,只是刚才 start() 冲突了
stream.getTracks().forEach(t => t.stop());
console.log('Permission probe successful, refreshing recognition instance...');
isStartingRef.current = false;
setIsListening(false);
initRecognition(); // 静默重置实例
return; // 不弹窗,用户再次点击就会成功
} catch (e) {
// 探测也失败,说明是真的被拒绝了
message = "麦克风访问被拒绝。\n\n解决办法:\n1. 点击地址栏左侧的“锁”或“设置”图标\n2. 找到“麦克风”并设为“允许”\n3. 刷新页面后重试";
onDebug?.('DEBUG: 麦克风硬件探测成功');
alert("麦克风访问被拒绝。\n\n请在浏览器设置中允许麦克风权限后重试。");
} catch (e: any) {
onDebug?.(`DEBUG: 麦克风硬件探测失败: ${e.message || e.name}`);
alert("麦克风访问被拒绝,且硬件探测失败。\n\n请检查设备设置并确保浏览器拥有麦克风权限。");
}
}
break;
case 'no-speech':
return;
case 'aborted':
return;
default:
break;
}
if (message) alert(message);
setIsListening(false);
isStartingRef.current = false;
// 出错后重新初始化,确保下次点击可用
initRecognition();
};
recognition.onend = () => {
console.log('Speech recognition ended');
setIsListening(false);
isStartingRef.current = false;
if (startTimeoutRef.current) clearTimeout(startTimeoutRef.current);
setVoiceState('idle');
onDebug?.('DEBUG: recognition.onend 触发');
if (isHoldingRef.current) {
isHoldingRef.current = false;
activeEventRef.current = null;
}
};
recognitionRef.current = recognition;
};
return recognition;
}, [onDebug]);
useEffect(() => {
initRecognition();
@@ -110,99 +136,188 @@ const VoiceInput: React.FC<VoiceInputProps> = ({ onTranscript, isProcessing, isF
} catch (e) {}
}
};
}, []);
}, [initRecognition]);
const startListening = (e: React.PointerEvent) => {
e.stopPropagation();
if (e.cancelable) e.preventDefault();
const stopListening = useCallback((e: React.PointerEvent | React.TouchEvent | React.MouseEvent) => {
const eventType = e.type.startsWith('touch') ? 'touch' : 'pointer';
onDebug?.(`DEBUG: stopListening 触发, event: ${e.type}, state: ${voiceState}, isHolding: ${isHoldingRef.current}`);
// 如果已经在处理中、正在监听或正在启动,则忽略
if (isProcessing || isListening || isStartingRef.current) return;
// 获取指针捕获,确保即使手指移出按钮范围也能触发 onPointerUp
try {
(e.currentTarget as HTMLElement).setPointerCapture(e.pointerId);
} catch (err) {
console.warn("Failed to set pointer capture", err);
}
if (recognitionRef.current) {
// 检查是否为安全上下文 (HTTPS 或 localhost)
if (!window.isSecureContext) {
alert("语音识别失败:浏览器限制在非 HTTPS 环境下使用麦克风。请确保使用 HTTPS 访问,或在 PC 端 Chrome 的 'Insecure origins treated as secure' 中设置白名单。");
// 如果不是当前激活的事件类型,忽略(除非已经没有激活类型了)
if (activeEventRef.current && activeEventRef.current !== eventType) {
return;
}
try {
isStartingRef.current = true;
recognitionRef.current.start();
// UI 状态会在 onstart 回调中更新,以保证真实感
} catch (err) {
console.error("Failed to start recognition", err);
isStartingRef.current = false;
setIsListening(false);
}
} else {
if (!window.isSecureContext) {
alert("浏览器不支持语音识别:当前环境不是安全上下文 (HTTPS)。");
} else {
alert("您的浏览器不支持语音识别,请使用 Chrome 或 Edge。");
}
}
};
isHoldingRef.current = false;
activeEventRef.current = null;
const stopListening = (e: React.PointerEvent) => {
if (startTimeoutRef.current) {
clearTimeout(startTimeoutRef.current);
startTimeoutRef.current = null;
}
// 处理极短时间的点击,防止闪现
const duration = Date.now() - startTimestampRef.current;
if (duration < 300) {
// 如果点击时间太短,稍微延迟停止,或者直接停止
setTimeout(() => {
try { recognitionRef.current?.stop(); } catch (e) {}
}, 300 - duration);
} else {
try { recognitionRef.current?.stop(); } catch (e) {}
}
}, [voiceState, onDebug]);
const startListening = useCallback((e: React.PointerEvent | React.TouchEvent | React.MouseEvent) => {
if (e.cancelable) e.preventDefault();
e.stopPropagation();
// 只有在正在运行或正在启动时才尝试停止
if (isListening || isStartingRef.current) {
const eventType = e.type.startsWith('touch') ? 'touch' : 'pointer';
onDebug?.(`DEBUG: startListening 触发, type: ${e.type}, currentActive: ${activeEventRef.current}`);
if (activeEventRef.current && activeEventRef.current !== eventType) {
onDebug?.(`DEBUG: 忽略冲突事件类型: ${eventType}`);
return;
}
const now = Date.now();
if (now - lastEventTimeRef.current < 400) return; // 略微增加防抖
lastEventTimeRef.current = now;
if (isProcessing || voiceState !== 'idle' || isHoldingRef.current || isServiceDenied) return;
activeEventRef.current = eventType;
isHoldingRef.current = true;
startTimestampRef.current = now;
if ('vibrate' in navigator) {
try { navigator.vibrate(20); } catch (e) {}
}
if (!window.isSecureContext) {
alert("语音识别失败:请使用 HTTPS 协议访问以开启麦克风权限。");
isHoldingRef.current = false;
activeEventRef.current = null;
return;
}
// 核心修复:针对 iOS,每次点击都强制获取一个全新的实例
onDebug?.('DEBUG: 正在获取全新识别实例...');
const recognition = initRecognition();
if (recognition) {
try {
recognitionRef.current?.stop();
} catch (err) {
console.error("Failed to stop recognition", err);
// 发生错误时强制重置本地状态
setIsListening(false);
isStartingRef.current = false;
// 指针捕获
if ('pointerId' in e && (e.currentTarget as any).setPointerCapture) {
try { (e.currentTarget as any).setPointerCapture(e.pointerId); } catch (err) {}
}
setVoiceState('starting');
// 核心修复:针对 iOS 增加 100ms 软启动延迟,避免手势冲突
setTimeout(() => {
if (isHoldingRef.current) {
onDebug?.('DEBUG: 延时结束,调用 recognition.start()');
try {
recognition.start();
} catch (err: any) {
onDebug?.(`DEBUG: 异步 start 失败: ${err.message || err}`);
setVoiceState('idle');
isHoldingRef.current = false;
activeEventRef.current = null;
}
}
};
}, 100);
startTimeoutRef.current = setTimeout(() => {
if (isHoldingRef.current && voiceState === 'starting') {
onDebug?.("DEBUG: 启动超时保护触发");
stopListening(e);
}
}, 5000);
} catch (err: any) {
onDebug?.(`DEBUG: recognition.start 抛出异常: ${err.message || err}`);
setVoiceState('idle');
isHoldingRef.current = false;
activeEventRef.current = null;
}
} else {
alert("您的浏览器不支持语音识别,请尝试使用 Chrome 或 Edge。");
isHoldingRef.current = false;
activeEventRef.current = null;
}
}, [isProcessing, voiceState, initRecognition, onDebug, stopListening]);
// 处理指针/触摸离开或取消的情况
const handleCancel = useCallback((e: any) => {
onDebug?.(`DEBUG: handleCancel 触发, event: ${e.type}, isHolding: ${isHoldingRef.current}`);
if (isHoldingRef.current) {
stopListening(e);
}
}, [stopListening, onDebug]);
return (
<div className={`${isFullWidth ? 'w-full' : ''} relative select-none touch-none`}>
<button
onPointerDown={startListening}
onPointerUp={stopListening}
onPointerCancel={stopListening}
onClick={(e) => e.preventDefault()}
onPointerCancel={handleCancel}
// 兼容性备选
onTouchStart={startListening}
onTouchEnd={stopListening}
onTouchCancel={handleCancel}
// 禁用右键菜单,防止长按弹出
onContextMenu={(e) => e.preventDefault()}
disabled={isProcessing}
style={{ WebkitTouchCallout: 'none' }}
className={`relative flex items-center justify-center transition-all duration-300 active:scale-95 shadow-lg ${
disabled={isProcessing || isServiceDenied}
className={`relative flex items-center justify-center transition-all duration-300 shadow-lg ${
isFullWidth
? 'w-full py-4 rounded-2xl font-bold text-sm tracking-widest'
: 'w-14 h-14 rounded-full'
} ${
isListening
voiceState === 'listening'
? 'bg-rose-500 text-white shadow-rose-200 scale-105'
: voiceState === 'starting'
? 'bg-amber-500 text-white shadow-amber-100 scale-102'
: isServiceDenied
? 'bg-gray-300 text-gray-500 cursor-not-allowed shadow-none pointer-events-none'
: 'bg-indigo-600 text-white shadow-indigo-100'
} ${isProcessing ? 'opacity-50 grayscale cursor-not-allowed' : ''}`}
style={{
WebkitTouchCallout: 'none',
WebkitTapHighlightColor: 'transparent',
userSelect: 'none',
WebkitUserSelect: 'none',
touchAction: 'none'
}}
>
{isListening && (
<div className="absolute inset-0 rounded-2xl animate-ping bg-rose-400 opacity-20"></div>
{(voiceState === 'listening' || voiceState === 'starting') && (
<div className={`absolute inset-0 rounded-2xl animate-ping ${voiceState === 'listening' ? 'bg-rose-400' : 'bg-amber-400'} opacity-20`}></div>
)}
<div className="flex items-center gap-2 z-10">
<svg xmlns="http://www.w3.org/2000/svg" className={`${isFullWidth ? 'h-5 w-5' : 'h-6 w-6'}`} fill="none" viewBox="0 0 24 24" stroke="currentColor">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2.5} d="M19 11a7 7 0 01-7 7m0 0a7 7 0 01-7-7m7 7v4m0 0H8m4 0h4m-4-8a3 3 0 01-3-3V5a3 3 0 116 0v6a3 3 0 01-3 3z" />
</svg>
{isFullWidth && <span>{isListening ? '松开 结束' : '按住 说话'}</span>}
{isFullWidth && (
<span>
{isServiceDenied
? '语音不可用'
: voiceState === 'listening'
? '松开 结束'
: voiceState === 'starting'
? '正在连接...'
: '按住 说话'}
</span>
)}
</div>
</button>
{isListening && isFullWidth && (
{voiceState !== 'idle' && !isServiceDenied && isFullWidth && (
<div className="absolute -top-16 left-0 right-0 flex justify-center animate-bounce">
<div className="bg-rose-500 text-white px-4 py-2 rounded-full text-xs font-bold shadow-xl border-2 border-white">
正在录音...
<div className={`${voiceState === 'listening' ? 'bg-rose-500' : 'bg-amber-500'} text-white px-4 py-2 rounded-full text-xs font-bold shadow-xl border-2 border-white`}>
{voiceState === 'listening' ? '正在录音...' : '准备录音...'}
</div>
</div>
)}