对语音接口进行了简略的异常处理
This commit is contained in:
parent
75016e3009
commit
3c60c9a418
|
|
@ -63,22 +63,25 @@ def parseChunkDelta(chunk):
|
||||||
|
|
||||||
#断句函数
|
#断句函数
|
||||||
def split_string_with_punctuation(current_sentence,text,is_first,is_end):
|
def split_string_with_punctuation(current_sentence,text,is_first,is_end):
|
||||||
result = []
|
try:
|
||||||
if is_end:
|
result = []
|
||||||
if current_sentence:
|
if is_end:
|
||||||
result.append(current_sentence)
|
if current_sentence:
|
||||||
current_sentence = ''
|
result.append(current_sentence)
|
||||||
|
current_sentence = ''
|
||||||
|
return result, current_sentence, is_first
|
||||||
|
for char in text:
|
||||||
|
current_sentence += char
|
||||||
|
if is_first and char in ',.?!，。？！':
|
||||||
|
result.append(current_sentence)
|
||||||
|
current_sentence = ''
|
||||||
|
is_first = False
|
||||||
|
elif char in '。？！':
|
||||||
|
result.append(current_sentence)
|
||||||
|
current_sentence = ''
|
||||||
return result, current_sentence, is_first
|
return result, current_sentence, is_first
|
||||||
for char in text:
|
except Exception as e:
|
||||||
current_sentence += char
|
logger.error(f"断句时出现错误: {str(e)}")
|
||||||
if is_first and char in ',.?!，。？！':
|
|
||||||
result.append(current_sentence)
|
|
||||||
current_sentence = ''
|
|
||||||
is_first = False
|
|
||||||
elif char in '。？！':
|
|
||||||
result.append(current_sentence)
|
|
||||||
current_sentence = ''
|
|
||||||
return result, current_sentence, is_first
|
|
||||||
|
|
||||||
#vad预处理
|
#vad预处理
|
||||||
def vad_preprocess(audio):
|
def vad_preprocess(audio):
|
||||||
|
|
@ -203,74 +206,84 @@ async def sct_user_input_handler(ws,user_input_q,llm_input_q,future_session_id,f
|
||||||
break
|
break
|
||||||
await user_input_q.put(sct_data_json['audio'])
|
await user_input_q.put(sct_data_json['audio'])
|
||||||
except KeyError as ke:
|
except KeyError as ke:
|
||||||
if sct_data_json['state'] == 1 and sct_data_json['method'] == 'heartbeat':
|
if 'state' in sct_data_json and 'method' in sct_data_json:
|
||||||
logger.debug("收到心跳包")
|
logger.debug("收到心跳包")
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"用户输入处理函数发生错误: {str(e)}")
|
||||||
|
|
||||||
#语音识别
|
#语音识别
|
||||||
async def sct_asr_handler(user_input_q,llm_input_q,user_input_finish_event):
|
async def sct_asr_handler(user_input_q,llm_input_q,user_input_finish_event):
|
||||||
logger.debug("语音识别函数启动")
|
logger.debug("语音识别函数启动")
|
||||||
current_message = ""
|
try:
|
||||||
while not (user_input_finish_event.is_set() and user_input_q.empty()):
|
current_message = ""
|
||||||
audio_data = await user_input_q.get()
|
while not (user_input_finish_event.is_set() and user_input_q.empty()):
|
||||||
asr_result = asr.streaming_recognize(audio_data)
|
audio_data = await user_input_q.get()
|
||||||
|
asr_result = asr.streaming_recognize(audio_data)
|
||||||
|
current_message += ''.join(asr_result['text'])
|
||||||
|
asr_result = asr.streaming_recognize(b'',is_end=True)
|
||||||
current_message += ''.join(asr_result['text'])
|
current_message += ''.join(asr_result['text'])
|
||||||
asr_result = asr.streaming_recognize(b'',is_end=True)
|
await llm_input_q.put(current_message)
|
||||||
current_message += ''.join(asr_result['text'])
|
except Exception as e:
|
||||||
await llm_input_q.put(current_message)
|
logger.error(f"语音识别函数发生错误: {str(e)}")
|
||||||
logger.debug(f"接收到用户消息: {current_message}")
|
logger.debug(f"接收到用户消息: {current_message}")
|
||||||
|
|
||||||
#大模型调用
|
#大模型调用
|
||||||
async def sct_llm_handler(ws,session_id,response_type,llm_info,tts_info,db,redis,llm_input_q,chat_finished_event):
|
async def sct_llm_handler(ws,session_id,response_type,llm_info,tts_info,db,redis,llm_input_q,chat_finished_event):
|
||||||
logger.debug("llm调用函数启动")
|
logger.debug("llm调用函数启动")
|
||||||
llm_response = ""
|
try:
|
||||||
current_sentence = ""
|
llm_response = ""
|
||||||
is_first = True
|
current_sentence = ""
|
||||||
is_end = False
|
is_first = True
|
||||||
session_content = get_session_content(session_id,redis,db)
|
is_end = False
|
||||||
messages = json.loads(session_content["messages"])
|
session_content = get_session_content(session_id,redis,db)
|
||||||
current_message = await llm_input_q.get()
|
messages = json.loads(session_content["messages"])
|
||||||
messages.append({'role': 'user', "content": current_message})
|
current_message = await llm_input_q.get()
|
||||||
payload = json.dumps({
|
messages.append({'role': 'user', "content": current_message})
|
||||||
"model": llm_info["model"],
|
payload = json.dumps({
|
||||||
"stream": True,
|
"model": llm_info["model"],
|
||||||
"messages": messages,
|
"stream": True,
|
||||||
"max_tokens": 10000,
|
"messages": messages,
|
||||||
"temperature": llm_info["temperature"],
|
"max_tokens": 10000,
|
||||||
"top_p": llm_info["top_p"]
|
"temperature": llm_info["temperature"],
|
||||||
})
|
"top_p": llm_info["top_p"]
|
||||||
headers = {
|
})
|
||||||
'Authorization': f"Bearer {Config.MINIMAX_LLM.API_KEY}",
|
headers = {
|
||||||
'Content-Type': 'application/json'
|
'Authorization': f"Bearer {Config.MINIMAX_LLM.API_KEY}",
|
||||||
}
|
'Content-Type': 'application/json'
|
||||||
response = requests.post(Config.MINIMAX_LLM.URL, headers=headers, data=payload,stream=True)
|
}
|
||||||
for chunk in response.iter_lines():
|
response = requests.post(Config.MINIMAX_LLM.URL, headers=headers, data=payload,stream=True) #调用大模型
|
||||||
chunk_data = parseChunkDelta(chunk)
|
except Exception as e:
|
||||||
is_end = chunk_data == "end"
|
logger.error(f"llm调用发生错误: {str(e)}")
|
||||||
if not is_end:
|
try:
|
||||||
llm_response += chunk_data
|
for chunk in response.iter_lines():
|
||||||
sentences,current_sentence,is_first = split_string_with_punctuation(current_sentence,chunk_data,is_first,is_end)
|
chunk_data = parseChunkDelta(chunk)
|
||||||
for sentence in sentences:
|
is_end = chunk_data == "end"
|
||||||
if response_type == RESPONSE_TEXT:
|
if not is_end:
|
||||||
response_message = {"type": "text", "code":200, "msg": sentence}
|
llm_response += chunk_data
|
||||||
await ws.send_text(json.dumps(response_message, ensure_ascii=False))
|
sentences,current_sentence,is_first = split_string_with_punctuation(current_sentence,chunk_data,is_first,is_end) #断句
|
||||||
elif response_type == RESPONSE_AUDIO:
|
for sentence in sentences:
|
||||||
sr,audio = tts.synthesize(sentence, tts_info["speaker_id"], tts_info["language"], tts_info["noise_scale"], tts_info["noise_scale_w"], tts_info["length_scale"],return_bytes=True)
|
if response_type == RESPONSE_TEXT:
|
||||||
response_message = {"type": "text", "code":200, "msg": sentence}
|
response_message = {"type": "text", "code":200, "msg": sentence}
|
||||||
await ws.send_bytes(audio)
|
await ws.send_text(json.dumps(response_message, ensure_ascii=False)) #返回文本信息
|
||||||
await ws.send_text(json.dumps(response_message, ensure_ascii=False))
|
elif response_type == RESPONSE_AUDIO:
|
||||||
|
sr,audio = tts.synthesize(sentence, tts_info["speaker_id"], tts_info["language"], tts_info["noise_scale"], tts_info["noise_scale_w"], tts_info["length_scale"],return_bytes=True)
|
||||||
|
response_message = {"type": "text", "code":200, "msg": sentence}
|
||||||
|
await ws.send_bytes(audio) #返回音频数据
|
||||||
|
await ws.send_text(json.dumps(response_message, ensure_ascii=False)) #返回文本信息
|
||||||
logger.debug(f"websocket返回: {sentence}")
|
logger.debug(f"websocket返回: {sentence}")
|
||||||
if is_end:
|
if is_end:
|
||||||
logger.debug(f"llm返回结果: {llm_response}")
|
logger.debug(f"llm返回结果: {llm_response}")
|
||||||
await ws.send_text(json.dumps({"type": "end", "code": 200, "msg": ""}, ensure_ascii=False))
|
await ws.send_text(json.dumps({"type": "end", "code": 200, "msg": ""}, ensure_ascii=False))
|
||||||
is_end = False
|
is_end = False #重置is_end标志位
|
||||||
|
session_content = get_session_content(session_id,redis,db)
|
||||||
session_content = get_session_content(session_id,redis,db)
|
messages = json.loads(session_content["messages"])
|
||||||
messages = json.loads(session_content["messages"])
|
messages.append({'role': 'assistant', "content": llm_response})
|
||||||
messages.append({'role': 'assistant', "content": llm_response})
|
session_content["messages"] = json.dumps(messages,ensure_ascii=False) #更新对话
|
||||||
session_content["messages"] = json.dumps(messages,ensure_ascii=False) #更新对话
|
redis.set(session_id,json.dumps(session_content,ensure_ascii=False)) #更新session
|
||||||
redis.set(session_id,json.dumps(session_content,ensure_ascii=False)) #更新session
|
is_first = True
|
||||||
is_first = True
|
llm_response = ""
|
||||||
llm_response = ""
|
except Exception as e:
|
||||||
|
logger.error(f"处理llm返回结果发生错误: {str(e)}")
|
||||||
chat_finished_event.set()
|
chat_finished_event.set()
|
||||||
|
|
||||||
async def streaming_chat_temporary_handler(ws: WebSocket, db, redis):
|
async def streaming_chat_temporary_handler(ws: WebSocket, db, redis):
|
||||||
|
|
@ -334,8 +347,9 @@ async def scl_user_input_handler(ws,user_input_q,llm_input_q,future_session_id,f
|
||||||
continue
|
continue
|
||||||
except asyncio.TimeoutError:
|
except asyncio.TimeoutError:
|
||||||
continue
|
continue
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"用户输入处理函数发生错误: {str(e)}")
|
||||||
|
break
|
||||||
|
|
||||||
#语音识别
|
#语音识别
|
||||||
async def scl_asr_handler(user_input_q,llm_input_q,input_finished_event,asr_finished_event):
|
async def scl_asr_handler(user_input_q,llm_input_q,input_finished_event,asr_finished_event):
|
||||||
|
|
@ -354,6 +368,9 @@ async def scl_asr_handler(user_input_q,llm_input_q,input_finished_event,asr_fini
|
||||||
current_message += ''.join(asr_result['text'])
|
current_message += ''.join(asr_result['text'])
|
||||||
except asyncio.TimeoutError:
|
except asyncio.TimeoutError:
|
||||||
continue
|
continue
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"语音识别函数发生错误: {str(e)}")
|
||||||
|
break
|
||||||
asr_finished_event.set()
|
asr_finished_event.set()
|
||||||
|
|
||||||
#大模型调用
|
#大模型调用
|
||||||
|
|
@ -413,6 +430,9 @@ async def scl_llm_handler(ws,session_id,response_type,llm_info,tts_info,db,redis
|
||||||
llm_response = ""
|
llm_response = ""
|
||||||
except asyncio.TimeoutError:
|
except asyncio.TimeoutError:
|
||||||
continue
|
continue
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"处理llm返回结果发生错误: {str(e)}")
|
||||||
|
break
|
||||||
chat_finished_event.set()
|
chat_finished_event.set()
|
||||||
|
|
||||||
async def streaming_chat_lasting_handler(ws,db,redis):
|
async def streaming_chat_lasting_handler(ws,db,redis):
|
||||||
|
|
@ -466,9 +486,13 @@ async def voice_call_audio_producer(ws,audio_q,future,input_finished_event):
|
||||||
vad_frame,audio_data = vad_preprocess(audio_data)
|
vad_frame,audio_data = vad_preprocess(audio_data)
|
||||||
await audio_q.put(vad_frame) #将音频数据存入audio_q
|
await audio_q.put(vad_frame) #将音频数据存入audio_q
|
||||||
except KeyError as ke:
|
except KeyError as ke:
|
||||||
logger.info(f"收到心跳包")
|
if 'state' in voice_call_data_json and 'method' in voice_call_data_json:
|
||||||
|
logger.info(f"收到心跳包")
|
||||||
except asyncio.TimeoutError:
|
except asyncio.TimeoutError:
|
||||||
continue
|
continue
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"音频数据生产函数发生错误: {str(e)}")
|
||||||
|
break
|
||||||
|
|
||||||
|
|
||||||
#音频数据消费函数
|
#音频数据消费函数
|
||||||
|
|
@ -498,6 +522,9 @@ async def voice_call_audio_consumer(ws,audio_q,asr_result_q,input_finished_event
|
||||||
vad_count = 0
|
vad_count = 0
|
||||||
except asyncio.TimeoutError:
|
except asyncio.TimeoutError:
|
||||||
continue
|
continue
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"音频数据消费者函数发生错误: {str(e)}")
|
||||||
|
break
|
||||||
asr_finished_event.set()
|
asr_finished_event.set()
|
||||||
|
|
||||||
#asr结果消费以及llm返回生产函数
|
#asr结果消费以及llm返回生产函数
|
||||||
|
|
@ -553,6 +580,9 @@ async def voice_call_llm_handler(ws,session_id,llm_info,tts_info,db,redis,asr_re
|
||||||
llm_response = ""
|
llm_response = ""
|
||||||
except asyncio.TimeoutError:
|
except asyncio.TimeoutError:
|
||||||
continue
|
continue
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"处理llm返回结果发生错误: {str(e)}")
|
||||||
|
break
|
||||||
voice_call_end_event.set()
|
voice_call_end_event.set()
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue