killua
/
TakwayPlatform
1
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

update: 增加杂音屏蔽词，暂时停止记忆、情感识别功能

This commit is contained in:
killua 2024-06-05 10:32:40 +08:00
parent 09ffeb6ab6
commit 2378fee258
2 changed files with 16 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
app/controllers/chat_controller.py
View File

@ -263,14 +263,15 @@ async def sct_asr_handler(ws,session_id,user_input_q,llm_input_q,user_input_fini
current_message += ''.join(asr_result['text']) current_message += ''.join(asr_result['text'])
asr_result = asr.streaming_recognize(session_id,b'',is_end=True) asr_result = asr.streaming_recognize(session_id,b'',is_end=True)
current_message += ''.join(asr_result['text']) current_message += ''.join(asr_result['text'])
if current_message == "": slice_arr = ["嗯",""]
if current_message in slice_arr:
await ws.send_text(json.dumps({"type": "close", "code": 201, "msg": ""}, ensure_ascii=False)) await ws.send_text(json.dumps({"type": "close", "code": 201, "msg": ""}, ensure_ascii=False))
return return
current_message = asr.punctuation_correction(current_message) current_message = asr.punctuation_correction(current_message)
emotion_dict = asr.emtion_recognition(audio) #情感辨识 # emotion_dict = asr.emtion_recognition(audio) #情感辨识
if not isinstance(emotion_dict, str): # if not isinstance(emotion_dict, str):
max_index = emotion_dict['scores'].index(max(emotion_dict['scores'])) # max_index = emotion_dict['scores'].index(max(emotion_dict['scores']))
current_message = f"{current_message},当前说话人的情绪：{emotion_dict['labels'][max_index]}" # current_message = f"{current_message},当前说话人的情绪：{emotion_dict['labels'][max_index]}"
await llm_input_q.put(current_message) await llm_input_q.put(current_message)
asr.session_signout(session_id) asr.session_signout(session_id)
except Exception as e: except Exception as e:
@ -288,8 +289,11 @@ async def sct_asr_handler(ws,session_id,user_input_q,llm_input_q,user_input_fini
elif status == CONTINUE_FRAME: elif status == CONTINUE_FRAME:
await xf_websocket.send(make_continue_frame(audio_data)) await xf_websocket.send(make_continue_frame(audio_data))
await xf_websocket.send(make_last_frame("")) await xf_websocket.send(make_last_frame(""))
current_message = parse_xfasr_recv(json.loads(await xf_websocket.recv())) current_message = parse_xfasr_recv(json.loads(await xf_websocket.recv()))
if current_message in ["","嗯"]:
await ws.send_text(json.dumps({"type": "close", "code": 201, "msg": ""}, ensure_ascii=False))
return
await llm_input_q.put(current_message) await llm_input_q.put(current_message)
logger.debug(f"接收到用户消息: {current_message}") logger.debug(f"接收到用户消息: {current_message}")
@ -305,9 +309,11 @@ async def sct_llm_handler(ws,session_id,response_type,llm_info,tts_info,db,redis
user_info = json.loads(session_content["user_info"]) user_info = json.loads(session_content["user_info"])
messages = json.loads(session_content["messages"]) messages = json.loads(session_content["messages"])
current_message = await llm_input_q.get() current_message = await llm_input_q.get()
if current_message == "":
return
messages.append({'role': 'user', "content": current_message}) messages.append({'role': 'user', "content": current_message})
messages_send = messages #创造一个message副本，在其中最后一条数据前面添加用户信息 messages_send = messages #创造一个message副本，在其中最后一条数据前面添加用户信息
messages_send[-1]['content'] = f"用户性格:{user_info['character']}\n事件摘要:{user_info['events']}" + messages_send[-1]['content'] # messages_send[-1]['content'] = f"用户性格:{user_info['character']}\n事件摘要:{user_info['events']}" + messages_send[-1]['content']
payload = json.dumps({ payload = json.dumps({
"model": llm_info["model"], "model": llm_info["model"],
"stream": True, "stream": True,

6
utils/xf_asr_utils.py
View File

@ -40,15 +40,15 @@ def generate_xf_asr_url():
def make_first_frame(buf): def make_first_frame(buf):
first_frame = {"common" : {"app_id":Config.XF_ASR.APP_ID},"business" : {"domain":"iat","language":"zh_cn","accent":"mandarin","vad_eos":10000}, first_frame = {"common" : {"app_id":Config.XF_ASR.APP_ID},"business" : {"domain":"iat","language":"zh_cn","accent":"mandarin","vad_eos":10000},
"data":{"status":0,"format":"audio/L16;rate=8000","audio":buf,"encoding":"raw"}} "data":{"status":0,"format":"audio/L16;rate=16000","audio":buf,"encoding":"raw"}}
return json.dumps(first_frame) return json.dumps(first_frame)
def make_continue_frame(buf): def make_continue_frame(buf):
continue_frame = {"data":{"status":1,"format":"audio/L16;rate=8000","audio":buf,"encoding":"raw"}} continue_frame = {"data":{"status":1,"format":"audio/L16;rate=16000","audio":buf,"encoding":"raw"}}
return json.dumps(continue_frame) return json.dumps(continue_frame)
def make_last_frame(buf): def make_last_frame(buf):
last_frame = {"data":{"status":2,"format":"audio/L16;rate=8000","audio":buf,"encoding":"raw"}} last_frame = {"data":{"status":2,"format":"audio/L16;rate=16000","audio":buf,"encoding":"raw"}}
return json.dumps(last_frame) return json.dumps(last_frame)
def parse_xfasr_recv(message): def parse_xfasr_recv(message):