TakwayBoard/takway/audio_utils.py

import io
import os
import time
import pyaudio
import wave
import json
import warnings
import threading
import numpy as np
from collections import deque

from .common_utils import encode_bytes2str, decode_str2bytes
   
from takway.board import *
try:
    import keyboard
except:
    pass
                        
def play_audio(audio_data, type='base64'):
    '''
    读取base64编码的音频流并播放
    '''
    # PyAudio配置
    p = pyaudio.PyAudio()
    stream = p.open(format=pyaudio.paInt16, channels=1, rate=22050, output=True)

    # 播放音频
    stream.write(audio_data)
    stream.stop_stream()
    stream.close()
    p.terminate()

'''
import librosa
def reshape_sample_rate(audio, sr_original=None, sr_target=16000):
    # 获取原始采样率和音频数据
    if isinstance(audio, tuple):
        sr_original, audio_data = audio
    elif isinstance(audio, bytes):
        audio_data = np.frombuffer(audio, dtype=np.int16)
    assert sr_original is not None, f"sr_original should be provided if audio is a \
        numpy.ndarray, but got sr_original `{sr_original}`."
    
    if isinstance(audio_data, np.ndarray):
        if audio_data.dtype == np.dtype('int16'):
            audio_data = audio_data.astype(np.float32) / np.iinfo(np.int16).max
        assert audio_data.dtype == np.dtype('float32'), f"audio_data should be float32, \
            but got {audio_data.dtype}."
    else:
        raise TypeError(f"audio_data should be numpy.ndarray, but got {type(audio_data)}.")
    
    # 重新采样音频数据
    audio_data_resampled = librosa.resample(audio_data, orig_sr=sr_original, target_sr=sr_target)
    
    if audio_data_resampled.dtype == np.dtype('float32'):
        audio_data_resampled = np.int16(audio_data_resampled * np.iinfo(np.int16).max)
    
    # If the input was bytes, return the resampled data as bytes
    if isinstance(audio, bytes):
        audio_data_resampled = audio_data_resampled.tobytes()
        
    return audio_data_resampled

# Example usage:
# If your audio data is in bytes:
# audio_bytes = b'...'  # Your audio data as bytes
# audio_data_resampled = reshape_sample_rate(audio_bytes)

# If your audio data is in numpy int16:
# audio_int16 = np.array([...], dtype=np.int16)  # Your audio data as numpy int16
# audio_data_resampled = reshape_sample_rate(audio_int16)
'''



# ####################################################### #
# base audio class
# ####################################################### #

class BaseAudio:
    def __init__(self, 
                 filename=None, 
                 input=False, 
                 output=False, 
                 CHUNK=1024, 
                 FORMAT=pyaudio.paInt16, 
                 CHANNELS=1, 
                 RATE=16000,
                 input_device_index=None,
                 output_device_index=None,
                 **kwargs):
        self.CHUNK = CHUNK
        self.FORMAT = FORMAT
        self.CHANNELS = CHANNELS
        self.RATE = RATE
        self.filename = filename
        assert input!= output, "input and output cannot be the same, \
            but got input={} and output={}.".format(input, output)
        print("------------------------------------------")
        print(f"{'Input' if input else 'Output'} Audio Initialization: ")
        print(f"CHUNK: {self.CHUNK} \nFORMAT: {self.FORMAT} \nCHANNELS: {self.CHANNELS} \nRATE: {self.RATE} \ninput_device_index: {input_device_index} \noutput_device_index: {output_device_index}")
        print("------------------------------------------")
        self.p = pyaudio.PyAudio()
        self.stream = self.p.open(format=FORMAT,
                                  channels=CHANNELS,
                                  rate=RATE,
                                  input=input,
                                  output=output,
                                  input_device_index=input_device_index,
                                  output_device_index=output_device_index,
                                  **kwargs)
    
    def load_audio_file(self, wav_file):
        with wave.open(wav_file, 'rb') as wf:
            params = wf.getparams()
            frames = wf.readframes(params.nframes)
            print("Audio file loaded.")
            # Audio Parameters
            # print("Channels:", params.nchannels)
            # print("Sample width:", params.sampwidth)
            # print("Frame rate:", params.framerate)
            # print("Number of frames:", params.nframes)
            # print("Compression type:", params.comptype)
        return frames
    
    def check_audio_type(self, audio_data, return_type=None):
        assert return_type in ['bytes', 'io', None], \
            "return_type should be 'bytes', 'io' or None."
        if isinstance(audio_data, str):
            if len(audio_data) > 50:
                audio_data = decode_str2bytes(audio_data)
            else:
                assert os.path.isfile(audio_data), \
                    "audio_data should be a file path or a bytes object."
                wf = wave.open(audio_data, 'rb')
                audio_data = wf.readframes(wf.getnframes())
        elif isinstance(audio_data, np.ndarray):
            if audio_data.dtype == np.dtype('float32'):
                audio_data = np.int16(audio_data * np.iinfo(np.int16).max)
            audio_data = audio_data.tobytes()
        elif isinstance(audio_data, bytes):
            pass
        else:
            raise TypeError(f"audio_data must be bytes, numpy.ndarray or str, \
                but got {type(audio_data)}")
        
        if return_type == None:
            return audio_data
        return self.write_wave(None, [audio_data], return_type)
    
    def write_wave(self, filename, frames, return_type='io'):
        """Write audio data to a file."""
        if isinstance(frames, bytes):
            frames = [frames]
        if not isinstance(frames, list): 
            raise TypeError("frames should be \
            a list of bytes or a bytes object, \
            but got {}.".format(type(frames)))
        
        if return_type == 'io':
            if filename is None:
                filename = io.BytesIO()
            if self.filename:
                filename = self.filename
            return self.write_wave_io(filename, frames) 
        elif return_type == 'bytes':
            return self.write_wave_bytes(frames)

    
    def write_wave_io(self, filename, frames):
        """
        Write audio data to a file-like object.
        
        Args:
            filename: [string or file-like object], file path or file-like object to write
            frames: list of bytes, audio data to write
        """
        wf = wave.open(filename, 'wb')
        
        # 设置WAV文件的参数
        wf.setnchannels(self.CHANNELS)
        wf.setsampwidth(self.p.get_sample_size(self.FORMAT))
        wf.setframerate(self.RATE)
        wf.writeframes(b''.join(frames))
        wf.close()
        if isinstance(filename, io.BytesIO):
            filename.seek(0)   # reset file pointer to beginning
        return filename
    
    def write_wave_bytes(self, frames):
        """Write audio data to a bytes object."""
        return b''.join(frames)


# ####################################################### #
# play audio data from Speaker
# ####################################################### #

class AudioPlayer(BaseAudio):
    def __init__(self, 
                 RATE=22050, 
                 **kwargs):
        super().__init__(output=True, RATE=RATE, **kwargs)

    def play(self, audio_data):
        # print("Playing audio data...")
        audio_data = self.check_audio_type(audio_data, return_type=None)
        
        for i in range(0, len(audio_data), self.CHUNK):
            self.stream.write(audio_data[i:i+self.CHUNK])
            # print("Playing audio data...{}/{}".format(i, len(audio_data)))
        self.stream.write(audio_data[i+self.CHUNK:])
        # print("Audio data played.")
        

    def close(self):
        self.stream.stop_stream()
        self.stream.close()
        self.p.terminate()

# ####################################################### #
# record audio data from microphone
# ####################################################### #
class BaseRecorder(BaseAudio):
    def __init__(self, 
                 input=True, 
                 base_chunk_size=None, 
                 RATE=16000, 
                 **kwargs):
        super().__init__(input=input, RATE=RATE, **kwargs)
        self.base_chunk_size = base_chunk_size
        if base_chunk_size is None:
            self.base_chunk_size = self.CHUNK

    def record(self, 
               filename,
               duration=5, 
               return_type='io',
               logger=None):
        if logger is not None:
            logger.info("Recording started.")
        else:
            print("Recording started.")
        frames = []
        for i in range(0, int(self.RATE / self.CHUNK * duration)):
            data = self.stream.read(self.CHUNK, exception_on_overflow=False)
            frames.append(data)
        if logger is not None:
            logger.info("Recording stopped.")
        else:
            print("Recording stopped.")
        return self.write_wave(filename, frames, return_type)

    def record_chunk_voice(self, 
                           return_type='bytes', 
                           CHUNK=None, 
                           exception_on_overflow=True, 
                           queue=None):
        data = self.stream.read(self.CHUNK if CHUNK is None else CHUNK, 
                                exception_on_overflow=exception_on_overflow)
        if return_type is not None:
            return self.write_wave(None, [data], return_type)
        return data


class HDRecorder(BaseRecorder):
    def __init__(self, 
                 board=None,
                 hd_trigger='keyboard', 
                 keyboard_key='space',
                 voice_trigger=True,
                 hd_chunk_size=None,
                 hd_detect_threshold=50,
                 **kwargs):
        super().__init__(**kwargs)
        assert hd_trigger in ['keyboard', 'button']
        
        self.hd_trigger = hd_trigger
        self.voice_trigger = voice_trigger
        
        self.hd_chunk_size = hd_chunk_size
        if hd_chunk_size is None:
            self.hd_chunk_size = self.base_chunk_size
        
        if board == None:
            assert hd_trigger == 'keyboard', "board should be `None` if hd_trigger is `keyboard`."
            self.keyboard_key = keyboard_key
            self.hardware = Keyboard(hd_trigger, keyboard_key, hd_detect_threshold)
        else:
            assert hd_trigger == 'button', f"hd_trigger should be `button` if board is `v329` or `orangepi`, but got `{hd_trigger}`."
            if board == 'v329':
                self.hardware = V329(hd_trigger, hd_detect_threshold)
            elif board == 'orangepi':
                self.hardware = OrangePi(hd_trigger, hd_detect_threshold, enable_start_light=True)
        print(f"Using {hd_trigger} as hardware trigger.")
        
    def wait_for_hardware_pressed(self):
        return self.hardware.wait_for_hardware_pressed()
    
    @property
    def is_hardware_pressed(self):
        return self.hardware.is_hardware_pressed
    
    def record_hardware(self, return_type='bytes'):
        """record audio when hardware trigger"""
        print("Recording started for hardware trigger.")
        frames = []
        self.wait_for_hardware_pressed()
        while True:
            if self.hd_trigger == 'keyboard':
                if keyboard.is_pressed(self.keyboard_key):
                    print("recording...")
                    data = self.record_chunk_voice(
                        CHUNK=self.CHUNK, 
                        return_type=None, 
                        exception_on_overflow=False)
                    frames.append(data)
                else:
                    break
                    print("Recording stopped.")
            elif self.hd_trigger == 'button':
                if self.get_button_status():
                    data = self.stream.read(self.CHUNK)
                    frames.append(data)
                else:
                    break
            else:
                recording = False
                raise ValueError("hd_trigger should be 'keyboard' or 'button'.")
        return self.write_wave(self.filename, frames, return_type)
    
    '''
    def record(self, return_type='bytes', queue=None):
        if self.hd_trigger == 'all':
            value_list = []  # 用于记录value的状态
            if keyboard.is_pressed(self.keyboard_key):
                audio_data = self.record_keyboard(return_type, queue)
            elif self.button.get_value() == 0:
                if self.get_button_status():
                    audio_data = self.record_button(return_type, queue)
            else:
                audio_data = self.record_voice(return_type, queue)
        elif self.hd_trigger == 'keyboard':
            print("Press SPACE to start recording.")
            keyboard.wait("space")
            audio_data = self.record_keyboard(return_type, queue)
        elif self.hd_trigger == 'button':
            print("Touch to start recording...")
            if self.button.get_value() == 0:
                if self.get_button_status():
                    audio_data = self.record_button(return_type, queue)
        else:
            audio_data = self.record_voice(return_type, queue)
            
        return audio_data
    
    def record_keyboard(self, return_type='bytes', queue=None):
        """record audio when keyboard pressing"""
        print("Recording started.")
        frames = []
        recording = True
        while recording:
            if keyboard.is_pressed(self.keyboard_key):
                data = self.stream.read(self.CHUNK)
                frames.append(data)
            else:
                recording = False
                print("Recording stopped.")
        return self.write_wave(self.filename, frames, return_type)
        
    def record_button(self, return_type='bytes', queue=None):
        """record audio when button pressing"""
        print("Recording started.")
        frames = []
        recording = True
        while recording:
            value = self.button.get_value()
            if value == 0:
                data = self.stream.read(CHUNK)
                frames.append(data)
            else:
                recording = False
                print("Recording stopped.")
        return self.write_wave(self.filename, frames, return_type)
    '''
        
# ####################################################### #
# record audio data from microphone with VAD
# ####################################################### #
try:
    import webrtcvad
    webrtcvad_available = True
except:
    warnings.warn("webrtcvad module not found, please install it if use `vad` hd_trigger.")
    webrtcvad_available = False

class VADRecorder(HDRecorder):
    def __init__(self, vad_sensitivity=0, frame_duration=30, vad_buffer_size=7, min_act_time=0.3,**kwargs):
        super().__init__(**kwargs)
        if webrtcvad_available:
            self.vad = webrtcvad.Vad(vad_sensitivity)
        self.vad_buffer_size = vad_buffer_size
        self.vad_chunk_size = int(self.RATE * frame_duration / 1000)
            
        self.min_act_time = min_act_time    # 最小活动时间，单位秒
            
        self.is_currently_speaking = False
        self.frames = []

    def is_speech(self, data):
        return self.vad.is_speech(data, self.RATE)
    
    def vad_filter(self, data):
        pass
        

    def vad_record(self, return_type='io', CHUNK=None, queue=None, save_file=False):
        """录音并进行语音活动检测人声并返回分割后的音频数据"""
        all_frames = []
        
        buffer_size = self.vad_buffer_size
        active_buffer = deque([False for i in range(buffer_size)], maxlen=buffer_size)
        audio_buffer = deque(maxlen=buffer_size)
        silence_buffer = deque([True for i in range(buffer_size)], maxlen=buffer_size)
        
        print("vad_recorded_audio VAD started. Press Ctrl+C to stop.")
        try:
            while True:
                data = self.stream.read(self.vad_chunk_size)
                all_frames.append(data)
                print(f"VAD processing..., is_speech: {self.is_speech(data)}")
                if self.is_speech(data):
                    # 标志位buffer
                    active_buffer.append(True); active_buffer.popleft()
                    silence_buffer.append(False); silence_buffer.popleft()
                    # 暂时增加到buffer中
                    audio_buffer.append(data)
                    # 如果满足检测要求
                    if all(active_buffer):
                        if not self.is_currently_speaking:
                            print("Speech start detected")
                            self.is_currently_speaking = True
                            self.frames.extend(audio_buffer)   # 把说话的buffer也加上
                    if self.is_currently_speaking:
                        self.frames.append(data)
                else:
                    # 标志位buffer
                    active_buffer.append(False); active_buffer.popleft()
                    silence_buffer.append(True); silence_buffer.popleft()
                    # 检测到人声并持续录音
                    if self.is_currently_speaking:
                        # 结束标志位
                        if all(silence_buffer):
                            print("Speech end detected")
                            break
        except KeyboardInterrupt:
            print("KeyboardInterrupt")
            
        finally:
            print("Stopping...")
            if len(all_frames) > 0:
                print(f"ALL frame: {len(all_frames)}")
                print(f"ASR frame: {len(self.frames)}")
                if save_file:
                    self.write_wave(f"output_{time.time()}_all.wav", all_frames)
                    self.write_wave(f"output_{time.time()}.wav", self.frames)
            return self.write_wave(None, self.frames, return_type='bytes')
        

# ####################################################### #
# record audio data from microphone with PicoVoice hot words detection
# ####################################################### #

import struct
from datetime import datetime
import pvporcupine

class PicovoiceRecorder(VADRecorder):
    def __init__(self, 
                 access_key, 
                 keywords=None, 
                 keyword_paths=None, 
                 model_path=None, 
                 sensitivities=0.5, 
                 library_path=None,
                 **kwargs):
        
        super().__init__(**kwargs)
        
        pico_cfg = dict(
            access_key=access_key,
            keywords=keywords,
            keyword_paths=keyword_paths,
            model_path=model_path,
            sensitivities=sensitivities,
            library_path=library_path,
        )
        
        self.pico_detector_init(pico_cfg)
        
        self.keywords = self.pico_cfg['keywords']
        print(f"PicovoiceRecorder initialized with keywords: {self.keywords}")

    def pico_detector_init(self, pico_cfg):
        if pico_cfg['keyword_paths'] is None:
            if pico_cfg['keywords'] is None:
                raise ValueError(f"Either `--keywords` or `--keyword_paths` must be set. \
                    Available keywords: {list(pvporcupine.KEYWORDS)}")

            keyword_paths = [pvporcupine.KEYWORD_PATHS[x] for x in pico_cfg['keywords']]
        else:
            keyword_paths = pico_cfg['keyword_paths']

        if pico_cfg['sensitivities'] is None:
            pico_cfg['sensitivities'] = [0.5] * len(keyword_paths)
        elif isinstance(pico_cfg['sensitivities'], float):
            pico_cfg['sensitivities'] = [pico_cfg['sensitivities']] * len(keyword_paths)
            
        if len(keyword_paths) != len(pico_cfg['sensitivities']):
            raise ValueError('Number of keywords does not match the number of sensitivities.')
        
        try:
            self.porcupine = pvporcupine.create(
                access_key=pico_cfg['access_key'],
                keywords=pico_cfg['keywords'],
                keyword_paths=keyword_paths,
                model_path=pico_cfg['model_path'],
                sensitivities=pico_cfg['sensitivities'],
                library_path=pico_cfg['library_path'])
        except pvporcupine.PorcupineInvalidArgumentError as e:
            print("One or more arguments provided to Porcupine is invalid: ", pico_cfg.keys())
            print(e)
            raise e
        except pvporcupine.PorcupineActivationError as e:
            print("AccessKey activation error")
            raise e
        except pvporcupine.PorcupineActivationLimitError as e:
            print("AccessKey '%s' has reached it's temporary device limit" % pico_cfg['access_key'])
            raise e
        except pvporcupine.PorcupineActivationRefusedError as e:
            print("AccessKey '%s' refused" % pico_cfg['access_key'])
            raise e
        except pvporcupine.PorcupineActivationThrottledError as e:
            print("AccessKey '%s' has been throttled" % pico_cfg['access_key'])
            raise e
        except pvporcupine.PorcupineError as e:
            print("Failed to initialize Porcupine")
            raise e

        self.pico_cfg = pico_cfg
        
    def is_wakeup(self, data):
        pcm = struct.unpack_from("h" * self.porcupine.frame_length, data)
        result = self.porcupine.process(pcm)
        # print(f"picovoice result: {result}")
        if result >= 0:
            print('[%s] Detected %s' % (str(datetime.now()), self.keywords[result]))
            return True
        # self.write_wave(f"output_{time.time()}.wav", [data])
        # print(f"write to: output_{time.time()}.wav")
        return False
        

    def record_picovoice(self, return_type=None, exception_on_overflow=False, queue=None):
        
        print("Recording started. Press Ctrl+C to stop.")
        while True:
            data = self.record_chunk_voice(
                return_type=None, 
                CHUNK=self.porcupine.frame_length, 
                exception_on_overflow=exception_on_overflow, 
                queue=queue)
            
            wake_up = self.is_wakeup(data)
            if wake_up:
                break
        return True