35 lines
1.1 KiB
Python
35 lines
1.1 KiB
Python
|
import os
|
|||
|
|
import json
|
|||
|
|
import torch
|
|||
|
|
import logging
|
|||
|
|
import concurrent.futures
|
|||
|
|
import librosa
|
|||
|
|
import torch.distributed as dist
|
|||
|
|
from typing import Collection
|
|||
|
|
import torch
|
|||
|
|
import torchaudio
|
|||
|
|
from torch import nn
|
|||
|
|
import random
|
|||
|
|
import re
|
|||
|
|
import string
|
|||
|
|
from funasr.tokenizer.cleaner import TextCleaner
|
|||
|
|
from funasr.register import tables
|
|||
|
|
|
|||
|
|
|
|||
|
|
@tables.register("preprocessor_classes", "TextPreprocessRemovePunctuation")
|
|||
|
|
class TextPreprocessRemovePunctuation(nn.Module):
|
|||
|
|
def __init__(self, **kwargs):
|
|||
|
|
super().__init__()
|
|||
|
|
|
|||
|
|
def forward(self, text, **kwargs):
|
|||
|
|
# 定义英文标点符号
|
|||
|
|
en_punct = string.punctuation
|
|||
|
|
# 定义中文标点符号（部分常用的）
|
|||
|
|
cn_punct = "。？！，、；：“”‘’（）《》【】…—～·"
|
|||
|
|
# 合并英文和中文标点符号
|
|||
|
|
all_punct = en_punct + cn_punct
|
|||
|
|
# 创建正则表达式模式，匹配任何在all_punct中的字符
|
|||
|
|
punct_pattern = re.compile("[{}]".format(re.escape(all_punct)))
|
|||
|
|
# 使用正则表达式的sub方法替换掉这些字符
|
|||
|
|
return punct_pattern.sub("", text)
|