forked from killua/TakwayPlatform
39 lines
979 B
Python
39 lines
979 B
Python
'''
|
||
Defines the set of symbols used in text input to the model.
|
||
'''
|
||
|
||
'''# japanese_cleaners
|
||
_pad = '_'
|
||
_punctuation = ',.!?-'
|
||
_letters = 'AEINOQUabdefghijkmnoprstuvwyzʃʧ↓↑ '
|
||
'''
|
||
|
||
'''# japanese_cleaners2
|
||
_pad = '_'
|
||
_punctuation = ',.!?-~…'
|
||
_letters = 'AEINOQUabdefghijkmnoprstuvwyzʃʧʦ↓↑ '
|
||
'''
|
||
|
||
'''# korean_cleaners
|
||
_pad = '_'
|
||
_punctuation = ',.!?…~'
|
||
_letters = 'ㄱㄴㄷㄹㅁㅂㅅㅇㅈㅊㅋㅌㅍㅎㄲㄸㅃㅆㅉㅏㅓㅗㅜㅡㅣㅐㅔ '
|
||
'''
|
||
|
||
'''# chinese_cleaners
|
||
_pad = '_'
|
||
_punctuation = '，。！？—…'
|
||
_letters = 'ㄅㄆㄇㄈㄉㄊㄋㄌㄍㄎㄏㄐㄑㄒㄓㄔㄕㄖㄗㄘㄙㄚㄛㄜㄝㄞㄟㄠㄡㄢㄣㄤㄥㄦㄧㄨㄩˉˊˇˋ˙ '
|
||
'''
|
||
|
||
# zh_ja_mixture_cleaners
|
||
_pad = '_'
|
||
_punctuation = ',.!?-~…'
|
||
_letters = 'AEINOQUabdefghijklmnoprstuvwyzʃʧʦɯɹəɥ⁼ʰ`→↓↑ '
|
||
|
||
|
||
# Export all symbols:
|
||
symbols = [_pad] + list(_punctuation) + list(_letters)
|
||
|
||
# Special symbol ids
|
||
SPACE_ID = symbols.index(" ") |