|
|
|
import numpy as np |
|
import re |
|
import codecs |
|
import textwrap |
|
|
|
|
|
_pad = "$" |
|
_punctuation = ';:,.!?¡¿—…"«»“” ' |
|
_letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz' |
|
_letters_ipa = "ɑɐɒæɓʙβɔɕçɗɖðʤəɘɚɛɜɝɞɟʄɡɠɢʛɦɧħɥʜɨɪʝɭɬɫɮʟɱɯɰŋɳɲɴøɵɸθœɶʘɹɺɾɻʀʁɽʂʃʈʧʉʊʋⱱʌɣɤʍχʎʏʑʐʒʔʡʕʢǀǁǂǃˈˌːˑʼʴʰʱʲʷˠˤ˞↓↑→↗↘'̩'ᵻ" |
|
|
|
|
|
symbols = [_pad] + list(_punctuation) + list(_letters) + list(_letters_ipa) |
|
|
|
dicts = {} |
|
for i in range(len((symbols))): |
|
dicts[symbols[i]] = i |
|
|
|
class TextCleaner: |
|
def __init__(self, dummy=None): |
|
self.word_index_dictionary = dicts |
|
print(len(dicts)) |
|
def __call__(self, text): |
|
indexes = [] |
|
for char in text: |
|
try: |
|
indexes.append(self.word_index_dictionary[char]) |
|
except KeyError: |
|
print(text) |
|
return indexes |
|
|
|
|
|
|
|
|
|
alphabets= "([A-Za-z])" |
|
prefixes = "(Mr|St|Mrs|Ms|Dr)[.]" |
|
suffixes = "(Inc|Ltd|Jr|Sr|Co)" |
|
starters = "(Mr|Mrs|Ms|Dr|Prof|Capt|Cpt|Lt|He\s|She\s|It\s|They\s|Their\s|Our\s|We\s|But\s|However\s|That\s|This\s|Wherever)" |
|
acronyms = "([A-Z][.][A-Z][.](?:[A-Z][.])?)" |
|
websites = "[.](com|net|org|io|gov|edu|me)" |
|
digits = "([0-9])" |
|
multiple_dots = r'\.{2,}' |
|
|
|
|
|
|
|
def split_into_sentences(text): |
|
""" |
|
Split the text into sentences. |
|
|
|
If the text contains substrings "<prd>" or "<stop>", they would lead |
|
to incorrect splitting because they are used as markers for splitting. |
|
|
|
:param text: text to be split into sentences |
|
:type text: str |
|
|
|
:return: list of sentences |
|
:rtype: list[str] |
|
|
|
https://stackoverflow.com/questions/4576077/how-can-i-split-a-text-into-sentences |
|
""" |
|
text = " " + text + " " |
|
text = text.replace("\n"," ") |
|
text = re.sub(prefixes,"\\1<prd>",text) |
|
text = re.sub(websites,"<prd>\\1",text) |
|
text = re.sub(digits + "[.]" + digits,"\\1<prd>\\2",text) |
|
text = re.sub(multiple_dots, lambda match: "<prd>" * len(match.group(0)) + "<stop>", text) |
|
if "Ph.D" in text: text = text.replace("Ph.D.","Ph<prd>D<prd>") |
|
text = re.sub("\s" + alphabets + "[.] "," \\1<prd> ",text) |
|
text = re.sub(acronyms+" "+starters,"\\1<stop> \\2",text) |
|
text = re.sub(alphabets + "[.]" + alphabets + "[.]" + alphabets + "[.]","\\1<prd>\\2<prd>\\3<prd>",text) |
|
text = re.sub(alphabets + "[.]" + alphabets + "[.]","\\1<prd>\\2<prd>",text) |
|
text = re.sub(" "+suffixes+"[.] "+starters," \\1<stop> \\2",text) |
|
text = re.sub(" "+suffixes+"[.]"," \\1<prd>",text) |
|
text = re.sub(" " + alphabets + "[.]"," \\1<prd>",text) |
|
if "”" in text: text = text.replace(".”","”.") |
|
if "\"" in text: text = text.replace(".\"","\".") |
|
if "!" in text: text = text.replace("!\"","\"!") |
|
if "?" in text: text = text.replace("?\"","\"?") |
|
text = text.replace(".",".<stop>") |
|
text = text.replace("?","?<stop>") |
|
text = text.replace("!","!<stop>") |
|
text = text.replace("<prd>",".") |
|
sentences = text.split("<stop>") |
|
sentences = [s.strip() for s in sentences] |
|
|
|
|
|
sentences = [sub_sent+' ' for s in sentences for sub_sent in textwrap.wrap(s, 400, break_long_words=0)] |
|
|
|
if sentences and not sentences[-1]: sentences = sentences[:-1] |
|
return sentences |
|
|
|
def store_ssml(text=None, |
|
voice=None): |
|
'''create ssml: |
|
text : list of sentences |
|
voice: https://github.com/MycroftAI/mimic3-voices |
|
''' |
|
print('\n___________________________\n', len(text), text[0], '\n___________________________________\n') |
|
_s = '<speak>' |
|
for short_text in text: |
|
|
|
rate = min(max(.87, len(short_text) / 76), 1.14) |
|
|
|
|
|
volume = int(74 * np.random.rand() + 24) |
|
|
|
_s += f'<prosody volume=\'{volume}\'>' |
|
_s += f'<prosody rate=\'{rate}\'>' |
|
_s += f'<voice name=\'{voice}\'>' |
|
_s += '<s>' |
|
_s += short_text |
|
_s += '</s>' |
|
_s += '</voice>' |
|
_s += '</prosody>' |
|
_s += '</prosody>' |
|
_s += '</speak>' |
|
print(len(text),'\n\n\n\n\n\n\n', _s) |
|
|
|
with codecs.open('_tmp_ssml.txt', 'w', "utf-8-sig") as f: |
|
f.write(_s) |
