skytnt commited on
Commit
e0cfda2
1 Parent(s): 7b1c05b
README.md CHANGED
@@ -4,7 +4,7 @@ emoji: 😊🎙️
4
  colorFrom: red
5
  colorTo: pink
6
  sdk: gradio
7
- sdk_version: 3.6
8
  app_file: app.py
9
  pinned: false
10
  license: mit
 
4
  colorFrom: red
5
  colorTo: pink
6
  sdk: gradio
7
+ sdk_version: 3.9
8
  app_file: app.py
9
  pinned: false
10
  license: mit
app.py CHANGED
@@ -152,7 +152,8 @@ if __name__ == '__main__':
152
  example = info["example"]
153
  config_path = f"saved_model/{i}/config.json"
154
  model_path = f"saved_model/{i}/model.pth"
155
- cover_path = f"saved_model/{i}/cover.jpg"
 
156
  hps = utils.get_hparams_from_file(config_path)
157
  model = SynthesizerTrn(
158
  len(hps.symbols),
@@ -188,8 +189,9 @@ if __name__ == '__main__':
188
  to_phoneme_fn) in enumerate(models_tts):
189
  with gr.TabItem(f"model{i}"):
190
  with gr.Column():
 
191
  gr.Markdown(f"## {name}\n\n"
192
- f"![cover](file/{cover_path})\n\n"
193
  f"lang: {lang}")
194
  tts_input1 = gr.TextArea(label="Text (120 words limitation)", value=example,
195
  elem_id=f"tts-input{i}")
@@ -234,12 +236,13 @@ if __name__ == '__main__':
234
  with gr.Tabs():
235
  for i, (name, cover_path, speakers, vc_fn) in enumerate(models_vc):
236
  with gr.TabItem(f"model{i}"):
 
237
  gr.Markdown(f"## {name}\n\n"
238
- f"![cover](file/{cover_path})")
239
  vc_input1 = gr.Dropdown(label="Original Speaker", choices=speakers, type="index",
240
  value=speakers[0])
241
  vc_input2 = gr.Dropdown(label="Target Speaker", choices=speakers, type="index",
242
- value=speakers[1])
243
  vc_input3 = gr.Audio(label="Input Audio (30s limitation)")
244
  vc_submit = gr.Button("Convert", variant="primary")
245
  vc_output1 = gr.Textbox(label="Output Message")
@@ -249,8 +252,9 @@ if __name__ == '__main__':
249
  with gr.Tabs():
250
  for i, (name, cover_path, speakers, soft_vc_fn) in enumerate(models_soft_vc):
251
  with gr.TabItem(f"model{i}"):
 
252
  gr.Markdown(f"## {name}\n\n"
253
- f"![cover](file/{cover_path})")
254
  vc_input1 = gr.Dropdown(label="Target Speaker", choices=speakers, type="index",
255
  value=speakers[0])
256
  source_tabs = gr.Tabs()
 
152
  example = info["example"]
153
  config_path = f"saved_model/{i}/config.json"
154
  model_path = f"saved_model/{i}/model.pth"
155
+ cover = info["cover"]
156
+ cover_path = f"saved_model/{i}/{cover}" if cover else None
157
  hps = utils.get_hparams_from_file(config_path)
158
  model = SynthesizerTrn(
159
  len(hps.symbols),
 
189
  to_phoneme_fn) in enumerate(models_tts):
190
  with gr.TabItem(f"model{i}"):
191
  with gr.Column():
192
+ cover_markdown = f"![cover](file/{cover_path})\n\n" if cover_path else ""
193
  gr.Markdown(f"## {name}\n\n"
194
+ f"{cover_markdown}"
195
  f"lang: {lang}")
196
  tts_input1 = gr.TextArea(label="Text (120 words limitation)", value=example,
197
  elem_id=f"tts-input{i}")
 
236
  with gr.Tabs():
237
  for i, (name, cover_path, speakers, vc_fn) in enumerate(models_vc):
238
  with gr.TabItem(f"model{i}"):
239
+ cover_markdown = f"![cover](file/{cover_path})\n\n" if cover_path else ""
240
  gr.Markdown(f"## {name}\n\n"
241
+ f"{cover_markdown}")
242
  vc_input1 = gr.Dropdown(label="Original Speaker", choices=speakers, type="index",
243
  value=speakers[0])
244
  vc_input2 = gr.Dropdown(label="Target Speaker", choices=speakers, type="index",
245
+ value=speakers[min(len(speakers) - 1, 1)])
246
  vc_input3 = gr.Audio(label="Input Audio (30s limitation)")
247
  vc_submit = gr.Button("Convert", variant="primary")
248
  vc_output1 = gr.Textbox(label="Output Message")
 
252
  with gr.Tabs():
253
  for i, (name, cover_path, speakers, soft_vc_fn) in enumerate(models_soft_vc):
254
  with gr.TabItem(f"model{i}"):
255
+ cover_markdown = f"![cover](file/{cover_path})\n\n" if cover_path else ""
256
  gr.Markdown(f"## {name}\n\n"
257
+ f"{cover_markdown}")
258
  vc_input1 = gr.Dropdown(label="Target Speaker", choices=speakers, type="index",
259
  value=speakers[0])
260
  source_tabs = gr.Tabs()
models.py CHANGED
@@ -140,7 +140,8 @@ class TextEncoder(nn.Module):
140
  n_heads,
141
  n_layers,
142
  kernel_size,
143
- p_dropout):
 
144
  super().__init__()
145
  self.n_vocab = n_vocab
146
  self.out_channels = out_channels
@@ -150,9 +151,12 @@ class TextEncoder(nn.Module):
150
  self.n_layers = n_layers
151
  self.kernel_size = kernel_size
152
  self.p_dropout = p_dropout
 
153
 
154
  if self.n_vocab != 0:
155
  self.emb = nn.Embedding(n_vocab, hidden_channels)
 
 
156
  nn.init.normal_(self.emb.weight, 0.0, hidden_channels ** -0.5)
157
 
158
  self.encoder = attentions.Encoder(
@@ -164,9 +168,11 @@ class TextEncoder(nn.Module):
164
  p_dropout)
165
  self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)
166
 
167
- def forward(self, x, x_lengths):
168
  if self.n_vocab != 0:
169
  x = self.emb(x) * math.sqrt(self.hidden_channels) # [b, t, h]
 
 
170
  x = torch.transpose(x, 1, -1) # [b, h, t]
171
  x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)), 1).to(x.dtype)
172
 
@@ -392,8 +398,8 @@ class MultiPeriodDiscriminator(torch.nn.Module):
392
 
393
  class SynthesizerTrn(nn.Module):
394
  """
395
- Synthesizer for Training
396
- """
397
 
398
  def __init__(self,
399
  n_vocab,
@@ -415,6 +421,7 @@ class SynthesizerTrn(nn.Module):
415
  n_speakers=0,
416
  gin_channels=0,
417
  use_sdp=True,
 
418
  **kwargs):
419
 
420
  super().__init__()
@@ -446,7 +453,8 @@ class SynthesizerTrn(nn.Module):
446
  n_heads,
447
  n_layers,
448
  kernel_size,
449
- p_dropout)
 
450
  self.dec = Generator(inter_channels, resblock, resblock_kernel_sizes, resblock_dilation_sizes, upsample_rates,
451
  upsample_initial_channel, upsample_kernel_sizes, gin_channels=gin_channels)
452
  self.enc_q = PosteriorEncoder(spec_channels, inter_channels, hidden_channels, 5, 1, 16,
@@ -461,9 +469,9 @@ class SynthesizerTrn(nn.Module):
461
  if n_speakers > 1:
462
  self.emb_g = nn.Embedding(n_speakers, gin_channels)
463
 
464
- def forward(self, x, x_lengths, y, y_lengths, sid=None):
465
 
466
- x, m_p, logs_p, x_mask = self.enc_p(x, x_lengths)
467
  if self.n_speakers > 1:
468
  g = self.emb_g(sid).unsqueeze(-1) # [b, h, 1]
469
  else:
@@ -502,8 +510,9 @@ class SynthesizerTrn(nn.Module):
502
  o = self.dec(z_slice, g=g)
503
  return o, l_length, attn, ids_slice, x_mask, y_mask, (z, z_p, m_p, logs_p, m_q, logs_q)
504
 
505
- def infer(self, x, x_lengths, sid=None, noise_scale=1, length_scale=1, noise_scale_w=1., max_len=None):
506
- x, m_p, logs_p, x_mask = self.enc_p(x, x_lengths)
 
507
  if self.n_speakers > 1:
508
  g = self.emb_g(sid).unsqueeze(-1) # [b, h, 1]
509
  else:
 
140
  n_heads,
141
  n_layers,
142
  kernel_size,
143
+ p_dropout,
144
+ emotion_embedding):
145
  super().__init__()
146
  self.n_vocab = n_vocab
147
  self.out_channels = out_channels
 
151
  self.n_layers = n_layers
152
  self.kernel_size = kernel_size
153
  self.p_dropout = p_dropout
154
+ self.emotion_embedding = emotion_embedding
155
 
156
  if self.n_vocab != 0:
157
  self.emb = nn.Embedding(n_vocab, hidden_channels)
158
+ if emotion_embedding:
159
+ self.emo_proj = nn.Linear(1024, hidden_channels)
160
  nn.init.normal_(self.emb.weight, 0.0, hidden_channels ** -0.5)
161
 
162
  self.encoder = attentions.Encoder(
 
168
  p_dropout)
169
  self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)
170
 
171
+ def forward(self, x, x_lengths, emotion_embedding=None):
172
  if self.n_vocab != 0:
173
  x = self.emb(x) * math.sqrt(self.hidden_channels) # [b, t, h]
174
+ if emotion_embedding is not None:
175
+ x = x + self.emo_proj(emotion_embedding.unsqueeze(1))
176
  x = torch.transpose(x, 1, -1) # [b, h, t]
177
  x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)), 1).to(x.dtype)
178
 
 
398
 
399
  class SynthesizerTrn(nn.Module):
400
  """
401
+ Synthesizer for Training
402
+ """
403
 
404
  def __init__(self,
405
  n_vocab,
 
421
  n_speakers=0,
422
  gin_channels=0,
423
  use_sdp=True,
424
+ emotion_embedding=False,
425
  **kwargs):
426
 
427
  super().__init__()
 
453
  n_heads,
454
  n_layers,
455
  kernel_size,
456
+ p_dropout,
457
+ emotion_embedding)
458
  self.dec = Generator(inter_channels, resblock, resblock_kernel_sizes, resblock_dilation_sizes, upsample_rates,
459
  upsample_initial_channel, upsample_kernel_sizes, gin_channels=gin_channels)
460
  self.enc_q = PosteriorEncoder(spec_channels, inter_channels, hidden_channels, 5, 1, 16,
 
469
  if n_speakers > 1:
470
  self.emb_g = nn.Embedding(n_speakers, gin_channels)
471
 
472
+ def forward(self, x, x_lengths, y, y_lengths, sid=None, emotion_embedding=None):
473
 
474
+ x, m_p, logs_p, x_mask = self.enc_p(x, x_lengths, emotion_embedding)
475
  if self.n_speakers > 1:
476
  g = self.emb_g(sid).unsqueeze(-1) # [b, h, 1]
477
  else:
 
510
  o = self.dec(z_slice, g=g)
511
  return o, l_length, attn, ids_slice, x_mask, y_mask, (z, z_p, m_p, logs_p, m_q, logs_q)
512
 
513
+ def infer(self, x, x_lengths, sid=None, noise_scale=1, length_scale=1, noise_scale_w=1., max_len=None,
514
+ emotion_embedding=None):
515
+ x, m_p, logs_p, x_mask = self.enc_p(x, x_lengths, emotion_embedding)
516
  if self.n_speakers > 1:
517
  g = self.emb_g(sid).unsqueeze(-1) # [b, h, 1]
518
  else:
requirements.txt CHANGED
@@ -8,11 +8,18 @@ tensorboard
8
  torch
9
  torchvision
10
  torchaudio
11
- Unidecode
12
  pyopenjtalk
13
  jamo
14
  pypinyin
15
  ko_pron
16
  jieba
17
  cn2an
18
- gradio
 
 
 
 
 
 
 
 
8
  torch
9
  torchvision
10
  torchaudio
11
+ unidecode
12
  pyopenjtalk
13
  jamo
14
  pypinyin
15
  ko_pron
16
  jieba
17
  cn2an
18
+ protobuf
19
+ inflect
20
+ eng_to_ipa
21
+ ko_pron
22
+ indic_transliteration
23
+ num_thai
24
+ opencc
25
+ gradio
saved_model/14/config.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:50618702d27249d4557a39afd74ca19191f9537a0e192a4afeb0559967aa5527
3
+ size 1592
saved_model/14/model.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2801051beb8f90bd9785604fad617bf95a8f05df93722ad8993128dd6bf91301
3
+ size 158912845
saved_model/15/config.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0d3cb3ce57d9e111d83d4f2570956be6621aff74166929f83f7b11d985a1858b
3
+ size 363860
saved_model/15/model.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5233ca0318ee16c38fa6ab5aaa8b3b12521f263b67bb8c8aeb12ec0e3bc2b067
3
+ size 161855565
saved_model/info.json CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:ecd789104d62a5fc85bd85dd8bd9d4494c30643993d85a5903de36ba38e38be7
3
- size 1849
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:db3e22ccce9b2dad016ac66e10a940f67757553d3656a385be93f717f47513c3
3
+ size 2761
text/cantonese.py ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import re
2
+ import cn2an
3
+ import opencc
4
+
5
+
6
+ converter = opencc.OpenCC('jyutjyu')
7
+
8
+ # List of (Latin alphabet, ipa) pairs:
9
+ _latin_to_ipa = [(re.compile('%s' % x[0]), x[1]) for x in [
10
+ ('A', 'ei˥'),
11
+ ('B', 'biː˥'),
12
+ ('C', 'siː˥'),
13
+ ('D', 'tiː˥'),
14
+ ('E', 'iː˥'),
15
+ ('F', 'e˥fuː˨˩'),
16
+ ('G', 'tsiː˥'),
17
+ ('H', 'ɪk̚˥tsʰyː˨˩'),
18
+ ('I', 'ɐi˥'),
19
+ ('J', 'tsei˥'),
20
+ ('K', 'kʰei˥'),
21
+ ('L', 'e˥llou˨˩'),
22
+ ('M', 'ɛːm˥'),
23
+ ('N', 'ɛːn˥'),
24
+ ('O', 'ou˥'),
25
+ ('P', 'pʰiː˥'),
26
+ ('Q', 'kʰiːu˥'),
27
+ ('R', 'aː˥lou˨˩'),
28
+ ('S', 'ɛː˥siː˨˩'),
29
+ ('T', 'tʰiː˥'),
30
+ ('U', 'juː˥'),
31
+ ('V', 'wiː˥'),
32
+ ('W', 'tʊk̚˥piː˥juː˥'),
33
+ ('X', 'ɪk̚˥siː˨˩'),
34
+ ('Y', 'waːi˥'),
35
+ ('Z', 'iː˨sɛːt̚˥')
36
+ ]]
37
+
38
+
39
+ def number_to_cantonese(text):
40
+ return re.sub(r'\d+(?:\.?\d+)?', lambda x: cn2an.an2cn(x.group()), text)
41
+
42
+
43
+ def latin_to_ipa(text):
44
+ for regex, replacement in _latin_to_ipa:
45
+ text = re.sub(regex, replacement, text)
46
+ return text
47
+
48
+
49
+ def cantonese_to_ipa(text):
50
+ text = number_to_cantonese(text.upper())
51
+ text = converter.convert(text).replace('-','').replace('$',' ')
52
+ text = re.sub(r'[A-Z]', lambda x: latin_to_ipa(x.group())+' ', text)
53
+ text = re.sub(r'[、;:]', ',', text)
54
+ text = re.sub(r'\s*,\s*', ', ', text)
55
+ text = re.sub(r'\s*。\s*', '. ', text)
56
+ text = re.sub(r'\s*?\s*', '? ', text)
57
+ text = re.sub(r'\s*!\s*', '! ', text)
58
+ text = re.sub(r'\s*$', '', text)
59
+ return text
text/cleaners.py CHANGED
@@ -4,8 +4,7 @@ import re
4
  def japanese_cleaners(text):
5
  from text.japanese import japanese_to_romaji_with_accent
6
  text = japanese_to_romaji_with_accent(text)
7
- if len(text) == 0 or re.match('[A-Za-z]', text[-1]):
8
- text += '.'
9
  return text
10
 
11
 
@@ -19,8 +18,7 @@ def korean_cleaners(text):
19
  text = latin_to_hangul(text)
20
  text = number_to_hangul(text)
21
  text = divide_hangul(text)
22
- if len(text) == 0 or re.match('[\u3131-\u3163]', text[-1]):
23
- text += '.'
24
  return text
25
 
26
 
@@ -30,32 +28,25 @@ def chinese_cleaners(text):
30
  text = number_to_chinese(text)
31
  text = chinese_to_bopomofo(text)
32
  text = latin_to_bopomofo(text)
33
- if len(text) == 0 or re.match('[ˉˊˇˋ˙]', text[-1]):
34
- text += '。'
35
  return text
36
 
37
 
38
  def zh_ja_mixture_cleaners(text):
39
  from text.mandarin import chinese_to_romaji
40
  from text.japanese import japanese_to_romaji_with_accent
41
- chinese_texts = re.findall(r'\[ZH\].*?\[ZH\]', text)
42
- japanese_texts = re.findall(r'\[JA\].*?\[JA\]', text)
43
- for chinese_text in chinese_texts:
44
- cleaned_text = chinese_to_romaji(chinese_text[4:-4])
45
- text = text.replace(chinese_text, cleaned_text + ' ', 1)
46
- for japanese_text in japanese_texts:
47
- cleaned_text = japanese_to_romaji_with_accent(
48
- japanese_text[4:-4]).replace('ts', 'ʦ').replace('u', 'ɯ').replace('...', '…')
49
- text = text.replace(japanese_text, cleaned_text + ' ', 1)
50
- text = text[:-1]
51
- if len(text) == 0 or re.match('[A-Za-zɯɹəɥ→↓↑]', text[-1]):
52
- text += '.'
53
  return text
54
 
55
 
56
  def sanskrit_cleaners(text):
57
  text = text.replace('॥', '।').replace('ॐ', 'ओम्')
58
- if len(text) == 0 or text[-1] != '।':
59
  text += ' ।'
60
  return text
61
 
@@ -65,23 +56,91 @@ def cjks_cleaners(text):
65
  from text.japanese import japanese_to_ipa
66
  from text.korean import korean_to_lazy_ipa
67
  from text.sanskrit import devanagari_to_ipa
68
- chinese_texts = re.findall(r'\[ZH\].*?\[ZH\]', text)
69
- japanese_texts = re.findall(r'\[JA\].*?\[JA\]', text)
70
- korean_texts = re.findall(r'\[KO\].*?\[KO\]', text)
71
- sanskrit_texts = re.findall(r'\[SA\].*?\[SA\]', text)
72
- for chinese_text in chinese_texts:
73
- cleaned_text = chinese_to_lazy_ipa(chinese_text[4:-4])
74
- text = text.replace(chinese_text, cleaned_text + ' ', 1)
75
- for japanese_text in japanese_texts:
76
- cleaned_text = japanese_to_ipa(japanese_text[4:-4])
77
- text = text.replace(japanese_text, cleaned_text + ' ', 1)
78
- for korean_text in korean_texts:
79
- cleaned_text = korean_to_lazy_ipa(korean_text[4:-4])
80
- text = text.replace(korean_text, cleaned_text + ' ', 1)
81
- for sanskrit_text in sanskrit_texts:
82
- cleaned_text = devanagari_to_ipa(sanskrit_text[4:-4])
83
- text = text.replace(sanskrit_text, cleaned_text + ' ', 1)
84
- text = text[:-1]
85
- if len(text) == 0 or re.match(r'[^\.,!\?\-…~]', text[-1]):
86
- text += '.'
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
87
  return text
 
4
  def japanese_cleaners(text):
5
  from text.japanese import japanese_to_romaji_with_accent
6
  text = japanese_to_romaji_with_accent(text)
7
+ text = re.sub(r'([A-Za-z])$', r'\1.', text)
 
8
  return text
9
 
10
 
 
18
  text = latin_to_hangul(text)
19
  text = number_to_hangul(text)
20
  text = divide_hangul(text)
21
+ text = re.sub(r'([\u3131-\u3163])$', r'\1.', text)
 
22
  return text
23
 
24
 
 
28
  text = number_to_chinese(text)
29
  text = chinese_to_bopomofo(text)
30
  text = latin_to_bopomofo(text)
31
+ text = re.sub(r'([ˉˊˇˋ˙])$', r'\1。', text)
 
32
  return text
33
 
34
 
35
  def zh_ja_mixture_cleaners(text):
36
  from text.mandarin import chinese_to_romaji
37
  from text.japanese import japanese_to_romaji_with_accent
38
+ text = re.sub(r'\[ZH\](.*?)\[ZH\]',
39
+ lambda x: chinese_to_romaji(x.group(1))+' ', text)
40
+ text = re.sub(r'\[JA\](.*?)\[JA\]', lambda x: japanese_to_romaji_with_accent(
41
+ x.group(1)).replace('ts', 'ʦ').replace('u', 'ɯ').replace('...', '…')+' ', text)
42
+ text = re.sub(r'\s+$', '', text)
43
+ text = re.sub(r'([^\.,!\?\-…~])$', r'\1.', text)
 
 
 
 
 
 
44
  return text
45
 
46
 
47
  def sanskrit_cleaners(text):
48
  text = text.replace('॥', '।').replace('ॐ', 'ओम्')
49
+ if text[-1] != '।':
50
  text += ' ।'
51
  return text
52
 
 
56
  from text.japanese import japanese_to_ipa
57
  from text.korean import korean_to_lazy_ipa
58
  from text.sanskrit import devanagari_to_ipa
59
+ from text.english import english_to_lazy_ipa
60
+ text = re.sub(r'\[ZH\](.*?)\[ZH\]',
61
+ lambda x: chinese_to_lazy_ipa(x.group(1))+' ', text)
62
+ text = re.sub(r'\[JA\](.*?)\[JA\]',
63
+ lambda x: japanese_to_ipa(x.group(1))+' ', text)
64
+ text = re.sub(r'\[KO\](.*?)\[KO\]',
65
+ lambda x: korean_to_lazy_ipa(x.group(1))+' ', text)
66
+ text = re.sub(r'\[SA\](.*?)\[SA\]',
67
+ lambda x: devanagari_to_ipa(x.group(1))+' ', text)
68
+ text = re.sub(r'\[EN\](.*?)\[EN\]',
69
+ lambda x: english_to_lazy_ipa(x.group(1))+' ', text)
70
+ text = re.sub(r'\s+$', '', text)
71
+ text = re.sub(r'([^\.,!\?\-…~])$', r'\1.', text)
72
+ return text
73
+
74
+
75
+ def cjke_cleaners(text):
76
+ from text.mandarin import chinese_to_lazy_ipa
77
+ from text.japanese import japanese_to_ipa
78
+ from text.korean import korean_to_ipa
79
+ from text.english import english_to_ipa2
80
+ text = re.sub(r'\[ZH\](.*?)\[ZH\]', lambda x: chinese_to_lazy_ipa(x.group(1)).replace(
81
+ 'ʧ', 'tʃ').replace('ʦ', 'ts').replace('ɥan', 'ɥæn')+' ', text)
82
+ text = re.sub(r'\[JA\](.*?)\[JA\]', lambda x: japanese_to_ipa(x.group(1)).replace('ʧ', 'tʃ').replace(
83
+ 'ʦ', 'ts').replace('ɥan', 'ɥæn').replace('ʥ', 'dz')+' ', text)
84
+ text = re.sub(r'\[KO\](.*?)\[KO\]',
85
+ lambda x: korean_to_ipa(x.group(1))+' ', text)
86
+ text = re.sub(r'\[EN\](.*?)\[EN\]', lambda x: english_to_ipa2(x.group(1)).replace('ɑ', 'a').replace(
87
+ 'ɔ', 'o').replace('ɛ', 'e').replace('ɪ', 'i').replace('ʊ', 'u')+' ', text)
88
+ text = re.sub(r'\s+$', '', text)
89
+ text = re.sub(r'([^\.,!\?\-…~])$', r'\1.', text)
90
+ return text
91
+
92
+
93
+ def cjke_cleaners2(text):
94
+ from text.mandarin import chinese_to_ipa
95
+ from text.japanese import japanese_to_ipa2
96
+ from text.korean import korean_to_ipa
97
+ from text.english import english_to_ipa2
98
+ text = re.sub(r'\[ZH\](.*?)\[ZH\]',
99
+ lambda x: chinese_to_ipa(x.group(1))+' ', text)
100
+ text = re.sub(r'\[JA\](.*?)\[JA\]',
101
+ lambda x: japanese_to_ipa2(x.group(1))+' ', text)
102
+ text = re.sub(r'\[KO\](.*?)\[KO\]',
103
+ lambda x: korean_to_ipa(x.group(1))+' ', text)
104
+ text = re.sub(r'\[EN\](.*?)\[EN\]',
105
+ lambda x: english_to_ipa2(x.group(1))+' ', text)
106
+ text = re.sub(r'\s+$', '', text)
107
+ text = re.sub(r'([^\.,!\?\-…~])$', r'\1.', text)
108
+ return text
109
+
110
+
111
+ def thai_cleaners(text):
112
+ from text.thai import num_to_thai, latin_to_thai
113
+ text = num_to_thai(text)
114
+ text = latin_to_thai(text)
115
+ return text
116
+
117
+
118
+ def shanghainese_cleaners(text):
119
+ from text.shanghainese import shanghainese_to_ipa
120
+ text = shanghainese_to_ipa(text)
121
+ text = re.sub(r'([^\.,!\?\-…~])$', r'\1.', text)
122
+ return text
123
+
124
+
125
+ def chinese_dialect_cleaners(text):
126
+ from text.mandarin import chinese_to_ipa2
127
+ from text.japanese import japanese_to_ipa3
128
+ from text.shanghainese import shanghainese_to_ipa
129
+ from text.cantonese import cantonese_to_ipa
130
+ from text.english import english_to_lazy_ipa2
131
+ from text.ngu_dialect import ngu_dialect_to_ipa
132
+ text = re.sub(r'\[ZH\](.*?)\[ZH\]',
133
+ lambda x: chinese_to_ipa2(x.group(1))+' ', text)
134
+ text = re.sub(r'\[JA\](.*?)\[JA\]',
135
+ lambda x: japanese_to_ipa3(x.group(1)).replace('Q', 'ʔ')+' ', text)
136
+ text = re.sub(r'\[SH\](.*?)\[SH\]', lambda x: shanghainese_to_ipa(x.group(1)).replace('1', '˥˧').replace('5',
137
+ '˧˧˦').replace('6', '˩˩˧').replace('7', '˥').replace('8', '˩˨').replace('ᴀ', 'ɐ').replace('ᴇ', 'e')+' ', text)
138
+ text = re.sub(r'\[GD\](.*?)\[GD\]',
139
+ lambda x: cantonese_to_ipa(x.group(1))+' ', text)
140
+ text = re.sub(r'\[EN\](.*?)\[EN\]',
141
+ lambda x: english_to_lazy_ipa2(x.group(1))+' ', text)
142
+ text = re.sub(r'\[([A-Z]{2})\](.*?)\[\1\]', lambda x: ngu_dialect_to_ipa(x.group(2), x.group(
143
+ 1)).replace('ʣ', 'dz').replace('ʥ', 'dʑ').replace('ʦ', 'ts').replace('ʨ', 'tɕ')+' ', text)
144
+ text = re.sub(r'\s+$', '', text)
145
+ text = re.sub(r'([^\.,!\?\-…~])$', r'\1.', text)
146
  return text
text/english.py ADDED
@@ -0,0 +1,188 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """ from https://github.com/keithito/tacotron """
2
+
3
+ '''
4
+ Cleaners are transformations that run over the input text at both training and eval time.
5
+
6
+ Cleaners can be selected by passing a comma-delimited list of cleaner names as the "cleaners"
7
+ hyperparameter. Some cleaners are English-specific. You'll typically want to use:
8
+ 1. "english_cleaners" for English text
9
+ 2. "transliteration_cleaners" for non-English text that can be transliterated to ASCII using
10
+ the Unidecode library (https://pypi.python.org/pypi/Unidecode)
11
+ 3. "basic_cleaners" if you do not want to transliterate (in this case, you should also update
12
+ the symbols in symbols.py to match your data).
13
+ '''
14
+
15
+
16
+ # Regular expression matching whitespace:
17
+
18
+
19
+ import re
20
+ import inflect
21
+ from unidecode import unidecode
22
+ import eng_to_ipa as ipa
23
+ _inflect = inflect.engine()
24
+ _comma_number_re = re.compile(r'([0-9][0-9\,]+[0-9])')
25
+ _decimal_number_re = re.compile(r'([0-9]+\.[0-9]+)')
26
+ _pounds_re = re.compile(r'£([0-9\,]*[0-9]+)')
27
+ _dollars_re = re.compile(r'\$([0-9\.\,]*[0-9]+)')
28
+ _ordinal_re = re.compile(r'[0-9]+(st|nd|rd|th)')
29
+ _number_re = re.compile(r'[0-9]+')
30
+
31
+ # List of (regular expression, replacement) pairs for abbreviations:
32
+ _abbreviations = [(re.compile('\\b%s\\.' % x[0], re.IGNORECASE), x[1]) for x in [
33
+ ('mrs', 'misess'),
34
+ ('mr', 'mister'),
35
+ ('dr', 'doctor'),
36
+ ('st', 'saint'),
37
+ ('co', 'company'),
38
+ ('jr', 'junior'),
39
+ ('maj', 'major'),
40
+ ('gen', 'general'),
41
+ ('drs', 'doctors'),
42
+ ('rev', 'reverend'),
43
+ ('lt', 'lieutenant'),
44
+ ('hon', 'honorable'),
45
+ ('sgt', 'sergeant'),
46
+ ('capt', 'captain'),
47
+ ('esq', 'esquire'),
48
+ ('ltd', 'limited'),
49
+ ('col', 'colonel'),
50
+ ('ft', 'fort'),
51
+ ]]
52
+
53
+
54
+ # List of (ipa, lazy ipa) pairs:
55
+ _lazy_ipa = [(re.compile('%s' % x[0]), x[1]) for x in [
56
+ ('r', 'ɹ'),
57
+ ('æ', 'e'),
58
+ ('ɑ', 'a'),
59
+ ('ɔ', 'o'),
60
+ ('ð', 'z'),
61
+ ('θ', 's'),
62
+ ('ɛ', 'e'),
63
+ ('ɪ', 'i'),
64
+ ('ʊ', 'u'),
65
+ ('ʒ', 'ʥ'),
66
+ ('ʤ', 'ʥ'),
67
+ ('ˈ', '↓'),
68
+ ]]
69
+
70
+ # List of (ipa, lazy ipa2) pairs:
71
+ _lazy_ipa2 = [(re.compile('%s' % x[0]), x[1]) for x in [
72
+ ('r', 'ɹ'),
73
+ ('ð', 'z'),
74
+ ('θ', 's'),
75
+ ('ʒ', 'ʑ'),
76
+ ('ʤ', 'dʑ'),
77
+ ('ˈ', '↓'),
78
+ ]]
79
+
80
+ # List of (ipa, ipa2) pairs
81
+ _ipa_to_ipa2 = [(re.compile('%s' % x[0]), x[1]) for x in [
82
+ ('r', 'ɹ'),
83
+ ('ʤ', 'dʒ'),
84
+ ('ʧ', 'tʃ')
85
+ ]]
86
+
87
+
88
+ def expand_abbreviations(text):
89
+ for regex, replacement in _abbreviations:
90
+ text = re.sub(regex, replacement, text)
91
+ return text
92
+
93
+
94
+ def collapse_whitespace(text):
95
+ return re.sub(r'\s+', ' ', text)
96
+
97
+
98
+ def _remove_commas(m):
99
+ return m.group(1).replace(',', '')
100
+
101
+
102
+ def _expand_decimal_point(m):
103
+ return m.group(1).replace('.', ' point ')
104
+
105
+
106
+ def _expand_dollars(m):
107
+ match = m.group(1)
108
+ parts = match.split('.')
109
+ if len(parts) > 2:
110
+ return match + ' dollars' # Unexpected format
111
+ dollars = int(parts[0]) if parts[0] else 0
112
+ cents = int(parts[1]) if len(parts) > 1 and parts[1] else 0
113
+ if dollars and cents:
114
+ dollar_unit = 'dollar' if dollars == 1 else 'dollars'
115
+ cent_unit = 'cent' if cents == 1 else 'cents'
116
+ return '%s %s, %s %s' % (dollars, dollar_unit, cents, cent_unit)
117
+ elif dollars:
118
+ dollar_unit = 'dollar' if dollars == 1 else 'dollars'
119
+ return '%s %s' % (dollars, dollar_unit)
120
+ elif cents:
121
+ cent_unit = 'cent' if cents == 1 else 'cents'
122
+ return '%s %s' % (cents, cent_unit)
123
+ else:
124
+ return 'zero dollars'
125
+
126
+
127
+ def _expand_ordinal(m):
128
+ return _inflect.number_to_words(m.group(0))
129
+
130
+
131
+ def _expand_number(m):
132
+ num = int(m.group(0))
133
+ if num > 1000 and num < 3000:
134
+ if num == 2000:
135
+ return 'two thousand'
136
+ elif num > 2000 and num < 2010:
137
+ return 'two thousand ' + _inflect.number_to_words(num % 100)
138
+ elif num % 100 == 0:
139
+ return _inflect.number_to_words(num // 100) + ' hundred'
140
+ else:
141
+ return _inflect.number_to_words(num, andword='', zero='oh', group=2).replace(', ', ' ')
142
+ else:
143
+ return _inflect.number_to_words(num, andword='')
144
+
145
+
146
+ def normalize_numbers(text):
147
+ text = re.sub(_comma_number_re, _remove_commas, text)
148
+ text = re.sub(_pounds_re, r'\1 pounds', text)
149
+ text = re.sub(_dollars_re, _expand_dollars, text)
150
+ text = re.sub(_decimal_number_re, _expand_decimal_point, text)
151
+ text = re.sub(_ordinal_re, _expand_ordinal, text)
152
+ text = re.sub(_number_re, _expand_number, text)
153
+ return text
154
+
155
+
156
+ def mark_dark_l(text):
157
+ return re.sub(r'l([^aeiouæɑɔəɛɪʊ ]*(?: |$))', lambda x: 'ɫ'+x.group(1), text)
158
+
159
+
160
+ def english_to_ipa(text):
161
+ text = unidecode(text).lower()
162
+ text = expand_abbreviations(text)
163
+ text = normalize_numbers(text)
164
+ phonemes = ipa.convert(text)
165
+ phonemes = collapse_whitespace(phonemes)
166
+ return phonemes
167
+
168
+
169
+ def english_to_lazy_ipa(text):
170
+ text = english_to_ipa(text)
171
+ for regex, replacement in _lazy_ipa:
172
+ text = re.sub(regex, replacement, text)
173
+ return text
174
+
175
+
176
+ def english_to_ipa2(text):
177
+ text = english_to_ipa(text)
178
+ text = mark_dark_l(text)
179
+ for regex, replacement in _ipa_to_ipa2:
180
+ text = re.sub(regex, replacement, text)
181
+ return text.replace('...', '…')
182
+
183
+
184
+ def english_to_lazy_ipa2(text):
185
+ text = english_to_ipa(text)
186
+ for regex, replacement in _lazy_ipa2:
187
+ text = re.sub(regex, replacement, text)
188
+ return text
text/japanese.py CHANGED
@@ -17,10 +17,9 @@ _symbols_to_japanese = [(re.compile('%s' % x[0]), x[1]) for x in [
17
  ]]
18
 
19
  # List of (romaji, ipa) pairs for marks:
20
- _romaji_to_ipa = [(re.compile('%s' % x[0], re.IGNORECASE), x[1]) for x in [
21
  ('ts', 'ʦ'),
22
  ('u', 'ɯ'),
23
- ('...', '…'),
24
  ('j', 'ʥ'),
25
  ('y', 'j'),
26
  ('ni', 'n^i'),
@@ -33,35 +32,37 @@ _romaji_to_ipa = [(re.compile('%s' % x[0], re.IGNORECASE), x[1]) for x in [
33
  ('r', 'ɾ')
34
  ]]
35
 
36
- # Dictinary of (consonant, sokuon) pairs:
37
- _real_sokuon = {
38
- 'k': 'k#',
39
- 'g': 'k#',
40
- 't': 't#',
41
- 'd': 't#',
42
- 'ʦ': 't#',
43
- 'ʧ': 't#',
44
- 'ʥ': 't#',
45
- 'j': 't#',
46
- 's': 's',
47
- 'ʃ': 's',
48
- 'p': 'p#',
49
- 'b': 'p#'
50
- }
51
-
52
- # Dictinary of (consonant, hatsuon) pairs:
53
- _real_hatsuon = {
54
- 'p': 'm',
55
- 'b': 'm',
56
- 'm': 'm',
57
- 't': 'n',
58
- 'd': 'n',
59
- 'n': 'n',
60
- 'ʧ': 'n^',
61
- 'ʥ': 'n^',
62
- 'k': 'ŋ',
63
- 'g': 'ŋ'
64
- }
 
 
65
 
66
 
67
  def symbols_to_japanese(text):
@@ -112,21 +113,41 @@ def japanese_to_romaji_with_accent(text):
112
 
113
 
114
  def get_real_sokuon(text):
115
- text=re.sub('Q[↑↓]*(.)',lambda x:_real_sokuon[x.group(1)]+x.group(0)[1:] if x.group(1) in _real_sokuon.keys() else x.group(0),text)
116
- return text
 
117
 
118
 
119
  def get_real_hatsuon(text):
120
- text=re.sub('N[↑↓]*(.)',lambda x:_real_hatsuon[x.group(1)]+x.group(0)[1:] if x.group(1) in _real_hatsuon.keys() else x.group(0),text)
121
- return text
 
122
 
123
 
124
  def japanese_to_ipa(text):
125
- text=japanese_to_romaji_with_accent(text)
 
 
 
 
126
  for regex, replacement in _romaji_to_ipa:
127
  text = re.sub(regex, replacement, text)
128
- text = re.sub(
129
- r'([A-Za-zɯ])\1+', lambda x: x.group(0)[0]+'ː'*(len(x.group(0))-1), text)
 
 
 
130
  text = get_real_sokuon(text)
131
  text = get_real_hatsuon(text)
 
 
 
 
 
 
 
 
 
 
 
132
  return text
 
17
  ]]
18
 
19
  # List of (romaji, ipa) pairs for marks:
20
+ _romaji_to_ipa = [(re.compile('%s' % x[0]), x[1]) for x in [
21
  ('ts', 'ʦ'),
22
  ('u', 'ɯ'),
 
23
  ('j', 'ʥ'),
24
  ('y', 'j'),
25
  ('ni', 'n^i'),
 
32
  ('r', 'ɾ')
33
  ]]
34
 
35
+ # List of (romaji, ipa2) pairs for marks:
36
+ _romaji_to_ipa2 = [(re.compile('%s' % x[0]), x[1]) for x in [
37
+ ('u', 'ɯ'),
38
+ ('ʧ', ''),
39
+ ('j', ''),
40
+ ('y', 'j'),
41
+ ('ni', 'n^i'),
42
+ ('nj', 'n^'),
43
+ ('hi', 'çi'),
44
+ ('hj', 'ç'),
45
+ ('f', 'ɸ'),
46
+ ('I', 'i*'),
47
+ ('U', 'ɯ*'),
48
+ ('r', 'ɾ')
49
+ ]]
50
+
51
+ # List of (consonant, sokuon) pairs:
52
+ _real_sokuon = [(re.compile('%s' % x[0]), x[1]) for x in [
53
+ (r'Q([↑↓]*[kg])', r'k#\1'),
54
+ (r'Q([↑↓]*[tdjʧ])', r't#\1'),
55
+ (r'Q([↑↓]*[sʃ])', r's\1'),
56
+ (r'Q([↑↓]*[pb])', r'p#\1')
57
+ ]]
58
+
59
+ # List of (consonant, hatsuon) pairs:
60
+ _real_hatsuon = [(re.compile('%s' % x[0]), x[1]) for x in [
61
+ (r'N([↑↓]*[pbm])', r'm\1'),
62
+ (r'N([↑↓]*[ʧʥj])', r'n^\1'),
63
+ (r'N([↑↓]*[tdn])', r'n\1'),
64
+ (r'N([↑↓]*[kg])', r'ŋ\1')
65
+ ]]
66
 
67
 
68
  def symbols_to_japanese(text):
 
113
 
114
 
115
  def get_real_sokuon(text):
116
+ for regex, replacement in _real_sokuon:
117
+ text = re.sub(regex, replacement, text)
118
+ return text
119
 
120
 
121
  def get_real_hatsuon(text):
122
+ for regex, replacement in _real_hatsuon:
123
+ text = re.sub(regex, replacement, text)
124
+ return text
125
 
126
 
127
  def japanese_to_ipa(text):
128
+ text = japanese_to_romaji_with_accent(text).replace('...', '…')
129
+ text = re.sub(
130
+ r'([aiueo])\1+', lambda x: x.group(0)[0]+'ː'*(len(x.group(0))-1), text)
131
+ text = get_real_sokuon(text)
132
+ text = get_real_hatsuon(text)
133
  for regex, replacement in _romaji_to_ipa:
134
  text = re.sub(regex, replacement, text)
135
+ return text
136
+
137
+
138
+ def japanese_to_ipa2(text):
139
+ text = japanese_to_romaji_with_accent(text).replace('...', '…')
140
  text = get_real_sokuon(text)
141
  text = get_real_hatsuon(text)
142
+ for regex, replacement in _romaji_to_ipa2:
143
+ text = re.sub(regex, replacement, text)
144
+ return text
145
+
146
+
147
+ def japanese_to_ipa3(text):
148
+ text = japanese_to_ipa2(text).replace('n^', 'ȵ').replace(
149
+ 'ʃ', 'ɕ').replace('*', '\u0325').replace('#', '\u031a')
150
+ text = re.sub(
151
+ r'([aiɯeo])\1+', lambda x: x.group(0)[0]+'ː'*(len(x.group(0))-1), text)
152
+ text = re.sub(r'((?:^|\s)(?:ts|tɕ|[kpt]))', r'\1ʰ', text)
153
  return text
text/korean.py CHANGED
@@ -199,7 +199,12 @@ def number_to_hangul(text):
199
  def korean_to_lazy_ipa(text):
200
  text = latin_to_hangul(text)
201
  text = number_to_hangul(text)
202
- text=re.sub('[\uac00-\ud7af]+',lambda x:ko_pron.romanise(x.group(0),'ipa'),text).split('] ~ [')[0]
203
  for regex, replacement in _ipa_to_lazy_ipa:
204
  text = re.sub(regex, replacement, text)
205
  return text
 
 
 
 
 
 
199
  def korean_to_lazy_ipa(text):
200
  text = latin_to_hangul(text)
201
  text = number_to_hangul(text)
202
+ text=re.sub('[\uac00-\ud7af]+',lambda x:ko_pron.romanise(x.group(0),'ipa').split('] ~ [')[0],text)
203
  for regex, replacement in _ipa_to_lazy_ipa:
204
  text = re.sub(regex, replacement, text)
205
  return text
206
+
207
+
208
+ def korean_to_ipa(text):
209
+ text = korean_to_lazy_ipa(text)
210
+ return text.replace('ʧ','tʃ').replace('ʥ','dʑ')
text/mandarin.py CHANGED
@@ -100,7 +100,6 @@ _bopomofo_to_romaji = [(re.compile('%s' % x[0]), x[1]) for x in [
100
  ('—', '-')
101
  ]]
102
 
103
-
104
  # List of (romaji, ipa) pairs:
105
  _romaji_to_ipa = [(re.compile('%s' % x[0], re.IGNORECASE), x[1]) for x in [
106
  ('ʃy', 'ʃ'),
@@ -112,6 +111,130 @@ _romaji_to_ipa = [(re.compile('%s' % x[0], re.IGNORECASE), x[1]) for x in [
112
  ('h', 'x')
113
  ]]
114
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
115
 
116
  def number_to_chinese(text):
117
  numbers = re.findall(r'\d+(?:\.?\d+)?', text)
@@ -130,8 +253,7 @@ def chinese_to_bopomofo(text):
130
  text += word
131
  continue
132
  for i in range(len(bopomofos)):
133
- if re.match('[\u3105-\u3129]', bopomofos[i][-1]):
134
- bopomofos[i] += 'ˉ'
135
  if text != '':
136
  text += ' '
137
  text += ''.join(bopomofos)
@@ -150,17 +272,28 @@ def bopomofo_to_romaji(text):
150
  return text
151
 
152
 
 
 
 
 
 
 
 
 
 
 
 
 
153
  def chinese_to_romaji(text):
154
  text = number_to_chinese(text)
155
  text = chinese_to_bopomofo(text)
156
  text = latin_to_bopomofo(text)
157
  text = bopomofo_to_romaji(text)
158
- text = re.sub('i[aoe]', lambda x: 'y'+x.group(0)[1:], text)
159
- text = re.sub('u[aoəe]', lambda x: 'w'+x.group(0)[1:], text)
160
- text = re.sub('([ʦsɹ]`[⁼ʰ]?)([→↓↑ ]+|$)', lambda x: x.group(1) +
161
- 'ɹ`'+x.group(2), text).replace('ɻ', 'ɹ`')
162
- text = re.sub('([ʦs][⁼ʰ]?)([→↓↑ ]+|$)',
163
- lambda x: x.group(1)+'ɹ'+x.group(2), text)
164
  return text
165
 
166
 
@@ -169,3 +302,28 @@ def chinese_to_lazy_ipa(text):
169
  for regex, replacement in _romaji_to_ipa:
170
  text = re.sub(regex, replacement, text)
171
  return text
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
100
  ('—', '-')
101
  ]]
102
 
 
103
  # List of (romaji, ipa) pairs:
104
  _romaji_to_ipa = [(re.compile('%s' % x[0], re.IGNORECASE), x[1]) for x in [
105
  ('ʃy', 'ʃ'),
 
111
  ('h', 'x')
112
  ]]
113
 
114
+ # List of (bopomofo, ipa) pairs:
115
+ _bopomofo_to_ipa = [(re.compile('%s' % x[0]), x[1]) for x in [
116
+ ('ㄅㄛ', 'p⁼wo'),
117
+ ('ㄆㄛ', 'pʰwo'),
118
+ ('ㄇㄛ', 'mwo'),
119
+ ('ㄈㄛ', 'fwo'),
120
+ ('ㄅ', 'p⁼'),
121
+ ('ㄆ', 'pʰ'),
122
+ ('ㄇ', 'm'),
123
+ ('ㄈ', 'f'),
124
+ ('ㄉ', 't⁼'),
125
+ ('ㄊ', 'tʰ'),
126
+ ('ㄋ', 'n'),
127
+ ('ㄌ', 'l'),
128
+ ('ㄍ', 'k⁼'),
129
+ ('ㄎ', 'kʰ'),
130
+ ('ㄏ', 'x'),
131
+ ('ㄐ', 'tʃ⁼'),
132
+ ('ㄑ', 'tʃʰ'),
133
+ ('ㄒ', 'ʃ'),
134
+ ('ㄓ', 'ts`⁼'),
135
+ ('ㄔ', 'ts`ʰ'),
136
+ ('ㄕ', 's`'),
137
+ ('ㄖ', 'ɹ`'),
138
+ ('ㄗ', 'ts⁼'),
139
+ ('ㄘ', 'tsʰ'),
140
+ ('ㄙ', 's'),
141
+ ('ㄚ', 'a'),
142
+ ('ㄛ', 'o'),
143
+ ('ㄜ', 'ə'),
144
+ ('ㄝ', 'ɛ'),
145
+ ('ㄞ', 'aɪ'),
146
+ ('ㄟ', 'eɪ'),
147
+ ('ㄠ', 'ɑʊ'),
148
+ ('ㄡ', 'oʊ'),
149
+ ('ㄧㄢ', 'jɛn'),
150
+ ('ㄩㄢ', 'ɥæn'),
151
+ ('ㄢ', 'an'),
152
+ ('ㄧㄣ', 'in'),
153
+ ('ㄩㄣ', 'ɥn'),
154
+ ('ㄣ', 'ən'),
155
+ ('ㄤ', 'ɑŋ'),
156
+ ('ㄧㄥ', 'iŋ'),
157
+ ('ㄨㄥ', 'ʊŋ'),
158
+ ('ㄩㄥ', 'jʊŋ'),
159
+ ('ㄥ', 'əŋ'),
160
+ ('ㄦ', 'əɻ'),
161
+ ('ㄧ', 'i'),
162
+ ('ㄨ', 'u'),
163
+ ('ㄩ', 'ɥ'),
164
+ ('ˉ', '→'),
165
+ ('ˊ', '↑'),
166
+ ('ˇ', '↓↑'),
167
+ ('ˋ', '↓'),
168
+ ('˙', ''),
169
+ (',', ','),
170
+ ('。', '.'),
171
+ ('!', '!'),
172
+ ('?', '?'),
173
+ ('—', '-')
174
+ ]]
175
+
176
+ # List of (bopomofo, ipa2) pairs:
177
+ _bopomofo_to_ipa2 = [(re.compile('%s' % x[0]), x[1]) for x in [
178
+ ('ㄅㄛ', 'pwo'),
179
+ ('ㄆㄛ', 'pʰwo'),
180
+ ('ㄇㄛ', 'mwo'),
181
+ ('ㄈㄛ', 'fwo'),
182
+ ('ㄅ', 'p'),
183
+ ('ㄆ', 'pʰ'),
184
+ ('ㄇ', 'm'),
185
+ ('ㄈ', 'f'),
186
+ ('ㄉ', 't'),
187
+ ('ㄊ', 'tʰ'),
188
+ ('ㄋ', 'n'),
189
+ ('ㄌ', 'l'),
190
+ ('ㄍ', 'k'),
191
+ ('ㄎ', 'kʰ'),
192
+ ('ㄏ', 'h'),
193
+ ('ㄐ', 'tɕ'),
194
+ ('ㄑ', 'tɕʰ'),
195
+ ('ㄒ', 'ɕ'),
196
+ ('ㄓ', 'tʂ'),
197
+ ('ㄔ', 'tʂʰ'),
198
+ ('ㄕ', 'ʂ'),
199
+ ('ㄖ', 'ɻ'),
200
+ ('ㄗ', 'ts'),
201
+ ('ㄘ', 'tsʰ'),
202
+ ('ㄙ', 's'),
203
+ ('ㄚ', 'a'),
204
+ ('ㄛ', 'o'),
205
+ ('ㄜ', 'ɤ'),
206
+ ('ㄝ', 'ɛ'),
207
+ ('ㄞ', 'aɪ'),
208
+ ('ㄟ', 'eɪ'),
209
+ ('ㄠ', 'ɑʊ'),
210
+ ('ㄡ', 'oʊ'),
211
+ ('ㄧㄢ', 'jɛn'),
212
+ ('ㄩㄢ', 'yæn'),
213
+ ('ㄢ', 'an'),
214
+ ('ㄧㄣ', 'in'),
215
+ ('ㄩㄣ', 'yn'),
216
+ ('ㄣ', 'ən'),
217
+ ('ㄤ', 'ɑŋ'),
218
+ ('ㄧㄥ', 'iŋ'),
219
+ ('ㄨㄥ', 'ʊŋ'),
220
+ ('ㄩㄥ', 'jʊŋ'),
221
+ ('ㄥ', 'ɤŋ'),
222
+ ('ㄦ', 'əɻ'),
223
+ ('ㄧ', 'i'),
224
+ ('ㄨ', 'u'),
225
+ ('ㄩ', 'y'),
226
+ ('ˉ', '˥'),
227
+ ('ˊ', '˧˥'),
228
+ ('ˇ', '˨˩˦'),
229
+ ('ˋ', '˥˩'),
230
+ ('˙', ''),
231
+ (',', ','),
232
+ ('。', '.'),
233
+ ('!', '!'),
234
+ ('?', '?'),
235
+ ('—', '-')
236
+ ]]
237
+
238
 
239
  def number_to_chinese(text):
240
  numbers = re.findall(r'\d+(?:\.?\d+)?', text)
 
253
  text += word
254
  continue
255
  for i in range(len(bopomofos)):
256
+ bopomofos[i] = re.sub(r'([\u3105-\u3129])$', r'\1ˉ', bopomofos[i])
 
257
  if text != '':
258
  text += ' '
259
  text += ''.join(bopomofos)
 
272
  return text
273
 
274
 
275
+ def bopomofo_to_ipa(text):
276
+ for regex, replacement in _bopomofo_to_ipa:
277
+ text = re.sub(regex, replacement, text)
278
+ return text
279
+
280
+
281
+ def bopomofo_to_ipa2(text):
282
+ for regex, replacement in _bopomofo_to_ipa2:
283
+ text = re.sub(regex, replacement, text)
284
+ return text
285
+
286
+
287
  def chinese_to_romaji(text):
288
  text = number_to_chinese(text)
289
  text = chinese_to_bopomofo(text)
290
  text = latin_to_bopomofo(text)
291
  text = bopomofo_to_romaji(text)
292
+ text = re.sub('i([aoe])', r'y\1', text)
293
+ text = re.sub('u([aoəe])', r'w\1', text)
294
+ text = re.sub('([ʦsɹ]`[⁼ʰ]?)([→↓↑ ]+|$)',
295
+ r'\1ɹ`\2', text).replace('ɻ', 'ɹ`')
296
+ text = re.sub('([ʦs][⁼ʰ]?)([→↓↑ ]+|$)', r'\1ɹ\2', text)
 
297
  return text
298
 
299
 
 
302
  for regex, replacement in _romaji_to_ipa:
303
  text = re.sub(regex, replacement, text)
304
  return text
305
+
306
+
307
+ def chinese_to_ipa(text):
308
+ text = number_to_chinese(text)
309
+ text = chinese_to_bopomofo(text)
310
+ text = latin_to_bopomofo(text)
311
+ text = bopomofo_to_ipa(text)
312
+ text = re.sub('i([aoe])', r'j\1', text)
313
+ text = re.sub('u([aoəe])', r'w\1', text)
314
+ text = re.sub('([sɹ]`[⁼ʰ]?)([→↓↑ ]+|$)',
315
+ r'\1ɹ`\2', text).replace('ɻ', 'ɹ`')
316
+ text = re.sub('([s][⁼ʰ]?)([→↓↑ ]+|$)', r'\1ɹ\2', text)
317
+ return text
318
+
319
+
320
+ def chinese_to_ipa2(text):
321
+ text = number_to_chinese(text)
322
+ text = chinese_to_bopomofo(text)
323
+ text = latin_to_bopomofo(text)
324
+ text = bopomofo_to_ipa2(text)
325
+ text = re.sub(r'i([aoe])', r'j\1', text)
326
+ text = re.sub(r'u([aoəe])', r'w\1', text)
327
+ text = re.sub(r'([ʂɹ]ʰ?)([˩˨˧˦˥ ]+|$)', r'\1ʅ\2', text)
328
+ text = re.sub(r'(sʰ?)([˩˨˧˦˥ ]+|$)', r'\1ɿ\2', text)
329
+ return text
text/ngu_dialect.py ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import re
2
+ import opencc
3
+
4
+
5
+ dialects = {'SZ': 'suzhou', 'WX': 'wuxi', 'CZ': 'changzhou', 'HZ': 'hangzhou',
6
+ 'SX': 'shaoxing', 'NB': 'ningbo', 'JJ': 'jingjiang', 'YX': 'yixing',
7
+ 'JD': 'jiading', 'ZR': 'zhenru', 'PH': 'pinghu', 'TX': 'tongxiang',
8
+ 'JS': 'jiashan', 'HN': 'xiashi', 'LP': 'linping', 'XS': 'xiaoshan',
9
+ 'FY': 'fuyang', 'RA': 'ruao', 'CX': 'cixi', 'SM': 'sanmen',
10
+ 'TT': 'tiantai', 'WZ': 'wenzhou', 'SC': 'suichang', 'YB': 'youbu'}
11
+
12
+ converters = {}
13
+
14
+ for dialect in dialects.values():
15
+ try:
16
+ converters[dialect] = opencc.OpenCC(dialect)
17
+ except:
18
+ pass
19
+
20
+
21
+ def ngu_dialect_to_ipa(text, dialect):
22
+ dialect = dialects[dialect]
23
+ text = converters[dialect].convert(text).replace('-','').replace('$',' ')
24
+ text = re.sub(r'[、;:]', ',', text)
25
+ text = re.sub(r'\s*,\s*', ', ', text)
26
+ text = re.sub(r'\s*。\s*', '. ', text)
27
+ text = re.sub(r'\s*?\s*', '? ', text)
28
+ text = re.sub(r'\s*!\s*', '! ', text)
29
+ text = re.sub(r'\s*$', '', text)
30
+ return text
text/shanghainese.py ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import re
2
+ import cn2an
3
+ import opencc
4
+
5
+
6
+ converter = opencc.OpenCC('zaonhe')
7
+
8
+ # List of (Latin alphabet, ipa) pairs:
9
+ _latin_to_ipa = [(re.compile('%s' % x[0]), x[1]) for x in [
10
+ ('A', 'ᴇ'),
11
+ ('B', 'bi'),
12
+ ('C', 'si'),
13
+ ('D', 'di'),
14
+ ('E', 'i'),
15
+ ('F', 'ᴇf'),
16
+ ('G', 'dʑi'),
17
+ ('H', 'ᴇtɕʰ'),
18
+ ('I', 'ᴀi'),
19
+ ('J', 'dʑᴇ'),
20
+ ('K', 'kʰᴇ'),
21
+ ('L', 'ᴇl'),
22
+ ('M', 'ᴇm'),
23
+ ('N', 'ᴇn'),
24
+ ('O', 'o'),
25
+ ('P', 'pʰi'),
26
+ ('Q', 'kʰiu'),
27
+ ('R', 'ᴀl'),
28
+ ('S', 'ᴇs'),
29
+ ('T', 'tʰi'),
30
+ ('U', 'ɦiu'),
31
+ ('V', 'vi'),
32
+ ('W', 'dᴀbɤliu'),
33
+ ('X', 'ᴇks'),
34
+ ('Y', 'uᴀi'),
35
+ ('Z', 'zᴇ')
36
+ ]]
37
+
38
+
39
+ def _number_to_shanghainese(num):
40
+ num = cn2an.an2cn(num).replace('一十','十').replace('二十', '廿').replace('二', '两')
41
+ return re.sub(r'((?:^|[^三四五六七八九])十|廿)两', r'\1二', num)
42
+
43
+
44
+ def number_to_shanghainese(text):
45
+ return re.sub(r'\d+(?:\.?\d+)?', lambda x: _number_to_shanghainese(x.group()), text)
46
+
47
+
48
+ def latin_to_ipa(text):
49
+ for regex, replacement in _latin_to_ipa:
50
+ text = re.sub(regex, replacement, text)
51
+ return text
52
+
53
+
54
+ def shanghainese_to_ipa(text):
55
+ text = number_to_shanghainese(text.upper())
56
+ text = converter.convert(text).replace('-','').replace('$',' ')
57
+ text = re.sub(r'[A-Z]', lambda x: latin_to_ipa(x.group())+' ', text)
58
+ text = re.sub(r'[、;:]', ',', text)
59
+ text = re.sub(r'\s*,\s*', ', ', text)
60
+ text = re.sub(r'\s*。\s*', '. ', text)
61
+ text = re.sub(r'\s*?\s*', '? ', text)
62
+ text = re.sub(r'\s*!\s*', '! ', text)
63
+ text = re.sub(r'\s*$', '', text)
64
+ return text
text/thai.py ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import re
2
+ from num_thai.thainumbers import NumThai
3
+
4
+
5
+ num = NumThai()
6
+
7
+ # List of (Latin alphabet, Thai) pairs:
8
+ _latin_to_thai = [(re.compile('%s' % x[0], re.IGNORECASE), x[1]) for x in [
9
+ ('a', 'เอ'),
10
+ ('b','บี'),
11
+ ('c','ซี'),
12
+ ('d','ดี'),
13
+ ('e','อี'),
14
+ ('f','เอฟ'),
15
+ ('g','จี'),
16
+ ('h','เอช'),
17
+ ('i','ไอ'),
18
+ ('j','เจ'),
19
+ ('k','เค'),
20
+ ('l','แอล'),
21
+ ('m','เอ็ม'),
22
+ ('n','เอ็น'),
23
+ ('o','โอ'),
24
+ ('p','พี'),
25
+ ('q','คิว'),
26
+ ('r','แอร์'),
27
+ ('s','เอส'),
28
+ ('t','ที'),
29
+ ('u','ยู'),
30
+ ('v','วี'),
31
+ ('w','ดับเบิลยู'),
32
+ ('x','เอ็กซ์'),
33
+ ('y','วาย'),
34
+ ('z','ซี')
35
+ ]]
36
+
37
+
38
+ def num_to_thai(text):
39
+ return re.sub(r'(?:\d+(?:,?\d+)?)+(?:\.\d+(?:,?\d+)?)?', lambda x: ''.join(num.NumberToTextThai(float(x.group(0).replace(',', '')))), text)
40
+
41
+ def latin_to_thai(text):
42
+ for regex, replacement in _latin_to_thai:
43
+ text = re.sub(regex, replacement, text)
44
+ return text