update default prompt, fix number transcription bug

.gitignore

@@ -24,4 +24,5 @@ thumbs.db
 src/audiocraft
 
 !/demo/
-!/demo/*
+!/demo/*
+/demo/temp

app.py

@@ -74,6 +74,8 @@ class WhisperxModel:
 
     def transcribe(self, audio_path):
         segments = self.model.transcribe(audio_path, batch_size=8)["segments"]
+        for segment in segments:
+            segment['text'] = replace_numbers_with_words(segment['text'])
         return self.align_model.align(segments, audio_path)
 
 @spaces.GPU(duration=120)
@@ -176,7 +178,7 @@ def align(seed, transcript, audio_path):
     if align_model is None:
         raise gr.Error("Align model not loaded")
     seed_everything(seed)
-
+    transcript = replace_numbers_with_words(transcript).replace("  ", " ").replace("  ", " ") # replace numbers with words, so that the phonemizer can do a better job
     fragments = align_segments(transcript, audio_path)
     segments = [{
         "start": float(fragment["begin"]),
@@ -185,7 +187,6 @@ def align(seed, transcript, audio_path):
     } for fragment in fragments["fragments"]]
     segments = align_model.align(segments, audio_path)
     state = get_transcribe_state(segments)
-
     return [
         state["transcript_with_start_time"], state["transcript_with_end_time"],
         gr.Dropdown(value=state["word_bounds"][-1], choices=state["word_bounds"], interactive=True), # prompt_to_word
@@ -384,32 +385,32 @@ If disabled, you should write the target transcript yourself:</br>
  - In Edit mode write full prompt</br>
 """
 
-demo_original_transcript = "Gwynplaine had, besides, for his work and for his feats of strength, round his neck and over his shoulders, an esclavine of leather."
+demo_original_transcript = "And again in two thousand and eight when the United States Central Bank, the Federal Reserve, printed over two trillion dollars."
 
 demo_text = {
     "TTS": {
         "smart": "I cannot believe that the same model can also do text to speech synthesis too!",
-        "regular": "Gwynplaine had, besides, for his work and for his feats of strength, I cannot believe that the same model can also do text to speech synthesis too!"
+        "regular": "And again in two thousand and eight when the United States Central Bank, I cannot believe that the same model can also do text to speech synthesis too!"
     },
     "Edit": {
         "smart": "take over the stage for half an hour,",
-        "regular": "Gwynplaine had, besides, for his work and for his feats of strength, take over the stage for half an hour, an esclavine of leather."
+        "regular": "And again in two thousand and eight when the United States Central Bank, take over the stage for half an hour, printed over two trillion dollars."
     },
     "Long TTS": {
         "smart": "You can run the model on a big text!\n"
-                 "Just write it line-by-line. Or sentence-by-sentence.\n"
+                 "Just write it line by line. Or sentence by sentence.\n"
                  "If some sentences sound odd, just rerun the model on them, no need to generate the whole text again!",
-        "regular": "Gwynplaine had, besides, for his work and for his feats of strength, You can run the model on a big text!\n"
-                   "Gwynplaine had, besides, for his work and for his feats of strength, Just write it line-by-line. Or sentence-by-sentence.\n"
-                   "Gwynplaine had, besides, for his work and for his feats of strength, If some sentences sound odd, just rerun the model on them, no need to generate the whole text again!"
+        "regular": "And again in two thousand and eight when the United States Central Bank, You can run the model on a big text!\n"
+                   "And again in two thousand and eight when the United States Central Bank, Just write it line by line. Or sentence by sentence.\n"
+                   "And again in two thousand and eight when the United States Central Bank, If some sentences sound odd, just rerun the model on them, no need to generate the whole text again!"
     }
 }
 
 all_demo_texts = {vv for k, v in demo_text.items() for kk, vv in v.items()}
 
-demo_words = ['0.069 Gwynplain 0.611', '0.671 had, 0.912', '0.952 besides, 1.414', '1.494 for 1.634', '1.695 his 1.835', '1.915 work 2.136', '2.196 and 2.297', '2.337 for 2.517', '2.557 his 2.678', '2.758 feats 3.019', '3.079 of 3.139', '3.2 strength, 3.561', '4.022 round 4.263', '4.303 his 4.444', '4.524 neck 4.705', '4.745 and 4.825', '4.905 over 5.086', '5.146 his 5.266', '5.307 shoulders, 5.768', '6.23 an 6.33', '6.531 esclavine 7.133', '7.213 of 7.293', '7.353 leather. 7.614']
+demo_words = ['0.12 And 0.221', '0.261 again 0.561', '0.622 in 0.682', '0.742 two 0.922', '0.983 thousand 1.464', '1.504 and 1.584', '1.684 eight 1.865', '1.945 when 2.085', '2.125 the 2.206', '2.266 United 2.667', '2.707 States 2.968', '3.008 Central 3.349', '3.389 Bank, 3.649', '3.83 the 3.93', '4.01 Federal 4.451', '4.532 Reserve 5.113', '5.314 printed 5.674', '5.835 over 6.035', '6.176 two 6.517', '6.637 trillion 7.098', '7.118 dollars. 7.479']
 
-demo_words_info = [{'word': 'Gwynplain', 'start': 0.069, 'end': 0.611, 'score': 0.833}, {'word': 'had,', 'start': 0.671, 'end': 0.912, 'score': 0.879}, {'word': 'besides,', 'start': 0.952, 'end': 1.414, 'score': 0.863}, {'word': 'for', 'start': 1.494, 'end': 1.634, 'score': 0.89}, {'word': 'his', 'start': 1.695, 'end': 1.835, 'score': 0.669}, {'word': 'work', 'start': 1.915, 'end': 2.136, 'score': 0.916}, {'word': 'and', 'start': 2.196, 'end': 2.297, 'score': 0.766}, {'word': 'for', 'start': 2.337, 'end': 2.517, 'score': 0.808}, {'word': 'his', 'start': 2.557, 'end': 2.678, 'score': 0.786}, {'word': 'feats', 'start': 2.758, 'end': 3.019, 'score': 0.97}, {'word': 'of', 'start': 3.079, 'end': 3.139, 'score': 0.752}, {'word': 'strength,', 'start': 3.2, 'end': 3.561, 'score': 0.742}, {'word': 'round', 'start': 4.022, 'end': 4.263, 'score': 0.916}, {'word': 'his', 'start': 4.303, 'end': 4.444, 'score': 0.666}, {'word': 'neck', 'start': 4.524, 'end': 4.705, 'score': 0.908}, {'word': 'and', 'start': 4.745, 'end': 4.825, 'score': 0.882}, {'word': 'over', 'start': 4.905, 'end': 5.086, 'score': 0.847}, {'word': 'his', 'start': 5.146, 'end': 5.266, 'score': 0.791}, {'word': 'shoulders,', 'start': 5.307, 'end': 5.768, 'score': 0.729}, {'word': 'an', 'start': 6.23, 'end': 6.33, 'score': 0.854}, {'word': 'esclavine', 'start': 6.531, 'end': 7.133, 'score': 0.803}, {'word': 'of', 'start': 7.213, 'end': 7.293, 'score': 0.772}, {'word': 'leather.', 'start': 7.353, 'end': 7.614, 'score': 0.896}]
+demo_words_info = [{'word': 'And', 'start': 0.12, 'end': 0.221, 'score': 0.792}, {'word': 'again', 'start': 0.261, 'end': 0.561, 'score': 0.795}, {'word': 'in', 'start': 0.622, 'end': 0.682, 'score': 0.75}, {'word': 'two', 'start': 0.742, 'end': 0.922, 'score': 0.755}, {'word': 'thousand', 'start': 0.983, 'end': 1.464, 'score': 0.82}, {'word': 'and', 'start': 1.504, 'end': 1.584, 'score': 0.715}, {'word': 'eight', 'start': 1.684, 'end': 1.865, 'score': 0.885}, {'word': 'when', 'start': 1.945, 'end': 2.085, 'score': 0.987}, {'word': 'the', 'start': 2.125, 'end': 2.206, 'score': 0.833}, {'word': 'United', 'start': 2.266, 'end': 2.667, 'score': 0.818}, {'word': 'States', 'start': 2.707, 'end': 2.968, 'score': 0.842}, {'word': 'Central', 'start': 3.008, 'end': 3.349, 'score': 0.852}, {'word': 'Bank,', 'start': 3.389, 'end': 3.649, 'score': 0.98}, {'word': 'the', 'start': 3.83, 'end': 3.93, 'score': 0.996}, {'word': 'Federal', 'start': 4.01, 'end': 4.451, 'score': 0.795}, {'word': 'Reserve', 'start': 4.532, 'end': 5.113, 'score': 0.852}, {'word': 'printed', 'start': 5.314, 'end': 5.674, 'score': 0.785}, {'word': 'over', 'start': 5.835, 'end': 6.035, 'score': 0.84}, {'word': 'two', 'start': 6.176, 'end': 6.517, 'score': 0.757}, {'word': 'trillion', 'start': 6.637, 'end': 7.098, 'score': 0.796}, {'word': 'dollars.', 'start': 7.118, 'end': 7.479, 'score': 0.939}]
 
 
 def update_demo(mode, smart_transcript, edit_word_mode, transcript, edit_from_word, edit_to_word):
@@ -445,7 +446,7 @@ def get_app():
 
         with gr.Row():
             with gr.Column(scale=2):
-                input_audio = gr.Audio(value=f"{DEMO_PATH}/5895_34622_000026_000002.wav", label="Input Audio", type="filepath", interactive=True)
+                input_audio = gr.Audio(value=f"{DEMO_PATH}/YOU1000000115_S0000252.wav", label="Input Audio", type="filepath", interactive=True)
                 with gr.Group():
                     original_transcript = gr.Textbox(label="Original transcript", lines=5, value=demo_original_transcript,
                                                     info="Use whisperx model to get the transcript. Fix and align it if necessary.")
@@ -473,16 +474,16 @@ def get_app():
                                                 info="What to do with first and last word", visible=False)
 
                     with gr.Group() as tts_mode_controls:
-                        prompt_to_word = gr.Dropdown(label="Last word in prompt", choices=demo_words, value=demo_words[11], interactive=True)
-                        prompt_end_time = gr.Slider(label="Prompt end time", minimum=0, maximum=7.614, step=0.001, value=3.600)
+                        prompt_to_word = gr.Dropdown(label="Last word in prompt", choices=demo_words, value=demo_words[12], interactive=True)
+                        prompt_end_time = gr.Slider(label="Prompt end time", minimum=0, maximum=7.86, step=0.001, value=3.675)
 
                     with gr.Group(visible=False) as edit_mode_controls:
                         with gr.Row():
-                            edit_from_word = gr.Dropdown(label="First word to edit", choices=demo_words, value=demo_words[12], interactive=True)
-                            edit_to_word = gr.Dropdown(label="Last word to edit", choices=demo_words, value=demo_words[18], interactive=True)
+                            edit_from_word = gr.Dropdown(label="First word to edit", choices=demo_words, value=demo_words[13], interactive=True)
+                            edit_to_word = gr.Dropdown(label="Last word to edit", choices=demo_words, value=demo_words[15], interactive=True)
                         with gr.Row():
-                            edit_start_time = gr.Slider(label="Edit from time", minimum=0, maximum=7.614, step=0.001, value=4.022)
-                            edit_end_time = gr.Slider(label="Edit to time", minimum=0, maximum=7.614, step=0.001, value=5.768)
+                            edit_start_time = gr.Slider(label="Edit from time", minimum=0, maximum=7.86, step=0.001, value=3.83)
+                            edit_end_time = gr.Slider(label="Edit to time", minimum=0, maximum=7.86, step=0.001, value=5.113)
 
                     run_btn = gr.Button(value="Run")
 
@@ -500,11 +501,11 @@ def get_app():
         with gr.Row():
             with gr.Accordion("Generation Parameters - change these if you are unhappy with the generation", open=False):
                 stop_repetition = gr.Radio(label="stop_repetition", choices=[-1, 1, 2, 3, 4], value=3,
-                                        info="if there are long silence in the generated audio, reduce the stop_repetition to 2 or 1. -1 = disabled")
-                sample_batch_size = gr.Number(label="speech rate", value=3, precision=0,
+                                        info="if there are long silence in the generated audio, reduce the stop_repetition to 1 or 2. -1 = disabled")
+                sample_batch_size = gr.Number(label="speech rate", value=2, precision=0,
                                             info="The higher the number, the faster the output will be. "
                                                 "Under the hood, the model will generate this many samples and choose the shortest one. "
-                                                "For giga330M_TTSEnhanced, 1 or 2 should be fine since the model is trained to do TTS.")
+                                                "For TTSEnhanced models, 1~3 should be fine since the model is trained to do TTS.")
                 seed = gr.Number(label="seed", value=-1, precision=0, info="random seeds always works :)")
                 kvcache = gr.Radio(label="kvcache", choices=[0, 1], value=1,
                                     info="set to 0 to use less VRAM, but with slower inference")

demo/temp/84_121550_000074_000000.txt

@@ -1 +0,0 @@
-But when I had approached so near to them The common object, which the sense deceives, Lost not by distance any of its marks,

demo/temp/mfa_alignments/84_121550_000074_000000.csv

@@ -1,109 +0,0 @@
-Begin,End,Label,Type,Speaker
-0.03,0.18,but,words,temp
-0.18,0.32,when,words,temp
-0.32,0.48,i,words,temp
-0.48,0.64,had,words,temp
-0.64,1.19,approached,words,temp
-1.22,1.58,so,words,temp
-1.58,1.91,near,words,temp
-1.91,2.07,to,words,temp
-2.07,2.42,them,words,temp
-2.53,2.61,the,words,temp
-2.61,3.01,common,words,temp
-3.05,3.62,object,words,temp
-3.68,3.93,which,words,temp
-3.93,4.02,the,words,temp
-4.02,4.34,sense,words,temp
-4.34,4.97,deceives,words,temp
-5.04,5.54,lost,words,temp
-5.54,6.0,not,words,temp
-6.0,6.14,by,words,temp
-6.14,6.67,distance,words,temp
-6.79,7.05,any,words,temp
-7.05,7.18,of,words,temp
-7.18,7.34,its,words,temp
-7.34,7.87,marks,words,temp
-0.03,0.06,B,phones,temp
-0.06,0.09,AH1,phones,temp
-0.09,0.18,T,phones,temp
-0.18,0.23,W,phones,temp
-0.23,0.27,EH1,phones,temp
-0.27,0.32,N,phones,temp
-0.32,0.48,AY1,phones,temp
-0.48,0.49,HH,phones,temp
-0.49,0.6,AE1,phones,temp
-0.6,0.64,D,phones,temp
-0.64,0.7,AH0,phones,temp
-0.7,0.83,P,phones,temp
-0.83,0.88,R,phones,temp
-0.88,0.99,OW1,phones,temp
-0.99,1.12,CH,phones,temp
-1.12,1.19,T,phones,temp
-1.22,1.4,S,phones,temp
-1.4,1.58,OW1,phones,temp
-1.58,1.7,N,phones,temp
-1.7,1.84,IH1,phones,temp
-1.84,1.91,R,phones,temp
-1.91,2.01,T,phones,temp
-2.01,2.07,AH0,phones,temp
-2.07,2.13,DH,phones,temp
-2.13,2.3,EH1,phones,temp
-2.3,2.42,M,phones,temp
-2.53,2.55,DH,phones,temp
-2.55,2.61,AH0,phones,temp
-2.61,2.73,K,phones,temp
-2.73,2.85,AA1,phones,temp
-2.85,2.9,M,phones,temp
-2.9,2.95,AH0,phones,temp
-2.95,3.01,N,phones,temp
-3.05,3.22,AA1,phones,temp
-3.22,3.27,B,phones,temp
-3.27,3.34,JH,phones,temp
-3.34,3.48,EH0,phones,temp
-3.48,3.54,K,phones,temp
-3.54,3.62,T,phones,temp
-3.68,3.69,HH,phones,temp
-3.69,3.76,W,phones,temp
-3.76,3.8,IH1,phones,temp
-3.8,3.93,CH,phones,temp
-3.93,3.95,DH,phones,temp
-3.95,4.02,AH0,phones,temp
-4.02,4.12,S,phones,temp
-4.12,4.21,EH1,phones,temp
-4.21,4.27,N,phones,temp
-4.27,4.34,S,phones,temp
-4.34,4.42,D,phones,temp
-4.42,4.45,IH0,phones,temp
-4.45,4.59,S,phones,temp
-4.59,4.79,IY1,phones,temp
-4.79,4.87,V,phones,temp
-4.87,4.97,Z,phones,temp
-5.04,5.12,L,phones,temp
-5.12,5.33,AO1,phones,temp
-5.33,5.42,S,phones,temp
-5.42,5.54,T,phones,temp
-5.54,5.7,N,phones,temp
-5.7,5.89,AA1,phones,temp
-5.89,6.0,T,phones,temp
-6.0,6.05,B,phones,temp
-6.05,6.14,AY1,phones,temp
-6.14,6.24,D,phones,temp
-6.24,6.3,IH1,phones,temp
-6.3,6.38,S,phones,temp
-6.38,6.45,T,phones,temp
-6.45,6.51,AH0,phones,temp
-6.51,6.57,N,phones,temp
-6.57,6.67,S,phones,temp
-6.79,6.89,EH1,phones,temp
-6.89,6.95,N,phones,temp
-6.95,7.05,IY0,phones,temp
-7.05,7.13,AH0,phones,temp
-7.13,7.18,V,phones,temp
-7.18,7.22,IH0,phones,temp
-7.22,7.29,T,phones,temp
-7.29,7.34,S,phones,temp
-7.34,7.39,M,phones,temp
-7.39,7.5,AA1,phones,temp
-7.5,7.58,R,phones,temp
-7.58,7.7,K,phones,temp
-7.7,7.87,S,phones,temp