seamless_m4t

Runtime error

App Files Files Community

reach-vb HF staff

hysts HF staff commited on Aug 22, 2023

Commit

df28e5a

•

1 Parent(s): 4a6c567

Update language list (#1)

Browse files

- Update language list (c105fbb6baa8436dc0a9c4521d9b9d137d404809)
- Fix the language list (a71e647e9f4a4c7d4611c3728c7fd7fb0b94768f)

Co-authored-by: hysts <[email protected]>

Files changed (3) hide show

app.py +48 -16
lang_list.py +254 -0
mlg_config.json +0 -186

app.py CHANGED Viewed

@@ -1,4 +1,3 @@
-import json
 import os
 import gradio as gr
@@ -7,11 +6,15 @@ import torch
 import torchaudio
 from seamless_communication.models.inference.translator import Translator
-DESCRIPTION = "# SeamlessM4T"
-with open("./mlg_config.json", "r") as f:
-    lang_idx_map = json.loads(f.read())
-LANGUAGES = lang_idx_map["multilingual"].keys()
 TASK_NAMES = [
     "S2ST (Speech to Speech translation)",
@@ -24,6 +27,8 @@ TASK_NAMES = [
 AUDIO_SAMPLE_RATE = 16000.0
 MAX_INPUT_AUDIO_LENGTH = 60  # in seconds
 device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 translator = Translator(
     model_name_or_card="multitask_unity_large",
@@ -43,6 +48,9 @@ def predict(
     target_language: str,
 ) -> tuple[tuple[int, np.ndarray] | None, str]:
     task_name = task_name.split()[0]
     if task_name in ["S2ST", "S2TT", "ASR"]:
         if audio_source == "microphone":
             input_data = input_audio_mic
@@ -61,8 +69,8 @@ def predict(
     text_out, wav, sr = translator.predict(
         input=input_data,
         task_str=task_name,
-        tgt_lang=target_language,
-        src_lang=source_language,
     )
     if task_name in ["S2ST", "T2ST"]:
         return (sr, wav.cpu().detach().numpy()), text_out
@@ -80,26 +88,50 @@ def update_audio_ui(audio_source: str) -> tuple[dict, dict]:
 def update_input_ui(task_name: str) -> tuple[dict, dict, dict, dict]:
     task_name = task_name.split()[0]
-    if task_name in ["S2ST", "S2TT"]:
         return (
             gr.update(visible=True),  # audio_box
             gr.update(visible=False),  # input_text
             gr.update(visible=False),  # source_language
-            gr.update(visible=True),  # target_language
         )
-    elif task_name in ["T2ST", "T2TT"]:
         return (
             gr.update(visible=False),  # audio_box
             gr.update(visible=True),  # input_text
             gr.update(visible=True),  # source_language
-            gr.update(visible=True),  # target_language
         )
     elif task_name == "ASR":
         return (
             gr.update(visible=True),  # audio_box
             gr.update(visible=False),  # input_text
             gr.update(visible=False),  # source_language
-            gr.update(visible=True),  # target_language
         )
     else:
         raise ValueError(f"Unknown task: {task_name}")
@@ -137,14 +169,14 @@ with gr.Blocks(css="style.css") as demo:
         with gr.Row():
             source_language = gr.Dropdown(
                 label="Source language",
-                choices=LANGUAGES,
-                value="eng",
                 visible=False,
             )
             target_language = gr.Dropdown(
                 label="Target language",
-                choices=LANGUAGES,
-                value="fra",
             )
         with gr.Row() as audio_box:
             audio_source = gr.Radio(

 import os
 import gradio as gr
 import torchaudio
 from seamless_communication.models.inference.translator import Translator
+from lang_list import (
+    LANGUAGE_NAME_TO_CODE,
+    S2ST_TARGET_LANGUAGE_NAMES,
+    S2TT_TARGET_LANGUAGE_NAMES,
+    T2TT_TARGET_LANGUAGE_NAMES,
+    TEXT_SOURCE_LANGUAGE_NAMES,
+)
+DESCRIPTION = "# SeamlessM4T"
 TASK_NAMES = [
     "S2ST (Speech to Speech translation)",
 AUDIO_SAMPLE_RATE = 16000.0
 MAX_INPUT_AUDIO_LENGTH = 60  # in seconds
+DEFAULT_TARGET_LANGUAGE = "French"
 device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 translator = Translator(
     model_name_or_card="multitask_unity_large",
     target_language: str,
 ) -> tuple[tuple[int, np.ndarray] | None, str]:
     task_name = task_name.split()[0]
+    source_language_code = LANGUAGE_NAME_TO_CODE[source_language]
+    target_language_code = LANGUAGE_NAME_TO_CODE[target_language]
     if task_name in ["S2ST", "S2TT", "ASR"]:
         if audio_source == "microphone":
             input_data = input_audio_mic
     text_out, wav, sr = translator.predict(
         input=input_data,
         task_str=task_name,
+        tgt_lang=target_language_code,
+        src_lang=source_language_code,
     )
     if task_name in ["S2ST", "T2ST"]:
         return (sr, wav.cpu().detach().numpy()), text_out
 def update_input_ui(task_name: str) -> tuple[dict, dict, dict, dict]:
     task_name = task_name.split()[0]
+    if task_name == "S2ST":
         return (
             gr.update(visible=True),  # audio_box
             gr.update(visible=False),  # input_text
             gr.update(visible=False),  # source_language
+            gr.update(
+                visible=True, choices=S2ST_TARGET_LANGUAGE_NAMES, value=DEFAULT_TARGET_LANGUAGE
+            ),  # target_language
+        )
+    elif task_name == "S2TT":
+        return (
+            gr.update(visible=True),  # audio_box
+            gr.update(visible=False),  # input_text
+            gr.update(visible=False),  # source_language
+            gr.update(
+                visible=True, choices=S2TT_TARGET_LANGUAGE_NAMES, value=DEFAULT_TARGET_LANGUAGE
+            ),  # target_language
+        )
+    elif task_name == "T2ST":
+        return (
+            gr.update(visible=False),  # audio_box
+            gr.update(visible=True),  # input_text
+            gr.update(visible=True),  # source_language
+            gr.update(
+                visible=True, choices=S2ST_TARGET_LANGUAGE_NAMES, value=DEFAULT_TARGET_LANGUAGE
+            ),  # target_language
         )
+    elif task_name == "T2TT":
         return (
             gr.update(visible=False),  # audio_box
             gr.update(visible=True),  # input_text
             gr.update(visible=True),  # source_language
+            gr.update(
+                visible=True, choices=T2TT_TARGET_LANGUAGE_NAMES, value=DEFAULT_TARGET_LANGUAGE
+            ),  # target_language
         )
     elif task_name == "ASR":
         return (
             gr.update(visible=True),  # audio_box
             gr.update(visible=False),  # input_text
             gr.update(visible=False),  # source_language
+            gr.update(
+                visible=True, choices=S2TT_TARGET_LANGUAGE_NAMES, value=DEFAULT_TARGET_LANGUAGE
+            ),  # target_language
         )
     else:
         raise ValueError(f"Unknown task: {task_name}")
         with gr.Row():
             source_language = gr.Dropdown(
                 label="Source language",
+                choices=TEXT_SOURCE_LANGUAGE_NAMES,
+                value="English",
                 visible=False,
             )
             target_language = gr.Dropdown(
                 label="Target language",
+                choices=S2ST_TARGET_LANGUAGE_NAMES,
+                value=DEFAULT_TARGET_LANGUAGE,
             )
         with gr.Row() as audio_box:
             audio_source = gr.Radio(

lang_list.py ADDED Viewed

	@@ -0,0 +1,254 @@

+# Language dict
+language_code_to_name = {
+    "afr": "Afrikaans",
+    "amh": "Amharic",
+    "arb": "Modern Standard Arabic",
+    "ary": "Moroccan Arabic",
+    "arz": "Egyptian Arabic",
+    "asm": "Assamese",
+    "ast": "Asturian",
+    "azj": "North Azerbaijani",
+    "bel": "Belarusian",
+    "ben": "Bengali",
+    "bos": "Bosnian",
+    "bul": "Bulgarian",
+    "cat": "Catalan",
+    "ceb": "Cebuano",
+    "ces": "Czech",
+    "ckb": "Central Kurdish",
+    "cmn": "Mandarin Chinese",
+    "cym": "Welsh",
+    "dan": "Danish",
+    "deu": "German",
+    "ell": "Greek",
+    "eng": "English",
+    "est": "Estonian",
+    "eus": "Basque",
+    "fin": "Finnish",
+    "fra": "French",
+    "gaz": "West Central Oromo",
+    "gle": "Irish",
+    "glg": "Galician",
+    "guj": "Gujarati",
+    "heb": "Hebrew",
+    "hin": "Hindi",
+    "hrv": "Croatian",
+    "hun": "Hungarian",
+    "hye": "Armenian",
+    "ibo": "Igbo",
+    "ind": "Indonesian",
+    "isl": "Icelandic",
+    "ita": "Italian",
+    "jav": "Javanese",
+    "jpn": "Japanese",
+    "kam": "Kamba",
+    "kan": "Kannada",
+    "kat": "Georgian",
+    "kaz": "Kazakh",
+    "kea": "Kabuverdianu",
+    "khk": "Halh Mongolian",
+    "khm": "Khmer",
+    "kir": "Kyrgyz",
+    "kor": "Korean",
+    "lao": "Lao",
+    "lit": "Lithuanian",
+    "ltz": "Luxembourgish",
+    "lug": "Ganda",
+    "luo": "Luo",
+    "lvs": "Standard Latvian",
+    "mai": "Maithili",
+    "mal": "Malayalam",
+    "mar": "Marathi",
+    "mkd": "Macedonian",
+    "mlt": "Maltese",
+    "mni": "Meitei",
+    "mya": "Burmese",
+    "nld": "Dutch",
+    "nno": "Norwegian Nynorsk",
+    "nob": "Norwegian Bokm\u00e5l",
+    "npi": "Nepali",
+    "nya": "Nyanja",
+    "oci": "Occitan",
+    "ory": "Odia",
+    "pan": "Punjabi",
+    "pbt": "Southern Pashto",
+    "pes": "Western Persian",
+    "pol": "Polish",
+    "por": "Portuguese",
+    "ron": "Romanian",
+    "rus": "Russian",
+    "slk": "Slovak",
+    "slv": "Slovenian",
+    "sna": "Shona",
+    "snd": "Sindhi",
+    "som": "Somali",
+    "spa": "Spanish",
+    "srp": "Serbian",
+    "swe": "Swedish",
+    "swh": "Swahili",
+    "tam": "Tamil",
+    "tel": "Telugu",
+    "tgk": "Tajik",
+    "tgl": "Tagalog",
+    "tha": "Thai",
+    "tur": "Turkish",
+    "ukr": "Ukrainian",
+    "urd": "Urdu",
+    "uzn": "Northern Uzbek",
+    "vie": "Vietnamese",
+    "xho": "Xhosa",
+    "yor": "Yoruba",
+    "yue": "Cantonese",
+    "zlm": "Colloquial Malay",
+    "zsm": "Standard Malay",
+    "zul": "Zulu",
+}
+LANGUAGE_NAME_TO_CODE = {v: k for k, v in language_code_to_name.items()}
+# Source langs: S2ST / S2TT / ASR don't need source lang
+# T2TT / T2ST use this
+text_source_language_codes = [
+    "afr",
+    "amh",
+    "arb",
+    "ary",
+    "arz",
+    "asm",
+    "azj",
+    "bel",
+    "ben",
+    "bos",
+    "bul",
+    "cat",
+    "ceb",
+    "ces",
+    "ckb",
+    "cmn",
+    "cym",
+    "dan",
+    "deu",
+    "ell",
+    "eng",
+    "est",
+    "eus",
+    "fin",
+    "fra",
+    "gaz",
+    "gle",
+    "glg",
+    "guj",
+    "heb",
+    "hin",
+    "hrv",
+    "hun",
+    "hye",
+    "ibo",
+    "ind",
+    "isl",
+    "ita",
+    "jav",
+    "jpn",
+    "kan",
+    "kat",
+    "kaz",
+    "khk",
+    "khm",
+    "kir",
+    "kor",
+    "lao",
+    "lit",
+    "lug",
+    "luo",
+    "lvs",
+    "mai",
+    "mal",
+    "mar",
+    "mkd",
+    "mlt",
+    "mni",
+    "mya",
+    "nld",
+    "nno",
+    "nob",
+    "npi",
+    "nya",
+    "ory",
+    "pan",
+    "pbt",
+    "pes",
+    "pol",
+    "por",
+    "ron",
+    "rus",
+    "slk",
+    "slv",
+    "sna",
+    "snd",
+    "som",
+    "spa",
+    "srp",
+    "swe",
+    "swh",
+    "tam",
+    "tel",
+    "tgk",
+    "tgl",
+    "tha",
+    "tur",
+    "ukr",
+    "urd",
+    "uzn",
+    "vie",
+    "yor",
+    "yue",
+    "zsm",
+    "zul",
+]
+TEXT_SOURCE_LANGUAGE_NAMES = sorted([language_code_to_name[code] for code in text_source_language_codes])
+# Target langs:
+# S2ST / T2ST
+s2st_target_language_codes = [
+    "eng",
+    "arb",
+    "ben",
+    "cat",
+    "ces",
+    "cmn",
+    "cym",
+    "dan",
+    "deu",
+    "est",
+    "fin",
+    "fra",
+    "hin",
+    "ind",
+    "ita",
+    "jpn",
+    "kor",
+    "mlt",
+    "nld",
+    "pes",
+    "pol",
+    "por",
+    "ron",
+    "rus",
+    "slk",
+    "spa",
+    "swe",
+    "swh",
+    "tel",
+    "tgl",
+    "tha",
+    "tur",
+    "ukr",
+    "urd",
+    "uzn",
+    "vie",
+]
+S2ST_TARGET_LANGUAGE_NAMES = sorted([language_code_to_name[code] for code in s2st_target_language_codes])
+# S2TT / ASR
+S2TT_TARGET_LANGUAGE_NAMES = TEXT_SOURCE_LANGUAGE_NAMES
+# T2TT
+T2TT_TARGET_LANGUAGE_NAMES = TEXT_SOURCE_LANGUAGE_NAMES

mlg_config.json DELETED Viewed

@@ -1,186 +0,0 @@
-{
-    "multilingual": {
-        "arb": 0,
-        "ben": 1,
-        "cat": 2,
-        "ces": 3,
-        "cmn": 4,
-        "cym": 5,
-        "dan": 6,
-        "deu": 7,
-        "eng": 8,
-        "est": 9,
-        "fin": 10,
-        "fra": 11,
-        "hin": 12,
-        "ind": 13,
-        "ita": 14,
-        "jpn": 15,
-        "kor": 16,
-        "mlt": 17,
-        "nld": 18,
-        "pes": 19,
-        "pol": 20,
-        "por": 21,
-        "ron": 22,
-        "rus": 23,
-        "slk": 24,
-        "spa": 25,
-        "swe": 26,
-        "swh": 27,
-        "tel": 28,
-        "tgl": 29,
-        "tha": 30,
-        "tur": 31,
-        "ukr": 32,
-        "urd": 33,
-        "uzn": 34,
-        "vie": 35
-    },
-    "multispkr": {
-        "arb": [
-            0
-        ],
-        "ben": [
-            2,
-            1
-        ],
-        "cat": [
-            3
-        ],
-        "ces": [
-            4
-        ],
-        "cmn": [
-            5
-        ],
-        "cym": [
-            6
-        ],
-        "dan": [
-            7,
-            8
-        ],
-        "deu": [
-            9
-        ],
-        "eng": [
-            10
-        ],
-        "est": [
-            11,
-            12,
-            13
-        ],
-        "fin": [
-            14
-        ],
-        "fra": [
-            15
-        ],
-        "hin": [
-            16
-        ],
-        "ind": [
-            17,
-            24,
-            18,
-            20,
-            19,
-            21,
-            23,
-            27,
-            26,
-            22,
-            25
-        ],
-        "ita": [
-            29,
-            28
-        ],
-        "jpn": [
-            30
-        ],
-        "kor": [
-            31
-        ],
-        "mlt": [
-            32,
-            33,
-            34
-        ],
-        "nld": [
-            35
-        ],
-        "pes": [
-            36
-        ],
-        "pol": [
-            37
-        ],
-        "por": [
-            38
-        ],
-        "ron": [
-            39
-        ],
-        "rus": [
-            40
-        ],
-        "slk": [
-            41
-        ],
-        "spa": [
-            42
-        ],
-        "swe": [
-            43,
-            45,
-            44
-        ],
-        "swh": [
-            46,
-            48,
-            47
-        ],
-        "tel": [
-            49
-        ],
-        "tgl": [
-            50
-        ],
-        "tha": [
-            51,
-            54,
-            55,
-            52,
-            53
-        ],
-        "tur": [
-            58,
-            57,
-            56
-        ],
-        "ukr": [
-            59
-        ],
-        "urd": [
-            60,
-            61,
-            62
-        ],
-        "uzn": [
-            63,
-            64,
-            65
-        ],
-        "vie": [
-            66,
-            67,
-            70,
-            71,
-            68,
-            69
-        ]
-    }
-}