Duplicate from kadirnar/Multilingual-Translation

Co-authored-by: Kadir Nar <[email protected]>

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,14 @@
+---
+title: Multilingual Translation
+emoji: 🌍
+colorFrom: yellow
+colorTo: pink
+sdk: gradio
+sdk_version: 3.17.0
+app_file: app.py
+pinned: false
+license: mit
+duplicated_from: kadirnar/Multilingual-Translation
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,61 @@
+from multilingual_translation import text_to_text_generation
+from utils import lang_ids, data_scraping
+import gradio as gr
+
+lang_list = list(lang_ids.keys())
+model_list = data_scraping()
+
+def multilingual_translate(
+    prompt: str, 
+    model_id: str, 
+    target_lang: str
+    ):
+    
+    return text_to_text_generation(
+        prompt=prompt, 
+        model_id=model_id, 
+        device='cpu',
+        target_lang=lang_ids[target_lang]
+        )
+
+
+inputs = [
+    gr.Textbox(lines=4, value="Hello world!", label="Input Text"),
+    gr.Dropdown(model_list, value="facebook/m2m100_418M", label="Model"),
+    gr.Dropdown(lang_list, value="Turkish", label="Target Language"),
+]
+
+output = gr.outputs.Textbox(label="Output Text")
+
+examples = [
+    [
+        "Hello world!",
+        "facebook/m2m100_418M",
+        "Turkish",
+    ],
+    [
+        "Omar ve Merve çok iyi arkadaşlar.",
+        "facebook/m2m100_418M",
+        "Spanish",
+    ],
+    [
+        "Hugging Face is a great company.",
+        "facebook/m2m100_418M",
+        "French",
+    ]
+]
+
+title = "Beyond English-Centric Multilingual Machine Translation"
+
+description = "M2M100 is a multilingual encoder-decoder (seq-to-seq) model trained for Many-to-Many multilingual translation. It was introduced in this [paper](https://arxiv.org/abs/2010.11125) and first released in [this](https://github.com/pytorch/fairseq/tree/master/examples/m2m_100) repository."
+
+app = gr.Interface(
+    fn=multilingual_translate,
+    inputs=inputs,
+    outputs=output,
+    examples=examples,
+    title=title,
+    description=description,
+    cache_examples=True
+)
+app.launch(debug=True, enable_queue=True)

requirements.txt

@@ -0,0 +1,4 @@
+torch
+beautifulsoup4==4.11.2
+multilingual_translation==0.0.5
+requests==2.28.1

utils.py

@@ -0,0 +1,128 @@
+from bs4 import BeautifulSoup
+import requests
+
+
+lang_ids = {
+    "Afrikaans": "af",
+    "Amharic": "am",
+    "Arabic": "ar",
+    "Asturian": "ast",
+    "Azerbaijani": "az",
+    "Bashkir": "ba",
+    "Belarusian": "be",
+    "Bulgarian": "bg",
+    "Bengali": "bn",
+    "Breton": "br",
+    "Bosnian": "bs",
+    "Catalan": "ca",
+    "Cebuano": "ceb",
+    "Czech": "cs",
+    "Welsh": "cy",
+    "Danish": "da",
+    "German": "de",
+    "Greeek": "el",
+    "English": "en",
+    "Spanish": "es",
+    "Estonian": "et",
+    "Persian": "fa",
+    "Fulah": "ff",
+    "Finnish": "fi",
+    "French": "fr",
+    "Western Frisian": "fy",
+    "Irish": "ga",
+    "Gaelic": "gd",
+    "Galician": "gl",
+    "Gujarati": "gu",
+    "Hausa": "ha",
+    "Hebrew": "he",
+    "Hindi": "hi",
+    "Croatian": "hr",
+    "Haitian": "ht",
+    "Hungarian": "hu",
+    "Armenian": "hy",
+    "Indonesian": "id",
+    "Igbo": "ig",
+    "Iloko": "ilo",
+    "Icelandic": "is",
+    "Italian": "it",
+    "Japanese": "ja",
+    "Javanese": "jv",
+    "Georgian": "ka",
+    "Kazakh": "kk",
+    "Central Khmer": "km",
+    "Kannada": "kn",
+    "Korean": "ko",
+    "Luxembourgish": "lb",
+    "Ganda": "lg",
+    "Lingala": "ln",
+    "Lao": "lo",
+    "Lithuanian": "lt",
+    "Latvian": "lv",
+    "Malagasy": "mg",
+    "Macedonian": "mk",
+    "Malayalam": "ml",
+    "Mongolian": "mn",
+    "Marathi": "mr",
+    "Malay": "ms",
+    "Burmese": "my",
+    "Nepali": "ne",
+    "Dutch": "nl",
+    "Norwegian": "no",
+    "Northern Sotho": "ns",
+    "Occitan": "oc",
+    "Oriya": "or",
+    "Panjabi": "pa",
+    "Polish": "pl",
+    "Pushto": "ps",
+    "Portuguese": "pt",
+    "Romanian": "ro",
+    "Russian": "ru",
+    "Sindhi": "sd",
+    "Sinhala": "si",
+    "Slovak": "sk",
+    "Slovenian": "sl",
+    "Somali": "so",
+    "Albanian": "sq",
+    "Serbian": "sr",
+    "Swati": "ss",
+    "Sundanese": "su",
+    "Swedish": "sv",
+    "Swahili": "sw",
+    "Tamil": "ta",
+    "Thai": "th",
+    "Tagalog": "tl",
+    "Tswana": "tn",
+    "Turkish": "tr",
+    "Ukrainian": "uk",
+    "Urdu": "ur",
+    "Uzbek": "uz",
+    "Vietnamese": "vi",
+    "Wolof": "wo",
+    "Xhosa": "xh",
+    "Yiddish": "yi",
+    "Yoruba": "yo",
+    "Chinese": "zh",
+    "Zulu": "zu",
+}
+
+def model_url_list():
+    url_list = []
+    for i in range(0, 5):
+        url_list.append(f"https://huggingface.co/models?other=m2m_100&p={i}&sort=downloads")
+    return url_list
+
+def data_scraping():
+    url_list = model_url_list()
+    model_list = []
+    for url in url_list:
+        response = requests.get(url)
+        soup = BeautifulSoup(response.text, "html.parser")
+        div_class = 'grid grid-cols-1 gap-5 2xl:grid-cols-2'
+        div = soup.find('div', {'class': div_class})
+        for a in div.find_all('a', href=True):
+            model_list.append(a['href'])
+    
+    for i in range(len(model_list)):
+        model_list[i] = model_list[i][1:]
+
+    return model_list