import gradio as gr import pandas as pd from langdetect import detect from datasets import load_dataset import threading, time, uuid, sqlite3, shutil, os, random, asyncio, threading from pathlib import Path from huggingface_hub import CommitScheduler, delete_file, hf_hub_download from gradio_client import Client, file import pyloudnorm as pyln import soundfile as sf import librosa from detoxify import Detoxify import os import tempfile from pydub import AudioSegment import itertools from typing import List, Tuple, Set, Dict from hashlib import md5, sha1 class User: def __init__(self, user_id: str): self.user_id = user_id self.voted_pairs: Set[Tuple[str, str]] = set() class Sample: def __init__(self, filename: str, transcript: str, modelName: str): self.filename = filename self.transcript = transcript self.modelName = modelName def match_target_amplitude(sound, target_dBFS): change_in_dBFS = target_dBFS - sound.dBFS return sound.apply_gain(change_in_dBFS) # from gradio_space_ci import enable_space_ci # enable_space_ci() toxicity = Detoxify('original') sents = [] with open('harvard_sentences.txt') as f: sents += f.read().strip().splitlines() with open('llama3_command-r_sentences.txt') as f: sents += f.read().strip().splitlines() # Credit: llama3_command-r sentences generated made by user KingNish #################################### # Constants #################################### AVAILABLE_MODELS = { # 'XTTSv2': 'xtts', # 'WhisperSpeech': 'whisperspeech', # 'ElevenLabs': 'eleven', # 'OpenVoice': 'openvoice', # 'OpenVoice V2': 'openvoicev2', # 'Play.HT 2.0': 'playht', # 'MetaVoice': 'metavoice', # 'MeloTTS': 'melo', # 'StyleTTS 2': 'styletts2', # 'GPT-SoVITS': 'sovits', # 'Vokan TTS': 'vokan', # 'VoiceCraft 2.0': 'voicecraft', # 'Parler TTS': 'parler' # HF Gradio Spaces: 'coqui/xtts': 'coqui/xtts', # 'collabora/WhisperSpeech': 'collabora/WhisperSpeech', # old gradio? 'myshell-ai/OpenVoice': 'myshell-ai/OpenVoice', # 4.29.0 'myshell-ai/OpenVoiceV2': 'myshell-ai/OpenVoiceV2', # 4.29.0 'mrfakename/MetaVoice-1B-v0.1': 'mrfakename/MetaVoice-1B-v0.1', # 4.29.0 #'Pendrokar/xVASynth-TTS': 'Pendrokar/xVASynth-TTS', # EN-GB 4.29.0 4.42.0 # 'coqui/CoquiTTS': 'coqui/CoquiTTS', 'LeeSangHoon/HierSpeech_TTS': 'LeeSangHoon/HierSpeech_TTS', # 4.29.0 'mrfakename/MeloTTS': 'mrfakename/MeloTTS', # 4.29.0 'fishaudio/fish-speech-1': 'fishaudio/fish-speech-1', # 4.29.0 # Parler 'parler-tts/parler_tts': 'parler-tts/parler_tts', # 4.29.0 4.42.0 'parler-tts/parler-tts-expresso': 'parler-tts/parler-tts-expresso', # 4.29.0 4.42.0 # Microsoft Edge TTS 'innoai/Edge-TTS-Text-to-Speech': 'innoai/Edge-TTS-Text-to-Speech', # TTS w issues # 'PolyAI/pheme': '/predict#0', # sleepy HF Space # 'amphion/Text-to-Speech': '/predict#0', # old running space, takes a whole minute to synthesize # 'suno/bark': '3#0', # Hallucinates # 'shivammehta25/Matcha-TTS': '5#0', # seems to require multiple requests for setup # 'styletts2/styletts2': '0#0', # API disabled # 'Manmay/tortoise-tts': '/predict#0', # Cannot skip text-from-file parameter # 'pytorch/Tacotron2': '0#0', # old gradio # 'parler-tts/parler_tts_mini': 'parler-tts/parler_tts_mini', # old gradio - ValueError: Unsupported protocol: sse_v3 } HF_SPACES = { # XTTS v2 'coqui/xtts': { 'name': 'XTTS v2', 'function': '1', 'text_param_index': 0, 'return_audio_index': 1, }, # WhisperSpeech 'collabora/WhisperSpeech': { 'name': 'WhisperSpeech', 'function': '/whisper_speech_demo', 'text_param_index': 0, 'return_audio_index': 0, }, # OpenVoice (MyShell.ai) 'myshell-ai/OpenVoice': { 'name':'OpenVoice', 'function': '1', 'text_param_index': 0, 'return_audio_index': 1, }, # OpenVoice v2 (MyShell.ai) 'myshell-ai/OpenVoiceV2': { 'name':'OpenVoice v2', 'function': '1', 'text_param_index': 0, 'return_audio_index': 1, }, # MetaVoice 'mrfakename/MetaVoice-1B-v0.1': { 'name':'MetaVoice', 'function': '/tts', 'text_param_index': 0, 'return_audio_index': 0, }, # xVASynth (CPU) 'Pendrokar/xVASynth-TTS': { 'name': 'xVASynth v3', 'function': '/predict', 'text_param_index': 0, 'return_audio_index': 0, }, # CoquiTTS (CPU) 'coqui/CoquiTTS': { 'name': 'CoquiTTS', 'function': '0', 'text_param_index': 0, 'return_audio_index': 0, }, # HierSpeech_TTS 'LeeSangHoon/HierSpeech_TTS': { 'name': 'HierSpeech++', 'function': '/predict', 'text_param_index': 0, 'return_audio_index': 0, }, # MeloTTS (MyShell.ai) 'mrfakename/MeloTTS': { 'name': 'MeloTTS', 'function': '/synthesize', 'text_param_index': 0, 'return_audio_index': 0, }, # Parler 'parler-tts/parler_tts': { 'name': 'Parler', 'function': '/gen_tts', 'text_param_index': 0, 'return_audio_index': 0, }, # Parler 'parler-tts/parler_tts_mini': { 'name': 'Parler Mini', 'function': '/gen_tts', 'text_param_index': 0, 'return_audio_index': 0, }, # Parler, using Expresso dataset 'parler-tts/parler-tts-expresso': { 'name': 'Parler Expresso', 'function': '/gen_tts', 'text_param_index': 0, 'return_audio_index': 0, }, # Microsoft Edge TTS 'innoai/Edge-TTS-Text-to-Speech': { 'name': 'Edge TTS', 'function': '/predict', 'text_param_index': 0, 'return_audio_index': 0, }, 'fishaudio/fish-speech-1': { 'name': 'Fish Speech', 'function': '/inference_wrapper', 'text_param_index': 0, 'return_audio_index': 1, }, # TTS w issues # 'PolyAI/pheme': '/predict#0', #sleepy HF Space # 'amphion/Text-to-Speech': '/predict#0', #takes a whole minute to synthesize # 'suno/bark': '3#0', # Hallucinates # 'shivammehta25/Matcha-TTS': '5#0', #seems to require multiple requests for setup # 'styletts2/styletts2': '0#0', #API disabled # 'Manmay/tortoise-tts': '/predict#0', #Cannot skip text-from-file parameter # 'pytorch/Tacotron2': '0#0', #old gradio # 'fishaudio/fish-speech-1': '/inference_wrapper#0', heavy hallucinations } # for zero-shot TTS - voice sample of Scarlett Johanson DEFAULT_VOICE_SAMPLE_STR = 'https://cdn-uploads.huggingface.co/production/uploads/641de0213239b631552713e4/iKHHqWxWy6Zfmp6QP6CZZ.wav' DEFAULT_VOICE_SAMPLE = file(DEFAULT_VOICE_SAMPLE_STR) DEFAULT_VOICE_TRANSCRIPT = "In the first half of the 20th century, science fiction familiarized the world with the concept of artificially intelligent robots. It began with the “heartless” Tin man from the Wizard of Oz and continued with the humanoid robot that impersonated Maria in Metropolis. By the 1950s, we had a generation of scientists, mathematicians, and philosophers with the concept of artificial intelligence (or AI) culturally assimilated in their minds." OVERRIDE_INPUTS = { 'coqui/xtts': { 1: 'en', 2: 'https://cdn-uploads.huggingface.co/production/uploads/63d52e0c4e5642795617f668/V6-rMmI-P59DA4leWDIcK.wav', # voice sample 3: None, # mic voice sample 4: False, #use_mic 5: False, #cleanup_reference 6: False, #auto_detect }, 'collabora/WhisperSpeech': { 1: DEFAULT_VOICE_SAMPLE, # voice sample 2: DEFAULT_VOICE_SAMPLE, # voice sample URL 3: 14.0, #Tempo - Gradio Slider issue: takes min. rather than value }, 'myshell-ai/OpenVoice': { 1: 'default', # style 2: DEFAULT_VOICE_SAMPLE_STR, # voice sample }, 'myshell-ai/OpenVoiceV2': { 1: 'en_default', # style 2: DEFAULT_VOICE_SAMPLE_STR, # voice sample }, 'PolyAI/pheme': { 1: 'YOU1000000044_S0000798', # voice 2: 210, 3: 0.7, #Tempo - Gradio Slider issue: takes min. rather than value }, 'Pendrokar/xVASynth-TTS': { 1: 'ccby_nvidia_hifi_92_F', #fine-tuned voice model name; #92 BRITISH 3: 1.0, #pacing/duration - Gradio Slider issue: takes min. rather than value }, 'suno/bark': { 1: 'Speaker 3 (en)', # voice }, 'amphion/Text-to-Speech': { 1: 'LikeManyWaters', # voice }, 'LeeSangHoon/HierSpeech_TTS': { 1: file('https://huggingface.co/spaces/LeeSangHoon/HierSpeech_TTS/resolve/main/example/female.wav'), # voice sample 2: 0.333, 3: 0.333, 4: 1, 5: 1, 6: 0, 7: 1111, }, 'Manmay/tortoise-tts': { 1: None, # text-from-file 2: 'angie', # voice 3: 'disabled', # second voice for a dialogue 4: 'No', # split by newline }, 'mrfakename/MeloTTS': { 1: 'EN-Default', # speaker; DEFAULT_VOICE_SAMPLE=EN-Default 2: 1, # speed 3: 'EN', # language }, 'mrfakename/MetaVoice-1B-v0.1': { 1: 5, # float (numeric value between 0.0 and 10.0) in 'Speech Stability - improves text following for a challenging speaker' Slider component 2: 5, # float (numeric value between 1.0 and 5.0) in 'Speaker similarity - How closely to match speaker identity and speech style.' Slider component 3: "Preset voices", # Literal['Preset voices', 'Upload target voice'] in 'Choose voice' Radio component 4: "Bria", # Literal['Bria', 'Alex', 'Jacob'] in 'Preset voices' Dropdown component 5: None, # filepath in 'Upload a clean sample to clone. Sample should contain 1 speaker, be between 30-90 seconds and not contain background noise.' Audio component }, 'parler-tts/parler_tts': { 1: 'Elisabeth; Elisabeth\'s female voice; very clear audio', # description/prompt }, 'parler-tts/parler-tts-expresso': { 1: 'Elisabeth; Elisabeth\'s female voice; very clear audio', # description/prompt }, 'innoai/Edge-TTS-Text-to-Speech': { 1: 'en-US-EmmaMultilingualNeural - en-US (Female)', # voice 2: 0, # pace rate 3: 0, # pitch }, 'fishaudio/fish-speech-1': { 1: True, # enable_reference_audio 2: file('https://huggingface.co/spaces/fishaudio/fish-speech-1/resolve/main/examples/English.wav'), # reference_audio 3: 'In the ancient land of Eldoria, where the skies were painted with shades of mystic hues and the forests whispered secrets of old, there existed a dragon named Zephyros. Unlike the fearsome tales of dragons that plagued human hearts with terror, Zephyros was a creature of wonder and wisdom, revered by all who knew of his existence.', # reference_text 4: 0, # max_new_tokens 5: 200, # chunk_length 6: 0.7, # top_p 7: 1.2, # repetition_penalty 8: 0.7, # temperature 9: 1, # batch_infer_num 10: False, # if_load_asr_model }, } hf_clients = {} # cache audio samples for quick voting cached_samples: List[Sample] = [] voting_users = { # userid as the key and USER() as the value } def generate_matching_pairs(samples: List[Sample]) -> List[Tuple[Sample, Sample]]: transcript_groups: Dict[str, List[Sample]] = {} samples = random.sample(samples, k=len(samples)) for sample in samples: if sample.transcript not in transcript_groups: transcript_groups[sample.transcript] = [] transcript_groups[sample.transcript].append(sample) matching_pairs: List[Tuple[Sample, Sample]] = [] for group in transcript_groups.values(): matching_pairs.extend(list(itertools.combinations(group, 2))) return matching_pairs # List[Tuple[Sample, Sample]] all_pairs = [] SPACE_ID = os.getenv('SPACE_ID') MAX_SAMPLE_TXT_LENGTH = 300 MIN_SAMPLE_TXT_LENGTH = 10 DB_DATASET_ID = os.getenv('DATASET_ID') DB_NAME = "database.db" SPACE_ID = 'TTS-AGI/TTS-Arena' # If /data available => means local storage is enabled => let's use it! DB_PATH = f"/data/{DB_NAME}" if os.path.isdir("/data") else DB_NAME print(f"Using {DB_PATH}") # AUDIO_DATASET_ID = "ttseval/tts-arena-new" CITATION_TEXT = """@misc{tts-arena, title = {Text to Speech Arena}, author = {mrfakename and Srivastav, Vaibhav and Fourrier, Clémentine and Pouget, Lucain and Lacombe, Yoach and main and Gandhi, Sanchit}, year = 2024, publisher = {Hugging Face}, howpublished = "\\url{https://huggingface.co/spaces/TTS-AGI/TTS-Arena}" }""" #################################### # Functions #################################### def create_db_if_missing(): conn = get_db() cursor = conn.cursor() cursor.execute(''' CREATE TABLE IF NOT EXISTS model ( name TEXT UNIQUE, upvote INTEGER, downvote INTEGER ); ''') cursor.execute(''' CREATE TABLE IF NOT EXISTS vote ( id INTEGER PRIMARY KEY AUTOINCREMENT, username TEXT, model TEXT, vote INTEGER, timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP ); ''') cursor.execute(''' CREATE TABLE IF NOT EXISTS votelog ( id INTEGER PRIMARY KEY AUTOINCREMENT, username TEXT, chosen TEXT, rejected TEXT, timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP ); ''') cursor.execute(''' CREATE TABLE IF NOT EXISTS spokentext ( id INTEGER PRIMARY KEY AUTOINCREMENT, spokentext TEXT, timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP ); ''') def get_db(): return sqlite3.connect(DB_PATH) #################################### # Space initialization #################################### # Download existing DB if not os.path.isfile(DB_PATH): print("Downloading DB...") try: cache_path = hf_hub_download(repo_id=DB_DATASET_ID, repo_type='dataset', filename=DB_NAME) shutil.copyfile(cache_path, DB_PATH) print("Downloaded DB") except Exception as e: print("Error while downloading DB:", e) # Create DB table (if doesn't exist) create_db_if_missing() hf_token = os.getenv('HF_TOKEN') # Sync local DB with remote repo every 5 minute (only if a change is detected) scheduler = CommitScheduler( repo_id=DB_DATASET_ID, repo_type="dataset", folder_path=Path(DB_PATH).parent, every=5, allow_patterns=DB_NAME, ) # Load audio dataset # audio_dataset = load_dataset(AUDIO_DATASET_ID) #################################### # Router API #################################### # router = Client("TTS-AGI/tts-router", hf_token=hf_token) router = {} #################################### # Gradio app #################################### MUST_BE_LOGGEDIN = "Please login with Hugging Face to participate in the TTS Arena." DESCR = """ # TTS Spaces Arena: Benchmarking Gradio hosted TTS Models in the Wild Vote to help the community find the best available text-to-speech model! """.strip() # INSTR = """ # ## Instructions # * Listen to two anonymous models # * Vote on which synthesized audio sounds more natural to you # * If there's a tie, click Skip # **When you're ready to begin, login and begin voting!** The model names will be revealed once you vote. # """.strip() INSTR = """ ## 🗳️ Vote * Press ⚡ to get cached sample pairs you've yet to vote on. (Fast 🐇) * Or press 🎲 to randomly use a sentence from the list. (Slow 🐢) * Or input text (🇺🇸 English only) to synthesize audio. (Slowest 🐌 due to _Toxicity_ test) * Listen to the two audio clips, one after the other. * _Vote on which audio sounds more natural to you._ * Model names are revealed after the vote is cast. ⚠ Note: It **may take up to 30 seconds** to ***synthesize*** audio. """.strip() request = '' if SPACE_ID: request = f""" ### Request a model Please [create a Discussion](https://huggingface.co/spaces/{SPACE_ID}/discussions/new) to request a model. """ ABOUT = f""" ## 📄 About The TTS Arena evaluates leading speech synthesis models. It is inspired by LMsys's [Chatbot Arena](https://chat.lmsys.org/). ### Motivation The field of speech synthesis has long lacked an accurate method to measure the quality of different models. Objective metrics like WER (word error rate) are unreliable measures of model quality, and subjective measures such as MOS (mean opinion score) are typically small-scale experiments conducted with few listeners. As a result, these measurements are generally not useful for comparing two models of roughly similar quality. To address these drawbacks, we are inviting the community to rank models in an easy-to-use interface, and opening it up to the public in order to make both the opportunity to rank models, as well as the results, more easily accessible to everyone. ### The Arena The leaderboard allows a user to enter text, which will be synthesized by two models. After listening to each sample, the user can vote on which model sounds more natural. Due to the risks of human bias and abuse, model names are revealed only after a vote is submitted. ### Credits Thank you to the following individuals who helped make this project possible: * VB ([Twitter](https://twitter.com/reach_vb) / [Hugging Face](https://huggingface.co/reach-vb)) * Clémentine Fourrier ([Twitter](https://twitter.com/clefourrier) / [Hugging Face](https://huggingface.co/clefourrier)) * Lucain Pouget ([Twitter](https://twitter.com/Wauplin) / [Hugging Face](https://huggingface.co/Wauplin)) * Yoach Lacombe ([Twitter](https://twitter.com/yoachlacombe) / [Hugging Face](https://huggingface.co/ylacombe)) * Main Horse ([Twitter](https://twitter.com/main_horse) / [Hugging Face](https://huggingface.co/main-horse)) * Sanchit Gandhi ([Twitter](https://twitter.com/sanchitgandhi99) / [Hugging Face](https://huggingface.co/sanchit-gandhi)) * Apolinário Passos ([Twitter](https://twitter.com/multimodalart) / [Hugging Face](https://huggingface.co/multimodalart)) * Pedro Cuenca ([Twitter](https://twitter.com/pcuenq) / [Hugging Face](https://huggingface.co/pcuenq)) {request} ### Privacy statement We may store text you enter and generated audio. We store a unique ID for each session. You agree that we may collect, share, and/or publish any data you input for research and/or commercial purposes. ### License Generated audio clips cannot be redistributed and may be used for personal, non-commercial use only. Random sentences are sourced from a filtered subset of the [Harvard Sentences](https://www.cs.columbia.edu/~hgs/audio/harvard.html). """.strip() LDESC = """ ## 🏆 Leaderboard Vote to help the community determine the best text-to-speech (TTS) models. The leaderboard displays models in descending order of how natural they sound (based on votes cast by the community). Important: In order to help keep results fair, the leaderboard hides results by default until the number of votes passes a threshold. Tick the `Reveal preliminary results` to show models without sufficient votes. Please note that preliminary results may be inaccurate. """.strip() # def reload_audio_dataset(): # global audio_dataset # audio_dataset = load_dataset(AUDIO_DATASET_ID) # return 'Reload Audio Dataset' def del_db(txt): if not txt.lower() == 'delete db': raise gr.Error('You did not enter "delete db"') # Delete local + remote os.remove(DB_PATH) delete_file(path_in_repo=DB_NAME, repo_id=DB_DATASET_ID, repo_type='dataset') # Recreate create_db_if_missing() return 'Delete DB' theme = gr.themes.Base( font=[gr.themes.GoogleFont('Libre Franklin'), gr.themes.GoogleFont('Public Sans'), 'system-ui', 'sans-serif'], ) model_names = { 'styletts2': 'StyleTTS 2', 'tacotron': 'Tacotron', 'tacotronph': 'Tacotron Phoneme', 'tacotrondca': 'Tacotron DCA', 'speedyspeech': 'Speedy Speech', 'overflow': 'Overflow TTS', 'vits': 'VITS', 'vitsneon': 'VITS Neon', 'neuralhmm': 'Neural HMM', 'glow': 'Glow TTS', 'fastpitch': 'FastPitch', 'jenny': 'Jenny', 'tortoise': 'Tortoise TTS', 'xtts2': 'Coqui XTTSv2', 'xtts': 'Coqui XTTS', 'openvoice': 'MyShell OpenVoice', 'elevenlabs': 'ElevenLabs', 'openai': 'OpenAI', 'hierspeech': 'HierSpeech++', 'pheme': 'PolyAI Pheme', 'speecht5': 'SpeechT5', 'metavoice': 'MetaVoice-1B', } model_licenses = { 'styletts2': 'MIT', 'tacotron': 'BSD-3', 'tacotronph': 'BSD-3', 'tacotrondca': 'BSD-3', 'speedyspeech': 'BSD-3', 'overflow': 'MIT', 'vits': 'MIT', 'openvoice': 'MIT', 'vitsneon': 'BSD-3', 'neuralhmm': 'MIT', 'glow': 'MIT', 'fastpitch': 'Apache 2.0', 'jenny': 'Jenny License', 'tortoise': 'Apache 2.0', 'xtts2': 'CPML (NC)', 'xtts': 'CPML (NC)', 'elevenlabs': 'Proprietary', 'eleven': 'Proprietary', 'openai': 'Proprietary', 'hierspeech': 'MIT', 'pheme': 'CC-BY', 'speecht5': 'MIT', 'metavoice': 'Apache 2.0', 'elevenlabs': 'Proprietary', 'whisperspeech': 'MIT', 'Pendrokar/xVASynth': 'GPT3', } model_links = { 'styletts2': 'https://github.com/yl4579/StyleTTS2', 'tacotron': 'https://github.com/NVIDIA/tacotron2', 'speedyspeech': 'https://github.com/janvainer/speedyspeech', 'overflow': 'https://github.com/shivammehta25/OverFlow', 'vits': 'https://github.com/jaywalnut310/vits', 'openvoice': 'https://github.com/myshell-ai/OpenVoice', 'neuralhmm': 'https://github.com/ketranm/neuralHMM', 'glow': 'https://github.com/jaywalnut310/glow-tts', 'fastpitch': 'https://fastpitch.github.io/', 'tortoise': 'https://github.com/neonbjb/tortoise-tts', 'xtts2': 'https://huggingface.co/coqui/XTTS-v2', 'xtts': 'https://huggingface.co/coqui/XTTS-v1', 'elevenlabs': 'https://elevenlabs.io/', 'openai': 'https://help.openai.com/en/articles/8555505-tts-api', 'hierspeech': 'https://github.com/sh-lee-prml/HierSpeechpp', 'pheme': 'https://github.com/PolyAI-LDN/pheme', 'speecht5': 'https://github.com/microsoft/SpeechT5', 'metavoice': 'https://github.com/metavoiceio/metavoice-src', } # def get_random_split(existing_split=None): # choice = random.choice(list(audio_dataset.keys())) # if existing_split and choice == existing_split: # return get_random_split(choice) # else: # return choice # def get_random_splits(): # choice1 = get_random_split() # choice2 = get_random_split(choice1) # return (choice1, choice2) def model_license(name): print(name) for k, v in AVAILABLE_MODELS.items(): if k == name: if v in model_licenses: return model_licenses[v] print('---') return 'Unknown' def get_leaderboard(reveal_prelim = False): conn = get_db() cursor = conn.cursor() sql = 'SELECT name, upvote, downvote FROM model' # if not reveal_prelim: sql += ' WHERE EXISTS (SELECT 1 FROM model WHERE (upvote + downvote) > 750)' if not reveal_prelim: sql += ' WHERE (upvote + downvote) > 300' cursor.execute(sql) data = cursor.fetchall() df = pd.DataFrame(data, columns=['name', 'upvote', 'downvote']) # df['license'] = df['name'].map(model_license) df['name'] = df['name'].replace(model_names) for i in range(len(df)): df.loc[i, "name"] = make_link_to_space(df['name'][i]) df['votes'] = df['upvote'] + df['downvote'] # df['score'] = round((df['upvote'] / df['votes']) * 100, 2) # Percentage score ## ELO SCORE df['score'] = 1200 for i in range(len(df)): for j in range(len(df)): if i != j: expected_a = 1 / (1 + 10 ** ((df['score'][j] - df['score'][i]) / 400)) expected_b = 1 / (1 + 10 ** ((df['score'][i] - df['score'][j]) / 400)) actual_a = df['upvote'][i] / df['votes'][i] actual_b = df['upvote'][j] / df['votes'][j] df.at[i, 'score'] += round(32 * (actual_a - expected_a)) df.at[j, 'score'] += round(32 * (actual_b - expected_b)) df['score'] = round(df['score']) ## ELO SCORE df = df.sort_values(by='score', ascending=False) # medals def assign_medal(rank, assign): rank = str(rank + 1) if assign: if rank == '1': rank += '🥇' elif rank == '2': rank += '🥈' elif rank == '3': rank += '🥉' return '#'+ rank df['order'] = [assign_medal(i, not reveal_prelim and len(df) > 2) for i in range(len(df))] df = df[['order', 'name', 'score', 'votes']] return df def make_link_to_space(model_name): # create a anchor link if a HF space style = 'text-decoration: underline;text-decoration-style: dotted;' title = '' # bolden actual name # model_name_split = model_name.split('/') # model_name_split = model_name_split[:-1].join('/') +'/'+ model_name_split[-1] +'' if model_name in AVAILABLE_MODELS: style += 'color: var(--link-text-color);' title = model_name else: style += 'font-style: italic;' title = 'Got disabled for Arena (See AVAILABLE_MODELS within code for why)' model_basename = model_name if model_name in HF_SPACES: model_basename = HF_SPACES[model_name]['name'] if '/' in model_name: return '🤗 '+ model_basename +'' # otherwise just return the model name return model_name def markdown_link_to_space(model_name): # create a anchor link if a HF space using markdown syntax if '/' in model_name: return '🤗 [' + model_name + '](https://huggingface.co/spaces/' + model_name + ')' # otherwise just return the model name return model_name def mkuuid(uid): if not uid: uid = uuid.uuid4() return uid def upvote_model(model, uname): conn = get_db() cursor = conn.cursor() cursor.execute('UPDATE model SET upvote = upvote + 1 WHERE name = ?', (model,)) if cursor.rowcount == 0: cursor.execute('INSERT OR REPLACE INTO model (name, upvote, downvote) VALUES (?, 1, 0)', (model,)) cursor.execute('INSERT INTO vote (username, model, vote) VALUES (?, ?, ?)', (uname, model, 1,)) with scheduler.lock: conn.commit() cursor.close() def log_text(text): conn = get_db() cursor = conn.cursor() cursor.execute('INSERT INTO spokentext (spokentext) VALUES (?)', (text,)) with scheduler.lock: conn.commit() cursor.close() def downvote_model(model, uname): conn = get_db() cursor = conn.cursor() cursor.execute('UPDATE model SET downvote = downvote + 1 WHERE name = ?', (model,)) if cursor.rowcount == 0: cursor.execute('INSERT OR REPLACE INTO model (name, upvote, downvote) VALUES (?, 0, 1)', (model,)) cursor.execute('INSERT INTO vote (username, model, vote) VALUES (?, ?, ?)', (uname, model, -1,)) with scheduler.lock: conn.commit() cursor.close() def a_is_better(model1, model2, userid): # print("A is better", model1, model2) if not model1 in AVAILABLE_MODELS.keys() and not model1 in AVAILABLE_MODELS.values(): raise gr.Error('Sorry, please try voting again.') userid = mkuuid(userid) if model1 and model2: conn = get_db() cursor = conn.cursor() cursor.execute('INSERT INTO votelog (username, chosen, rejected) VALUES (?, ?, ?)', (str(userid), model1, model2,)) with scheduler.lock: conn.commit() cursor.close() upvote_model(model1, str(userid)) downvote_model(model2, str(userid)) return reload(model1, model2, userid, chose_a=True) def b_is_better(model1, model2, userid): # print("B is better", model1, model2) if not model1 in AVAILABLE_MODELS.keys() and not model1 in AVAILABLE_MODELS.values(): raise gr.Error('Sorry, please try voting again.') userid = mkuuid(userid) if model1 and model2: conn = get_db() cursor = conn.cursor() cursor.execute('INSERT INTO votelog (username, chosen, rejected) VALUES (?, ?, ?)', (str(userid), model2, model1,)) with scheduler.lock: conn.commit() cursor.close() upvote_model(model2, str(userid)) downvote_model(model1, str(userid)) return reload(model1, model2, userid, chose_b=True) def both_bad(model1, model2, userid): userid = mkuuid(userid) if model1 and model2: downvote_model(model1, str(userid)) downvote_model(model2, str(userid)) return reload(model1, model2, userid) def both_good(model1, model2, userid): userid = mkuuid(userid) if model1 and model2: upvote_model(model1, str(userid)) upvote_model(model2, str(userid)) return reload(model1, model2, userid) def reload(chosenmodel1=None, chosenmodel2=None, userid=None, chose_a=False, chose_b=False): # Select random splits # row = random.choice(list(audio_dataset['train'])) # options = list(random.choice(list(audio_dataset['train'])).keys()) # split1, split2 = random.sample(options, 2) # choice1, choice2 = (row[split1], row[split2]) # if chosenmodel1 in model_names: # chosenmodel1 = model_names[chosenmodel1] # if chosenmodel2 in model_names: # chosenmodel2 = model_names[chosenmodel2] # out = [ # (choice1['sampling_rate'], choice1['array']), # (choice2['sampling_rate'], choice2['array']), # split1, # split2 # ] # if userid: out.append(userid) # if chosenmodel1: out.append(f'This model was {chosenmodel1}') # if chosenmodel2: out.append(f'This model was {chosenmodel2}') # return out # return (f'This model was {chosenmodel1}', f'This model was {chosenmodel2}', gr.update(visible=False), gr.update(visible=False)) # return (gr.update(variant='secondary', value=chosenmodel1, interactive=False), gr.update(variant='secondary', value=chosenmodel2, interactive=False)) chosenmodel1 = make_link_to_space(chosenmodel1) chosenmodel2 = make_link_to_space(chosenmodel2) out = [ gr.update(interactive=False, visible=False), gr.update(interactive=False, visible=False) ] style = 'text-align: center; font-size: 1rem; margin-bottom: 0; padding: var(--input-padding)' if chose_a == True: out.append(gr.update(value=f'

Your vote: {chosenmodel1}

', visible=True)) out.append(gr.update(value=f'

{chosenmodel2}

', visible=True)) else: out.append(gr.update(value=f'

{chosenmodel1}

', visible=True)) out.append(gr.update(value=f'

Your vote: {chosenmodel2}

', visible=True)) out.append(gr.update(visible=True)) return out with gr.Blocks() as leaderboard: gr.Markdown(LDESC) # df = gr.Dataframe(interactive=False, value=get_leaderboard()) df = gr.Dataframe( interactive=False, min_width=0, wrap=True, column_widths=[30, 200, 50, 50], datatype=["str", "html", "number", "number"] ) with gr.Row(): reveal_prelim = gr.Checkbox(label="Reveal preliminary results", info="Show all models, including models with very few human ratings.", scale=1) reloadbtn = gr.Button("Refresh", scale=3) reveal_prelim.input(get_leaderboard, inputs=[reveal_prelim], outputs=[df]) leaderboard.load(get_leaderboard, inputs=[reveal_prelim], outputs=[df]) reloadbtn.click(get_leaderboard, inputs=[reveal_prelim], outputs=[df]) # gr.Markdown("DISCLAIMER: The licenses listed may not be accurate or up to date, you are responsible for checking the licenses before using the models. Also note that some models may have additional usage restrictions.") # with gr.Blocks() as vote: # useridstate = gr.State() # gr.Markdown(INSTR) # # gr.LoginButton() # with gr.Row(): # gr.HTML('

Model A

') # gr.HTML('

Model B

') # model1 = gr.Textbox(interactive=False, visible=False, lines=1, max_lines=1) # model2 = gr.Textbox(interactive=False, visible=False, lines=1, max_lines=1) # # with gr.Group(): # # with gr.Row(): # # prevmodel1 = gr.Textbox(interactive=False, show_label=False, container=False, value="Vote to reveal model A") # # prevmodel2 = gr.Textbox(interactive=False, show_label=False, container=False, value="Vote to reveal model B", text_align="right") # # with gr.Row(): # # aud1 = gr.Audio(interactive=False, show_label=False, show_download_button=False, show_share_button=False, waveform_options={'waveform_progress_color': '#3C82F6'}) # # aud2 = gr.Audio(interactive=False, show_label=False, show_download_button=False, show_share_button=False, waveform_options={'waveform_progress_color': '#3C82F6'}) # with gr.Group(): # with gr.Row(): # with gr.Column(): # with gr.Group(): # prevmodel1 = gr.Textbox(interactive=False, show_label=False, container=False, value="Vote to reveal model A", lines=1, max_lines=1) # aud1 = gr.Audio(interactive=False, show_label=False, show_download_button=False, show_share_button=False, waveform_options={'waveform_progress_color': '#3C82F6'}) # with gr.Column(): # with gr.Group(): # prevmodel2 = gr.Textbox(interactive=False, show_label=False, container=False, value="Vote to reveal model B", text_align="right", lines=1, max_lines=1) # aud2 = gr.Audio(interactive=False, show_label=False, show_download_button=False, show_share_button=False, waveform_options={'waveform_progress_color': '#3C82F6'}) # with gr.Row(): # abetter = gr.Button("A is Better", variant='primary', scale=4) # # skipbtn = gr.Button("Skip", scale=1) # bbetter = gr.Button("B is Better", variant='primary', scale=4) # with gr.Row(): # bothbad = gr.Button("Both are Bad", scale=2) # skipbtn = gr.Button("Skip", scale=1) # bothgood = gr.Button("Both are Good", scale=2) # outputs = [aud1, aud2, model1, model2, useridstate, prevmodel1, prevmodel2] # abetter.click(a_is_better, outputs=outputs, inputs=[model1, model2, useridstate]) # bbetter.click(b_is_better, outputs=outputs, inputs=[model1, model2, useridstate]) # skipbtn.click(b_is_better, outputs=outputs, inputs=[model1, model2, useridstate]) # bothbad.click(both_bad, outputs=outputs, inputs=[model1, model2, useridstate]) # bothgood.click(both_good, outputs=outputs, inputs=[model1, model2, useridstate]) # vote.load(reload, outputs=[aud1, aud2, model1, model2]) def doloudnorm(path): data, rate = sf.read(path) meter = pyln.Meter(rate) loudness = meter.integrated_loudness(data) loudness_normalized_audio = pyln.normalize.loudness(data, loudness, -12.0) sf.write(path, loudness_normalized_audio, rate) def doresample(path_to_wav): pass ########################## # 2x speedup (hopefully) # ########################## def synthandreturn(text): text = text.strip() if len(text) > MAX_SAMPLE_TXT_LENGTH: raise gr.Error(f'You exceeded the limit of {MAX_SAMPLE_TXT_LENGTH} characters') if len(text) < MIN_SAMPLE_TXT_LENGTH: raise gr.Error(f'Please input a text longer than {MIN_SAMPLE_TXT_LENGTH} characters') if ( # test toxicity if not prepared text text not in sents and toxicity.predict(text)['toxicity'] > 0.8 ): print(f'Detected toxic content! "{text}"') raise gr.Error('Your text failed the toxicity test') if not text: raise gr.Error(f'You did not enter any text') # Check language try: if ( text not in sents and not detect(text) == "en" ): gr.Warning('Warning: The input text may not be in English') except: pass # Get two random models # forced model: your TTS model versus The World!!! # mdl1 = 'Pendrokar/xVASynth' vsModels = dict(AVAILABLE_MODELS) # del vsModels[mdl1] # randomize position of the forced model mdl2 = random.sample(list(vsModels.keys()), 1) # forced random # mdl1, mdl2 = random.sample(list([mdl1, mdl2[0]]), 2) # actual random mdl1, mdl2 = random.sample(list(AVAILABLE_MODELS.keys()), 2) log_text(text) print("[debug] Using", mdl1, mdl2) def predict_and_update_result(text, model, result_storage): # 3 attempts attempt_count = 0 while attempt_count < 3: try: if model in AVAILABLE_MODELS: if '/' in model: # Use public HF Space #if (model not in hf_clients): hf_clients[model] = Client(model, hf_token=hf_token) mdl_space = hf_clients[model] # print(f"{model}: Fetching endpoints of HF Space") # assume the index is one of the first 9 return params return_audio_index = int(HF_SPACES[model]['return_audio_index']) endpoints = mdl_space.view_api(all_endpoints=True, print_info=False, return_format='dict') api_name = None fn_index = None end_parameters = None # has named endpoint if '/' == HF_SPACES[model]['function'][0]: # audio sync function name api_name = HF_SPACES[model]['function'] end_parameters = _get_param_examples( endpoints['named_endpoints'][api_name]['parameters'] ) # has unnamed endpoint else: # endpoint index is the first character fn_index = int(HF_SPACES[model]['function']) end_parameters = _get_param_examples( endpoints['unnamed_endpoints'][str(fn_index)]['parameters'] ) space_inputs = _override_params(end_parameters, model) # force text space_inputs[HF_SPACES[model]['text_param_index']] = text print(f"{model}: Sending request to HF Space") results = mdl_space.predict(*space_inputs, api_name=api_name, fn_index=fn_index) # return path to audio result = results if (not isinstance(results, str)): # return_audio_index may be a filepath string result = results[return_audio_index] if (isinstance(result, dict)): # return_audio_index is a dictionary result = results[return_audio_index]['value'] else: # Use the private HF Space result = router.predict(text, AVAILABLE_MODELS[model].lower(), api_name="/synthesize") else: result = router.predict(text, model.lower(), api_name="/synthesize") break except Exception as e: attempt_count += 1 print(repr(e)) print(f"{model}: Unable to call API (attempt: {attempt_count})") # sleep for three seconds time.sleep(3) # Fetch and store client again #hf_clients[model] = Client(model, hf_token=hf_token) if attempt_count > 2: raise gr.Error(f"{model}: Failed to call model") else: print('Done with', model) try: with tempfile.NamedTemporaryFile(suffix='.wav', delete=False) as f: audio = AudioSegment.from_file(result) current_sr = audio.frame_rate if current_sr > 24000: print(f"{model}: Resampling") audio = audio.set_frame_rate(24000) try: print(f"{model}: Trying to normalize audio") audio = match_target_amplitude(audio, -20) except: print(f"{model}: [WARN] Unable to normalize audio") audio.export(f.name, format="wav") os.unlink(result) result = f.name except: print(f"{model}: [WARN] Unable to resample audio") pass if model in AVAILABLE_MODELS.keys(): model = AVAILABLE_MODELS[model] result_storage[model] = result def _get_param_examples(parameters): example_inputs = [] for param_info in parameters: if ( param_info['component'] == 'Radio' or param_info['component'] == 'Dropdown' or param_info['component'] == 'Audio' or param_info['python_type']['type'] == 'str' ): example_inputs.append(str(param_info['example_input'])) continue if param_info['python_type']['type'] == 'int': example_inputs.append(int(param_info['example_input'])) continue if param_info['python_type']['type'] == 'float': example_inputs.append(float(param_info['example_input'])) continue if param_info['python_type']['type'] == 'bool': example_inputs.append(bool(param_info['example_input'])) continue return example_inputs def _override_params(inputs, modelname): try: for key,value in OVERRIDE_INPUTS[modelname].items(): inputs[key] = value print(f"{modelname}: Default inputs overridden by Arena") except: pass return inputs mdl1k = mdl1 mdl2k = mdl2 print(mdl1k, mdl2k) if mdl1 in AVAILABLE_MODELS.keys(): mdl1k=AVAILABLE_MODELS[mdl1] if mdl2 in AVAILABLE_MODELS.keys(): mdl2k=AVAILABLE_MODELS[mdl2] results = {} print(f"Sending models {mdl1k} and {mdl2k} to API") thread1 = threading.Thread(target=predict_and_update_result, args=(text, mdl1k, results)) thread2 = threading.Thread(target=predict_and_update_result, args=(text, mdl2k, results)) thread1.start() thread2.start() thread1.join(180) thread2.join(180) #debug # print(results) # print(list(results.keys())[0]) # y, sr = librosa.load(results[list(results.keys())[0]], sr=None) # print(sr) # print(list(results.keys())[1]) # y, sr = librosa.load(results[list(results.keys())[1]], sr=None) # print(sr) #debug # outputs = [text, btn, r2, model1, model2, aud1, aud2, abetter, bbetter, prevmodel1, prevmodel2, nxtroundbtn] # cache the result for model in [mdl1k, mdl2k]: # skip caching if not hardcoded sentence if (text not in sents): break already_cached = False # check if already cached for cached_sample in cached_samples: # TODO:replace cached if (cached_sample.transcript == text and cached_sample.modelName == model): already_cached = True break if (already_cached): continue cached_samples.append(Sample(results[model], text, model)) # all_pairs = generate_matching_pairs(cached_samples) print(f"Retrieving models {mdl1k} and {mdl2k} from API") return ( text, "Synthesize", gr.update(visible=True), # r2 mdl1, # model1 mdl2, # model2 gr.update(visible=True, value=results[mdl1k], interactive=False, autoplay=True), # aud1 gr.update(visible=True, value=results[mdl2k], interactive=False, autoplay=False), # aud2 gr.update(visible=True, interactive=False), #abetter gr.update(visible=True, interactive=False), #bbetter gr.update(visible=False), #prevmodel1 gr.update(visible=False), #prevmodel2 gr.update(visible=False), #nxt round btn # reset gr.State aplayed & bplayed False, #aplayed False, #bplayed ) # return ( # text, # "Synthesize", # gr.update(visible=True), # r2 # mdl1, # model1 # mdl2, # model2 # # 'Vote to reveal model A', # prevmodel1 # gr.update(visible=True, value=router.predict( # text, # AVAILABLE_MODELS[mdl1], # api_name="/synthesize" # )), # aud1 # # 'Vote to reveal model B', # prevmodel2 # gr.update(visible=True, value=router.predict( # text, # AVAILABLE_MODELS[mdl2], # api_name="/synthesize" # )), # aud2 # gr.update(visible=True, interactive=True), # gr.update(visible=True, interactive=True), # gr.update(visible=False), # gr.update(visible=False), # gr.update(visible=False), #nxt round btn # ) def unlock_vote(btn_index, aplayed, bplayed): # sample played if btn_index == 0: aplayed = True if btn_index == 1: bplayed = True # both audio samples played if bool(aplayed) and bool(bplayed): # print('Both audio samples played, voting unlocked') return [gr.update(interactive=True), gr.update(interactive=True), True, True] return [gr.update(), gr.update(), aplayed, bplayed] def play_other(bplayed): return bplayed def get_userid(session_hash: str, request): # JS cookie if (session_hash != ''): # print('auth by session cookie') return sha1(bytes(session_hash.encode('ascii')), usedforsecurity=False).hexdigest() if request.username: # print('auth by username') # by HuggingFace username - requires `auth` to be enabled therefore denying access to anonymous users return sha1(bytes(request.username.encode('ascii')), usedforsecurity=False).hexdigest() else: # print('auth by ip') # by IP address - unreliable when gradio within HTML iframe # return sha1(bytes(request.client.host.encode('ascii')), usedforsecurity=False).hexdigest() # by browser session cookie - Gradio on HF is run in an HTML iframe, access to parent session required to reach session token # return sha1(bytes(request.headers.encode('ascii'))).hexdigest() # by browser session hash - Not a cookie, session hash changes on page reload return sha1(bytes(request.session_hash.encode('ascii')), usedforsecurity=False).hexdigest() # Give user a cached audio sample pair they have yet to vote on def give_cached_sample(session_hash: str, request: gr.Request): # add new userid to voting_users from Browser session hash # stored only in RAM userid = get_userid(session_hash, request) if userid not in voting_users: voting_users[userid] = User(userid) def get_next_pair(user: User): # FIXME: all_pairs var out of scope # all_pairs = generate_matching_pairs(cached_samples) # for pair in all_pairs: for pair in generate_matching_pairs(cached_samples): hash1 = md5(bytes((pair[0].modelName + pair[0].transcript).encode('ascii'))).hexdigest() hash2 = md5(bytes((pair[1].modelName + pair[1].transcript).encode('ascii'))).hexdigest() pair_key = (hash1, hash2) if ( pair_key not in user.voted_pairs # or in reversed order and (pair_key[1], pair_key[0]) not in user.voted_pairs ): return pair return None pair = get_next_pair(voting_users[userid]) if pair is None: return [ *clear_stuff(), # disable get cached sample button gr.update(interactive=False) ] return ( gr.update(visible=True, value=pair[0].transcript, elem_classes=['blurred-text']), "Synthesize", gr.update(visible=True), # r2 pair[0].modelName, # model1 pair[1].modelName, # model2 gr.update(visible=True, value=pair[0].filename, interactive=False, autoplay=True), # aud1 gr.update(visible=True, value=pair[1].filename, interactive=False, autoplay=False), # aud2 gr.update(visible=True, interactive=False), #abetter gr.update(visible=True, interactive=False), #bbetter gr.update(visible=False), #prevmodel1 gr.update(visible=False), #prevmodel2 gr.update(visible=False), #nxt round btn # reset aplayed, bplayed audio playback events False, #aplayed False, #bplayed # fetch cached btn gr.update(interactive=True) ) # note the vote on cached sample pair def voted_on_cached(modelName1: str, modelName2: str, transcript: str, session_hash: str, request: gr.Request): userid = get_userid(session_hash, request) # print(f'userid voted on cached: {userid}') if userid not in voting_users: voting_users[userid] = User(userid) hash1 = md5(bytes((modelName1 + transcript).encode('ascii'))).hexdigest() hash2 = md5(bytes((modelName2 + transcript).encode('ascii'))).hexdigest() voting_users[userid].voted_pairs.add((hash1, hash2)) def randomsent(): return '⚡', random.choice(sents), '🎲' def clear_stuff(): return [ gr.update(visible=True, value="", elem_classes=[]), "Synthesize", gr.update(visible=False), # r2 '', # model1 '', # model2 gr.update(visible=False, interactive=False, autoplay=False), # aud1 gr.update(visible=False, interactive=False, autoplay=False), # aud2 gr.update(visible=False, interactive=False), #abetter gr.update(visible=False, interactive=False), #bbetter gr.update(visible=False), #prevmodel1 gr.update(visible=False), #prevmodel2 gr.update(visible=False), #nxt round btn False, #aplayed False, #bplayed ] def disable(): return [gr.update(interactive=False), gr.update(interactive=False), gr.update(interactive=False), gr.update(interactive=False)] def enable(): return [gr.update(interactive=True), gr.update(interactive=True), gr.update(interactive=True), gr.update(interactive=True)] def unblur_text(): return gr.update(elem_classes=[]) unblur_js = 'document.getElementById("arena-text-input").classList.remove("blurred-text")' shortcut_js = """ """ with gr.Blocks() as vote: session_hash = gr.Textbox(visible=False, value='') # sample played, using Checkbox so that JS can fetch the value aplayed = gr.Checkbox(visible=False, value=False) bplayed = gr.Checkbox(visible=False, value=False) # voter ID useridstate = gr.State() gr.Markdown(INSTR) with gr.Group(): with gr.Row(): cachedt = gr.Button('⚡', scale=0, min_width=0, variant='tool', interactive=True) text = gr.Textbox( container=False, show_label=False, placeholder="Enter text to synthesize", lines=1, max_lines=1, scale=9999999, min_width=0, elem_id="arena-text-input", elem_classes="blurred-text", ) randomt = gr.Button('🎲', scale=0, min_width=0, variant='tool') randomt\ .click(randomsent, outputs=[cachedt, text, randomt])\ .then(None, js="() => "+ unblur_js) btn = gr.Button("Synthesize", variant='primary') model1 = gr.Textbox(interactive=False, lines=1, max_lines=1, visible=False) model2 = gr.Textbox(interactive=False, lines=1, max_lines=1, visible=False) with gr.Row(visible=False) as r2: with gr.Column(): with gr.Group(): aud1 = gr.Audio( interactive=False, show_label=False, show_download_button=False, show_share_button=False, waveform_options={'waveform_progress_color': '#EF4444'}, # var(--color-red-500)'}); gradio only accepts HEX and CSS color ) abetter = gr.Button( "A is better [a]", elem_id='arena-a-better', variant='primary', interactive=False, ) prevmodel1 = gr.HTML(show_label=False, value="Vote to reveal model A", visible=False) with gr.Column(): with gr.Group(): aud2 = gr.Audio( interactive=False, show_label=False, show_download_button=False, show_share_button=False, waveform_options={'waveform_progress_color': '#3C82F6'}, # var(--secondary-500)'}); gradio only accepts HEX and CSS color ) bbetter = gr.Button( "B is better [b]", elem_id='arena-b-better', variant='primary', interactive=False ) prevmodel2 = gr.HTML(show_label=False, value="Vote to reveal model B", visible=False) nxtroundbtn = gr.Button( '⚡ Next round [n]', elem_id='arena-next-round', visible=False ) # outputs = [text, btn, r2, model1, model2, prevmodel1, aud1, prevmodel2, aud2, abetter, bbetter] outputs = [ text, btn, r2, model1, model2, aud1, aud2, abetter, bbetter, prevmodel1, prevmodel2, nxtroundbtn, aplayed, bplayed, ] """ text, "Synthesize", gr.update(visible=True), # r2 mdl1, # model1 mdl2, # model2 gr.update(visible=True, value=results[mdl1]), # aud1 gr.update(visible=True, value=results[mdl2]), # aud2 gr.update(visible=True, interactive=False), #abetter gr.update(visible=True, interactive=False), #bbetter gr.update(visible=False), #prevmodel1 gr.update(visible=False), #prevmodel2 gr.update(visible=False), #nxt round btn""" btn\ .click(disable, outputs=[btn, abetter, bbetter, cachedt])\ .then(synthandreturn, inputs=[text], outputs=outputs)\ .then(enable, outputs=[btn, gr.State(), gr.State(), cachedt])\ .then(None, js="() => "+ unblur_js) nxtroundbtn\ .click(disable, outputs=[btn, abetter, bbetter, cachedt])\ .then(give_cached_sample, inputs=[session_hash], outputs=[*outputs, cachedt])\ .then(enable, outputs=[btn, gr.State(), gr.State(), gr.State()]) # fetch a comparison pair from cache cachedt\ .click(disable, outputs=[btn, abetter, bbetter, cachedt])\ .then(give_cached_sample, inputs=[session_hash], outputs=[*outputs, cachedt])\ .then(enable, outputs=[btn, gr.State(), gr.State(), gr.State()]) # TODO: await download of sample before allowing playback # Allow interaction with the vote buttons only when both audio samples have finished playing aud1\ .stop( unlock_vote, outputs=[abetter, bbetter, aplayed, bplayed], inputs=[gr.State(value=0), aplayed, bplayed], )\ .then( None, inputs=[bplayed], js="(b) => b ? 0 : document.querySelector('.stretch .gap+.gap button.play-pause-button').click()", ) # autoplay if unplayed aud2\ .stop( unlock_vote, outputs=[abetter, bbetter, aplayed, bplayed], inputs=[gr.State(value=1), aplayed, bplayed], )\ .then(None, js="() => "+ unblur_js) # aud1.input( # None, # js="() => {console.log(new Date().getTime().toString()+'input')}", # ) # aud1.change( # None, # js="() => {console.log(new Date().getTime().toString()+'change')}", # ) # nxt_outputs = [prevmodel1, prevmodel2, abetter, bbetter] nxt_outputs = [abetter, bbetter, prevmodel1, prevmodel2, nxtroundbtn] abetter\ .click(a_is_better, outputs=nxt_outputs, inputs=[model1, model2, useridstate])\ .then(voted_on_cached, inputs=[model1, model2, text, session_hash]) bbetter\ .click(b_is_better, outputs=nxt_outputs, inputs=[model1, model2, useridstate])\ .then(voted_on_cached, inputs=[model1, model2, text, session_hash]) # skipbtn.click(b_is_better, outputs=outputs, inputs=[model1, model2, useridstate]) # bothbad.click(both_bad, outputs=outputs, inputs=[model1, model2, useridstate]) # bothgood.click(both_good, outputs=outputs, inputs=[model1, model2, useridstate]) # get session cookie vote\ .load( None, None, session_hash, js="() => { return getArenaCookie('session') }", ) # give a cached sample pair to voter; .then() did not work here vote\ .load(give_cached_sample, inputs=[session_hash], outputs=[*outputs, cachedt]) with gr.Blocks() as about: gr.Markdown(ABOUT) # with gr.Blocks() as admin: # rdb = gr.Button("Reload Audio Dataset") # # rdb.click(reload_audio_dataset, outputs=rdb) # with gr.Group(): # dbtext = gr.Textbox(label="Type \"delete db\" to confirm", placeholder="delete db") # ddb = gr.Button("Delete DB") # ddb.click(del_db, inputs=dbtext, outputs=ddb) # Blur cached sample text so the voting user picks up mispronouncements with gr.Blocks(theme=theme, css="footer {visibility: hidden}textbox{resize:none} .blurred-text {filter: blur(0.15em);}", head=shortcut_js, js="cookie.js", title="TTS Arena") as demo: gr.Markdown(DESCR) # gr.TabbedInterface([vote, leaderboard, about, admin], ['Vote', 'Leaderboard', 'About', 'Admin (ONLY IN BETA)']) gr.TabbedInterface([vote, leaderboard, about], ['🗳️ Vote', '🏆 Leaderboard', '📄 About']) if CITATION_TEXT: with gr.Row(): with gr.Accordion("Citation", open=False): gr.Markdown(f"If you use this data in your publication, please cite us!\n\nCopy the BibTeX citation to cite this source:\n\n```bibtext\n{CITATION_TEXT}\n```\n\nPlease remember that all generated audio clips should be assumed unsuitable for redistribution or commercial use.") demo\ .queue(api_open=False, default_concurrency_limit=40)\ .launch(show_api=False, show_error=True)