import os os.system("pip install git+https://github.com/openai/whisper.git") import whisper import evaluate from evaluate.utils import launch_gradio_widget import gradio as gr import torch import classify from whisper.model import Whisper from whisper.tokenizer import get_tokenizer from speechbrain.pretrained.interfaces import foreign_class from transformers import AutoModelForSequenceClassification, pipeline, WhisperTokenizer, RobertaForSequenceClassification, RobertaTokenizer, AutoTokenizer # pull in emotion detection # --- Add element for specification # pull in text classification # --- Add custom labels # --- Associate labels with radio elements # add logic to initiate mock notificaiton when detected # pull in misophonia-specific model model_cache = {} # Building prediction function for gradio emo_dict = { 'sad': 'Sad', 'hap': 'Happy', 'ang': 'Anger', 'neu': 'Neutral' } # static classes for now, but it would be best ot have the user select from multiple, and to enter their own class_options = { "racism": ["racism", "hate speech", "bigotry", "racially targeted", "racially diminutive", "racial slur", "ethnic slur", "ethnic hate", "pro-white nationalism"], "LGBTQ+ hate": ["gay slur", "trans slur", "homophobic slur", "transphobia", "anti-LBGTQ+", "hate speech"], "sexually explicit": ["sexually explicit", "sexually coercive", "sexual exploitation", "vulgar", "raunchy", "sexually demeaning", "sexual violence", "victim blaming"], "misophonia": ["chewing", "breathing", "mouthsounds", "popping", "sneezing", "yawning", "smacking", "sniffling", "panting"] } pipe = pipeline("automatic-speech-recognition", model="openai/whisper-large") # Create a Gradio interface with audio file and text inputs def classify_toxicity(audio_file, text_input, classify_anxiety): # Transcribe the audio file using Whisper ASR if audio_file != None: transcribed_text = pipe(audio_file)["text"] #### Emotion classification #### emotion_classifier = foreign_class(source="speechbrain/emotion-recognition-wav2vec2-IEMOCAP", pymodule_file="custom_interface.py", classname="CustomEncoderWav2vec2Classifier") out_prob, score, index, text_lab = emotion_classifier.classify_file(audio_file) else: transcribed_text = text_input if classify_anxiety != "misophonia": #### Toxicity Classifier #### toxicity_module = evaluate.load("toxicity", "facebook/roberta-hate-speech-dynabench-r4-target") #toxicity_module = evaluate.load("toxicity", 'DaNLP/da-electra-hatespeech-detection', module_type="measurement") toxicity_results = toxicity_module.compute(predictions=[transcribed_text]) toxicity_score = toxicity_results["toxicity"][0] print(toxicity_score) #### Text classification ##### device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu") text_classifier = pipeline("zero-shot-classification", model="facebook/bart-large-mnli") sequence_to_classify = transcribed_text print(classify_anxiety, class_options) candidate_labels = class_options.get(classify_anxiety, []) # classification_output = classifier(sequence_to_classify, candidate_labels, multi_label=False) classification_output = text_classifier(sequence_to_classify, candidate_labels, multi_label=True) print(classification_output) #### Emotion classification #### emotion_classifier = foreign_class(source="speechbrain/emotion-recognition-wav2vec2-IEMOCAP", pymodule_file="custom_interface.py", classname="CustomEncoderWav2vec2Classifier") out_prob, score, index, text_lab = emotion_classifier.classify_file(audio_file) return toxicity_score, classification_output, emo_dict[text_lab[0]], transcribed_text # return f"Toxicity Score ({available_models[selected_model]}): {toxicity_score:.4f}" else: model = whisper.load_model("large") model_cache[model_name] = model # model = model_cache[model_name] # class_names = classify_anxiety.split(",") class_names_list = class_options.get(classify_anxiety, []) class_str = "" for elm in class_names_list: class_str += elm + "," #class_names = class_names_temp.split(",") class_names = class_str.split(",") print("class names ", class_names, "classify_anxiety ", classify_anxiety) # tokenizer = get_tokenizer(multilingual=".en" not in model_name) tokenizer= WhisperTokenizer.from_pretrained("openai/whisper-large") model = "whisper-large" internal_lm_average_logprobs = classify.calculate_internal_lm_average_logprobs( model=model, class_names=class_names, # class_names=classify_anxiety, tokenizer=tokenizer, ) audio_features = classify.calculate_audio_features(audio_path, model) average_logprobs = classify.calculate_average_logprobs( model=model, audio_features=audio_features, class_names=class_names, tokenizer=tokenizer, ) average_logprobs -= internal_lm_average_logprobs scores = average_logprobs.softmax(-1).tolist() return {class_name: score for class_name, score in zip(class_names, scores)} return classify_anxiety with gr.Blocks() as iface: with gr.Column(): anxiety_class = gr.Radio(["racism", "LGBTQ+ hate", "sexually explicit", "misophonia"]) with gr.Column(): aud_input = gr.Audio(source="upload", type="filepath", label="Upload Audio File") text = gr.Textbox(label="Enter Text", placeholder="Enter text here...") submit_btn = gr.Button(label="Run") with gr.Column(): out_text = gr.Textbox() submit_btn.click(fn=classify_toxicity, inputs=[aud_input, text, anxiety_class], outputs=out_text) iface.launch()