Audio question and audio response

app.py

@@ -1,26 +1,27 @@
 import streamlit as st
-from transformers import GPT2Tokenizer, GPT2LMHeadModel
-from langchain.prompts import PromptTemplate
+import os
+from generate_answer import generate_answer
+from audio_to_text import audio_to_text
+from text_to_audio import text_to_audio
 
-tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
-model = GPT2LMHeadModel.from_pretrained('gpt2')
+file_path_relative = 'SDLC.wav'
+file_path_abs = os.path.abspath(file_path_relative)
 
 st.title("GPT 2 Chat Bot?")
-input_text = st.text_area("Enter text to classify:")
-
 
+input_text = st.text_area("Search query:")
 if st.button("""Enter >>>>> """):
     if input_text:
-        prompt_template = PromptTemplate(template="Answer the following question and classify it: {question}",
-                                         input_variables=["question"], output_variables=["answer", "classification"])
-        # Model loading
-        format_prompt = prompt_template.format(question=input_text)
-        encoded_input = tokenizer(format_prompt, return_tensors='pt')
-        # Run the model
-        output = model.generate(**encoded_input, max_length=100)  # Use generate method for text generation
-        # Decode the model output to text
-        decoded_output = tokenizer.decode(output[0])
-        response_text = decoded_output.split('\n\n')
-        st.write(response_text[1])
+        response_text = generate_answer(input_text)
+        st.write(response_text)
+
+st.audio(data=file_path_abs, format='audio/wav')
+if st.button("""Use Audio question """):
+    audio_text = audio_to_text(file_path_abs)
+    answer = generate_answer(audio_text)
+    audio_bytes = text_to_audio(answer)
+    st.audio(audio_bytes, format="audio/wav")
+    st.write(answer)
+
 
 

audio_to_text.py

@@ -0,0 +1,23 @@
+from transformers import WhisperProcessor, WhisperForConditionalGeneration
+import torchaudio
+
+
+# load model and processor
+processor = WhisperProcessor.from_pretrained("openai/whisper-tiny")
+model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-tiny")
+model.config.forced_decoder_ids = None
+
+
+def audio_to_text(file_path_abs):
+    # Load the audio and resample it
+    waveform, sample_rate = torchaudio.load(file_path_abs)
+    resampler = torchaudio.transforms.Resample(orig_freq=sample_rate, new_freq=16000)
+    waveform = resampler(waveform)
+    waveform = waveform.squeeze().numpy()
+    input_features = processor(waveform, sampling_rate=16000, return_tensors="pt").input_features
+
+    # generate token ids
+    predicted_ids = model.generate(input_features)
+    # decode token ids to text
+    transcription = processor.batch_decode(predicted_ids, skip_special_tokens=True)
+    return transcription

generate_answer.py

@@ -0,0 +1,19 @@
+from transformers import GPT2Tokenizer, GPT2LMHeadModel
+from langchain.prompts import PromptTemplate
+
+tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
+model = GPT2LMHeadModel.from_pretrained('gpt2')
+
+
+def generate_answer(question):
+    prompt_template = PromptTemplate(template="Answer the following question within 100 words: {question}",
+                                     input_variables=["question"], output_variables=["answer"])
+    # Model loading
+    format_prompt = prompt_template.format(question=question)
+    encoded_input = tokenizer(format_prompt, return_tensors='pt')
+    # Run the model
+    output = model.generate(**encoded_input, max_length=100)  # Use generate method for text generation
+    # Decode the model output to text
+    decoded_output = tokenizer.decode(output[0])
+    response_text = decoded_output
+    return response_text

requirements.txt

@@ -4,4 +4,7 @@ langchain-community==0.2.5
 python-multipart==0.0.9
 transformers==4.41.2
 torch==2.3.1
-tensorflow==2.16.2
+tensorflow==2.16.2
+sentencepiece
+soundfile
+datasets

text_to_audio.py

@@ -0,0 +1,22 @@
+from transformers import pipeline
+from datasets import load_dataset
+import soundfile as sf
+import torch
+
+synthesiser = pipeline("text-to-speech", "microsoft/speecht5_tts")
+
+
+def text_to_audio(text):
+    # clean the response and max_size is 600
+    text_clean = text.replace('\n', '').replace('*', '')
+    text_550 = text_clean[:590]
+    # get speaker embeddings
+    embeddings_dataset = load_dataset("Matthijs/cmu-arctic-xvectors", split="validation")
+    speaker_embedding = torch.tensor(embeddings_dataset[7306]["xvector"]).unsqueeze(0)
+    # You can replace this embedding with your own as well.
+
+    speech = synthesiser(text_550, forward_params={"speaker_embeddings": speaker_embedding})
+    sf.write("output.wav", speech["audio"], samplerate=speech["sampling_rate"])
+    audio_file = open("output.wav", "rb")
+    audio_bytes = audio_file.read()
+    return audio_bytes