array input audio

app.py

@@ -6,6 +6,7 @@ import google.generativeai as genai
 import re
 import torch
 from transformers import pipeline
+from transformers.pipelines.audio_utils import ffmpeg_read
 import time
 import spaces
 
@@ -54,14 +55,14 @@ def summarize_transcription(transcription, model, gemini_prompt):
         return f"Error summarizing transcription: {str(e)}"
     
 @spaces.GPU(duration=120)
-def process_audio(audio_file, pipe, language):
+def process_audio(inputs, pipe, language):
     print("Starting transcription...")
     if language:
         print(f"Using language: {language}")
-        transcription = pipe(f"{audio_file}", batch_size=8, generate_kwargs={"task": "transcribe", "language": language}, return_timestamps=True)["text"]
+        transcription = pipe(inputs, batch_size=8, generate_kwargs={"task": "transcribe", "language": language}, return_timestamps=True)["text"]
     else:
         print("No language defined, using default language")
-        transcription = pipe(f"{audio_file}", batch_size=8, generate_kwargs={"task": "transcribe"}, return_timestamps=True)["text"]
+        transcription = pipe(inputs, batch_size=8, generate_kwargs={"task": "transcribe"}, return_timestamps=True)["text"]
     return transcription
 
 def transcribe(youtube_url, audio_file, whisper_model, gemini_api_key, gemini_prompt, gemini_model_variant, language, progress=gr.Progress()):
@@ -72,15 +73,13 @@ def transcribe(youtube_url, audio_file, whisper_model, gemini_api_key, gemini_pr
             gemini_api_key = default_gemini_api_key
         model = configure_genai(gemini_api_key, gemini_model_variant)
 
-        # device = 0 if torch.cuda.is_available() else "cpu"
-        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        device = 0 if torch.cuda.is_available() else "cpu"
         pipe = pipeline(
             task="automatic-speech-recognition",
             model=whisper_model,
             chunk_length_s=30,
             device=device,
         )
-        pipe.model = pipe.model.to(device)
 
         if youtube_url:
             progress(0.1, desc="Extracting YouTube ID")
@@ -97,9 +96,15 @@ def transcribe(youtube_url, audio_file, whisper_model, gemini_api_key, gemini_pr
             progress(0.2, desc="Reading audio file")
             audio_file = f"{audio_file.name}"
             print(f"Audio file read: {audio_file}")
+
+        with open(audio_file, "rb") as f:
+            inputs = f.read()
+
+        inputs = ffmpeg_read(inputs, pipe.feature_extractor.sampling_rate)
+        inputs = {"array": inputs, "sampling_rate": pipe.feature_extractor.sampling_rate}
         
         progress(0.4, desc="Starting transcription")
-        transcription = process_audio(audio_file, pipe, language)
+        transcription = process_audio(inputs, pipe, language)
         
         progress(0.6, desc="Cleaning up")
         # Delete the audio file after transcription

packages.txt

@@ -0,0 +1 @@
+ffmpeg