streaming speech in the debug

audio_stream_processor.py

@@ -0,0 +1,42 @@
+import subprocess
+from threading import Thread
+from queue import Queue
+from typing import Iterator
+
+class AudioStreamProcessor:
+    def __init__(self):
+        self.queue = Queue()
+        self.thread = Thread(target=self._process_audio_streams)
+        self.thread.start()
+
+    def add_audio_stream(self, audio_stream: Iterator[bytes]):
+        self.queue.put(audio_stream)
+
+    def _process_audio_streams(self):
+        while True:
+            audio_stream = self.queue.get()
+            if audio_stream is None:  # We'll use None as a sentinel to mark the end
+                break
+            self._stream(audio_stream)
+
+    def _stream(self, audio_stream: Iterator[bytes]):
+        mpv_command = ["mpv", "--no-cache", "--no-terminal", "--", "fd://0"]
+        mpv_process = subprocess.Popen(
+            mpv_command,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+        )
+
+        for chunk in audio_stream:
+            if chunk is not None:
+                mpv_process.stdin.write(chunk)
+                mpv_process.stdin.flush()
+
+        if mpv_process.stdin:
+            mpv_process.stdin.close()
+        mpv_process.wait()
+
+    def close(self):
+        self.queue.put(None)  # Signal the processing thread to terminate
+        self.thread.join()

chat_service.py

@@ -55,7 +55,7 @@ class ChatService:
                 max_new_tokens=200,
                 do_sample=True,
                 top_k=40,
-                temperature=1.0,
+                temperature=1.0, # use 1.0 for debugging/deteministic results
                 pad_token_id=self._tokenizer.eos_token_id,
             )
             agent_response = self._tokenizer.decode(outputs[0], truncate_before_pattern=[r"\n\n^#", "^'''", "\n\n\n"])

debug.py

@@ -2,34 +2,47 @@ from clip_transform import CLIPTransform
 from chat_service import ChatService
 from dotenv import load_dotenv
 from speech_service import SpeechService
+from concurrent.futures import ThreadPoolExecutor
+from audio_stream_processor import AudioStreamProcessor
 
-load_dotenv()
 
-# print ("Initializing CLIP templates")
-# clip_transform = CLIPTransform()
-# print ("CLIP success")
+def run_debug_code():
+    load_dotenv()
 
-print ("Initializing Chat")
-chat_service = ChatService()
+    # print ("Initializing CLIP templates")
+    # clip_transform = CLIPTransform()
+    # print ("CLIP success")
 
-user_speech_service = SpeechService(voice_id="Adam")
-ai_speech_service = SpeechService(voice_id="2OviOUQc1JsQRQgNkVBj") # Chales003
+    print ("Initializing Chat")
+    chat_service = ChatService()
 
-user_speech_service.print_voices()
+    user_speech_service = SpeechService(voice_id="Adam")
+    ai_speech_service = SpeechService(voice_id="2OviOUQc1JsQRQgNkVBj") # Chales003
+    processor = AudioStreamProcessor()
 
+    # user_speech_service.print_voices() # if you want to see your custom voices
 
-prompts = [
-    "hello, how are you today?",
-    "tell me about your shadow self?",
-    "hmm, interesting, tell me more about that.",
-    "wait, that is so interesting, what else?",
-]
-for prompt in prompts:
-    print (f'prompt: "{prompt}"')
-    user_speech_service.speak(prompt)
-    response = chat_service.chat(prompt)
-    print (f'response: "{response}"')
-    ai_speech_service.speak(response)
+    prompts = [
+        "hello, how are you today?",
+        "tell me about your shadow self?",
+        "hmm, interesting, tell me more about that.",
+        "wait, that is so interesting, what else?",
+    ]
+    for prompt in prompts:
+        print ("")
+        print (f'prompt: "{prompt}"')
+        stream = user_speech_service.stream(prompt)
+        processor.add_audio_stream(stream)
 
+        response = chat_service.chat(prompt)
+        print ("")
+        print (f'response: "{response}"')
+        stream = ai_speech_service.stream(response)
+        processor.add_audio_stream(stream)
 
-print ("Chat success")
+    processor.close()
+    print ("Chat success")
+
+
+if __name__ == '__main__':
+    run_debug_code()

speech_service.py

@@ -46,3 +46,12 @@ class SpeechService:
         play(audio)
         return
     
+    def stream(self, prompt):
+        audio_stream = generate(
+            text=prompt,
+            voice=self._voice_id,
+            model=self._model_id,
+            stream=True
+            )
+        return audio_stream
+