Spaces:
Sleeping
Sleeping
File size: 4,195 Bytes
9740bc5 edce499 da9d44b edce499 c6ad8e3 edce499 e2846c4 da9d44b edce499 da9d44b 9740bc5 da9d44b 9740bc5 da9d44b 5a17040 e2846c4 9740bc5 e2846c4 5a17040 149eeaf 9740bc5 e2846c4 5a17040 e2846c4 5a17040 9740bc5 da9d44b 9740bc5 da9d44b e2846c4 149eeaf da9d44b 162d5c8 da9d44b 6130167 149eeaf da9d44b 162d5c8 9740bc5 da9d44b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 |
import asyncio
from collections import deque
import os
import threading
import time
import traceback
import av
import numpy as np
import streamlit as st
from streamlit_webrtc import WebRtcMode, webrtc_streamer
import pydub
import torch
# import av
# import cv2
from sample_utils.turn import get_ice_servers
import json
from typing import List
st.set_page_config(layout="wide")
from vosk import SetLogLevel, Model, KaldiRecognizer
SetLogLevel(-1) # mutes vosk verbosity
from dotenv import load_dotenv
load_dotenv()
webrtc_ctx = None
# Initialize Ray
import ray
if not ray.is_initialized():
# Try to connect to a running Ray cluster
ray_address = os.getenv('RAY_ADDRESS')
if ray_address:
ray.init(ray_address, namespace="project_charles")
else:
ray.init(namespace="project_charles")
async def main():
st.title("Project Charles")
col1, col2 = st.columns(2)
with col1:
nested_col1, nested_col2 = st.columns(2)
with nested_col1:
listening = st.checkbox("Listen", value=True)
with nested_col2:
looking = st.checkbox("Look", value=False)
charles_actor_debug_output = st.empty()
environment_state_ouput = st.empty()
with col2:
if "streamlit_av_queue" not in st.session_state:
from streamlit_av_queue import StreamlitAVQueue
st.session_state.streamlit_av_queue = StreamlitAVQueue()
playing = st.checkbox("Playing", value=True)
webrtc_ctx = webrtc_streamer(
key="charles",
desired_playing_state=playing,
queued_audio_frames_callback=st.session_state.streamlit_av_queue.queued_audio_frames_callback,
queued_video_frames_callback=st.session_state.streamlit_av_queue.queued_video_frames_callback,
mode=WebRtcMode.SENDRECV,
media_stream_constraints={
"video": True,
"audio": {
"sampleRate": 48000,
"sampleSize": 16,
"noiseSuppression": True,
"echoCancellation": True,
"channelCount": 1,
}
},
rtc_configuration={"iceServers": get_ice_servers()},
async_processing=True,
)
system_one_audio_status = st.markdown("Initializing streaming")
if not webrtc_ctx.state.playing:
exit
from charles_actor import CharlesActor
charles_actor = None
try:
while True:
if "streamlit_av_queue" in st.session_state:
st.session_state.streamlit_av_queue.set_looking_listening(looking, listening)
if not webrtc_ctx.state.playing:
system_one_audio_status.write("Stopped.")
await asyncio.sleep(0.1)
continue
if charles_actor is None:
try:
charles_actor = ray.get_actor("CharlesActor")
system_one_audio_status.write("Charles is here.")
except ValueError as e:
system_one_audio_status.write("Charles is sleeping.")
pass
if charles_actor is not None:
try:
# new_environment_state = await charles_actor.get_environment_state.remote()
# environment_state_ouput.markdown(f"{new_environment_state}")
charles_debug_str = await charles_actor.get_charles_actor_debug_output.remote()
charles_actor_debug_output.markdown(charles_debug_str)
except Exception as e:
# assume we disconnected
charles_actor = None
await asyncio.sleep(0.1)
except Exception as e:
print(f"An error occurred: {e}")
traceback.print_exc()
raise e
if __name__ == "__main__":
try:
asyncio.run(main())
except Exception as e:
if webrtc_ctx is not None:
del webrtc_ctx
webrtc_ctx = None
if "streamlit_av_queue" in st.session_state:
del st.session_state.streamlit_av_queue
finally:
pass
|