Spaces:
Sleeping
Sleeping
| from typing import List | |
| import av | |
| import asyncio | |
| from collections import deque | |
| import threading | |
| import cv2 | |
| import numpy as np | |
| import ray | |
| from ray.util.queue import Queue | |
| from app_interface_actor import AppInterfaceActor | |
| import pydub | |
| import torch | |
| class StreamlitAVQueue: | |
| def __init__(self, audio_bit_rate=16000): | |
| self._output_channels = 2 | |
| self._audio_bit_rate = audio_bit_rate | |
| self._listening = True | |
| self._looking = False | |
| self._lock = threading.Lock() | |
| self.app_interface_actor = AppInterfaceActor.get_singleton() | |
| self._video_output_frame = None | |
| def set_looking_listening(self, looking, listening: bool): | |
| with self._lock: | |
| self._looking = looking | |
| self._listening = listening | |
| async def queued_video_frames_callback( | |
| self, | |
| frames: List[av.VideoFrame], | |
| ) -> av.VideoFrame: | |
| updated_frames = [] | |
| try: | |
| with self._lock: | |
| should_look = self._looking | |
| next_video_output_frame = await self.app_interface_actor.get_video_output_frame.remote() | |
| if next_video_output_frame is not None: | |
| self._video_output_frame = next_video_output_frame | |
| for i, frame in enumerate(frames): | |
| user_image = frame.to_ndarray(format="rgb24") | |
| if should_look: | |
| shared_tensor_ref = ray.put(user_image) | |
| await self.app_interface_actor.enqueue_video_input_frame.remote(shared_tensor_ref) | |
| if self._video_output_frame is not None: | |
| frame = self._video_output_frame | |
| # resize user image to 1/4 size | |
| user_frame = cv2.resize(user_image, (user_image.shape[1]//4, user_image.shape[0]//4), interpolation=cv2.INTER_AREA) | |
| # flip horizontally | |
| user_frame = cv2.flip(user_frame, 1) | |
| x_user = 0 | |
| y_user = frame.shape[0] - user_frame.shape[0] | |
| final_frame = frame.copy() | |
| final_frame[y_user:y_user+user_frame.shape[0], x_user:x_user+user_frame.shape[1]] = user_frame | |
| frame = av.VideoFrame.from_ndarray(final_frame, format="rgb24") | |
| updated_frames.append(frame) | |
| # print (f"tesnor len: {len(shared_tensor)}, tensor shape: {shared_tensor.shape}, tensor type:{shared_tensor.dtype} tensor ref: {shared_tensor_ref}") | |
| except Exception as e: | |
| print (e) | |
| return updated_frames | |
| async def queued_audio_frames_callback( | |
| self, | |
| frames: List[av.AudioFrame], | |
| ) -> av.AudioFrame: | |
| try: | |
| with self._lock: | |
| should_listed = self._listening | |
| sound_chunk = pydub.AudioSegment.empty() | |
| if len(frames) > 0 and should_listed: | |
| for frame in frames: | |
| sound = pydub.AudioSegment( | |
| data=frame.to_ndarray().tobytes(), | |
| sample_width=frame.format.bytes, | |
| frame_rate=frame.sample_rate, | |
| channels=len(frame.layout.channels), | |
| ) | |
| sound = sound.set_channels(1) | |
| sound = sound.set_frame_rate(self._audio_bit_rate) | |
| sound_chunk += sound | |
| shared_buffer = np.array(sound_chunk.get_array_of_samples()) | |
| shared_buffer_ref = ray.put(shared_buffer) | |
| await self.app_interface_actor.enqueue_audio_input_frame.remote(shared_buffer_ref) | |
| except Exception as e: | |
| print (e) | |
| # return empty frames to avoid echo | |
| new_frames = [] | |
| try: | |
| for frame in frames: | |
| required_samples = frame.samples | |
| # print (f"frame: {frame.format.name}, {frame.layout.name}, {frame.sample_rate}, {frame.samples}") | |
| assert frame.format.bytes == 2 | |
| assert frame.format.name == 's16' | |
| frame_as_bytes = await self.app_interface_actor.get_audio_output_frame.remote() | |
| if frame_as_bytes: | |
| # print(f"frame_as_bytes: {len(frame_as_bytes)}") | |
| assert len(frame_as_bytes) == frame.samples * frame.format.bytes | |
| samples = np.frombuffer(frame_as_bytes, dtype=np.int16) | |
| else: | |
| samples = np.zeros((required_samples * 2 * 1), dtype=np.int16) | |
| if self._output_channels == 2: | |
| samples = np.vstack((samples, samples)).reshape((-1,), order='F') | |
| samples = samples.reshape(1, -1) | |
| layout = 'stereo' if self._output_channels == 2 else 'mono' | |
| new_frame = av.AudioFrame.from_ndarray(samples, format='s16', layout=layout) | |
| new_frame.sample_rate = frame.sample_rate | |
| new_frames.append(new_frame) | |
| except Exception as e: | |
| print (e) | |
| return new_frames | |
| async def get_audio_input_frames_async(self) -> List[av.AudioFrame]: | |
| shared_buffers = await self.app_interface_actor.get_audio_input_frames.remote() | |
| return shared_buffers | |
| async def get_video_frames_async(self) -> List[av.AudioFrame]: | |
| shared_tensors = await self.app_interface_actor.get_video_input_frames.remote() | |
| return shared_tensors | |
| def get_audio_output_queue(self)->Queue: | |
| return self.app_interface_actor.get_audio_output_queue.remote() | |
| def get_video_output_queue(self)->Queue: | |
| return self.app_interface_actor.get_video_output_queue.remote() | |