<jupyter_start><jupyter_text>EMISSOR chat bot with audio backend  UNDER CONSTRUCTION   Running with ASR<jupyter_code>import requests
import numpy as np

%matplotlib inline
import matplotlib.pyplot as plt

from cltl.asr.speechbrain_asr import SpeechbrainASR
from cltl.asr.wav2vec_asr import Wav2Vec2ASR
from cltl.backend.api.camera import CameraResolution 
from cltl.backend.source.pyaudio_source import PyAudioSource
from cltl.backend.source.cv2_source import SystemImageSource
from cltl.backend.source.client_source import ClientAudioSource, ClientImageSource
from cltl.vad.webrtc_vad import WebRtcVAD

from cltl.backend.api.util import raw_frames_to_np
run_remote = False

audio_source = ClientAudioSource("http://192.168.1.176:8000/audio") if run_remote else PyAudioSource(16000, 1, 480)
image_source = ClientImageSource("http://192.168.1.176:8000/video") if run_remote else SystemImageSource(CameraResolution.QVGA)


# Add a `storage` directory to store audio files to VAD or ASR for debugging
vad = WebRtcVAD(allow_gap=250, padding=5)
asr=Wav2Vec2ASR("jonatasgrosman/wav2vec2-large-xlsr-53-english", 16000)
# asr = SpeechbrainASR(model_id="speechbrain/asr-transformer-transformerlm-librispeech", sampling_rate=16000, storage="wav")

    
def detect_and_transcribe():
    with audio_source as audio:
        frames = raw_frames_to_np(audio, audio_source.frame_size, audio_source.channels, audio_source.depth)

        print("Listening")
        speech, offset, consumed = tuple(vad.detect_vad(frames, audio_source.rate))
        print("Voice activity detected")

        with image_source as img_src:
            image = img_src.capture()

        text = asr.speech_to_text(np.concatenate(tuple(speech)), audio_source.rate)

        return text, image
for _ in range(5):
    print("Please talk to calibrate Voice Activity Detection")
    try:
        detect_and_transcribe()
    except Exception as e:
        print("Failed", e)

print("Calibrated VAD")
tts_headers = {'Content-type': 'text/plain'}

while True:
    try:
        text, image = detect_and_transcribe()
        print("Detected text:", text)
        
        print("Captured image:", image.image.shape)
        plt.imshow(image.image)
        plt.show()
        
        if run_remote:
            requests.post("http://192.168.1.176:8000/text", data=f"Did you say {text}?", headers=tts_headers)
    except Exception as e:
        print("Failed", e)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09771054783319706, 'fraction_numerical': 0.03107113654946852, 'mean_word_length': 4.162447257383966, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 3, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '223412', 'license': 'Apache-2.0', 'n_tokens_mistral': 865, 'n_tokens_neox': 790, 'n_words': 177, 'path': 'src/notebooks/lets-chat_with_backend_old.ipynb', 'repo_name': 'leolani/cltl-chatbots', 'size': '2446'}