updates

.gitignore

@@ -0,0 +1,2 @@
+.vscode/settings.json
+__pycache__/

.vscode/settings.json

@@ -1,26 +0,0 @@
-{
-  "[python]": {
-    "editor.defaultFormatter": "ms-python.python"
-  },
-  "python.formatting.provider": "none",
-  "workbench.colorCustomizations": {
-    "activityBar.activeBackground": "#fbed80",
-    "activityBar.background": "#fbed80",
-    "activityBar.foreground": "#15202b",
-    "activityBar.inactiveForeground": "#15202b99",
-    "activityBarBadge.background": "#06b9a5",
-    "activityBarBadge.foreground": "#15202b",
-    "commandCenter.border": "#15202b99",
-    "sash.hoverBorder": "#fbed80",
-    "statusBar.background": "#f9e64f",
-    "statusBar.foreground": "#15202b",
-    "statusBarItem.hoverBackground": "#f7df1e",
-    "statusBarItem.remoteBackground": "#f9e64f",
-    "statusBarItem.remoteForeground": "#15202b",
-    "titleBar.activeBackground": "#f9e64f",
-    "titleBar.activeForeground": "#15202b",
-    "titleBar.inactiveBackground": "#f9e64f99",
-    "titleBar.inactiveForeground": "#15202b99"
-  },
-  "peacock.remoteColor": "#f9e64f"
-}

seamless_server/.gitignore

@@ -0,0 +1,4 @@
+__pycache__/
+src/__pycache__/
+debug/
+.vscode/

seamless_server/README.md

@@ -0,0 +1,72 @@
+# Seamless Streaming Server
+
+## Setting up the AWS server
+
+- Clone working AMI with things set up: https://us-west-2.console.aws.amazon.com/ec2/home?region=us-west-2#ImageDetails:imageId=ami-013fc3fb27faa03e5
+- This has the conda environment, repository, model etc set up so minimal install was required
+
+- Open port tunnelling via ssh so you can connect to the AWS server via localhost (gets around some security issues): e.g. `ssh -L 8000:localhost:8000 -L 3000:localhost:3000 -L 5173:localhost:5173 -i ~/.ssh/mduppes_devserver.pem  [email protected]`
+
+## Setting up the dev environment
+
+Clone the repo
+
+`cd seamless-experiences/seamless_vc/seamless_server`
+
+If running for the first time, create conda environment from the environment.yaml `conda env create -f environment.yml`
+(or if you are on Mac OS, replace `environment.yml` with `environment_mac.yml`)
+
+In each new terminal you use you will need to activate the conda environment:
+`conda activate smlss_server`
+
+Install dependencies with pip:
+`pip install -r requirements.txt`
+
+Install Meta-specific dependencies with the dedicated install script:
+`./scripts/update_meta_dependencies.sh`
+
+If needed, download and extract the latest model using the dedicated script:
+`./scripts/download_latest_models.sh`
+
+## Setting up the docker environment
+
+Alternatively we can build a docker image (for ease of deployment).
+
+1. Make sure github keys are loaded into ssh-agent. Copy over the keys into ~/.ssh/ then run `eval $(ssh-agent)`, then `ssh-add` to load the keys.
+2. Run `docker-compose build` if you just want to build the image. If you want to also run the server: `docker compose up --build`.
+3. Optionally to push the built docker image to ECR so it could be pulled into deployments: `./deploy.sh` 
+
+NOTE: Add your models to the ./models directory.
+
+# Running the v2 ("pubsub") server
+
+The pubsub server can be loaded with docker above or run locally with uvicorn below.
+Run the server in dev mode:
+
+```
+uvicorn app_pubsub:app --reload
+```
+
+Run the server in prod mode:
+
+```
+uvicorn app_pubsub:app --host 0.0.0.0
+```
+
+To enable additional logging from uvicorn pass `--log-level debug` or `--log-level trace`.
+
+## Running the frontend
+
+- For the v2 pubsub server, the frontend lives in the streaming-react-app root directory (not seamless_ui).
+
+## Updating the conda environment after changes to environment.yml
+
+Run this command to install/remove packages to match the current environment.yml file:
+
+`conda env update --prefix ./env --file environment.yml  --prune`
+
+See: https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html?highlight=sharing#updating-an-environment
+
+## Debuging
+
+If you enable "Server Debug Flag" when starting streaming from the client, this enables extensive debug logging and it saves audio files in /debug folder. test_no_silence.wav contains data with silence chunks removed.

seamless_server/models/s2s_m4t_expr-emma_v0.3/vad_s2st_sc_24khz_main.yaml

@@ -0,0 +1,24 @@
+agent_class: seamless_communication.streaming.agents.mma_m4t_s2st.SeamlessS2STJointVADAgent
+# checkpoint: checkpoint_best.pt
+monotonic_decoder_model_name: seamless_streaming_monotonic_decoder
+unity_model_name: seamless_streaming_unity
+sentencepiece_model: spm_256k_nllb100.model
+
+task: s2st
+tgt_lang: "eng"
+min_unit_chunk_size: 50
+decision_threshold: 0.7
+no_early_stop: True
+block_ngrams: True
+vocoder_name: vocoder_pretssel
+wav2vec_yaml: wav2vec.yaml
+# min_starting_wait: 12
+# min_starting_wait_w2vbert: 192
+
+config_yaml: cfg_fbank_u2t.yaml
+vocoder_sample_rate: 24000
+upstream_idx: 1
+detokenize_only: True
+device: cuda:0
+max_len_a: 0
+max_len_b: 1000

seamless_server/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "seamless_server",
+  "version": "1.0.0",
+  "scripts": {
+    "dev": "npm run start:debug -- --reload",
+    "start": "uvicorn app_pubsub:app --host 0.0.0.0",
+    "start:debug": "npm run start -- --log-level debug",
+    "start:trace": "npm run start -- --log-level trace",
+    "start:prod": "npm run build:client && npm run start",
+    "build:client": "cd ../streaming-react-app/ && npm run build"
+  },
+  "author": ""
+}

seamless_server/requirements.txt

@@ -0,0 +1,25 @@
+# fairseq2
+# seamless_communication
+Flask==2.1.3
+Flask_Sockets==0.2.1
+g2p_en==2.1.0
+gevent==22.10.2
+gevent_websocket==0.10.1
+librosa==0.9.2
+numpy==1.24.4
+openai_whisper==20230124
+protobuf==4.24.2
+psola==0.0.1
+pydub==0.25.1
+silero==0.4.1
+simuleval==1.1.1
+soundfile==0.11.0
+stable_ts==1.4.0
+torch  # specific torch version depends on fairseq2 installation
+Werkzeug==2.0.3
+whisper==1.1.10
+colorlog==6.7.0
+python-socketio==5.9.0
+uvicorn[standard]==0.23.2
+parallel-wavegan==0.5.5
+python-jose[cryptography]==3.3.0

seamless_server/src/simuleval_agent_directory.py

@@ -123,7 +123,7 @@ class SimulevalAgentDirectory:
                 self.add_agent(agent_info)
         else:
             s2s_m4t_expr_agent = self.build_agent_if_available(
-                SimulevalAgentDirectory.s2s_m4t_emma_agent,
+                SimulevalAgentDirectory.s2s_m4t_expr_emma_agent,
                 config_name="vad_s2st_sc_24khz_main.yaml",
             )
 
@@ -131,12 +131,12 @@ class SimulevalAgentDirectory:
                 self.add_agent(
                     AgentWithInfo(
                         agent=s2s_m4t_expr_agent,
-                        name=SimulevalAgentDirectory.s2s_m4t_emma_agent,
+                        name=SimulevalAgentDirectory.s2s_m4t_expr_emma_agent,
                         modalities=["s2t", "s2s"],
                         source_langs=M4T_P0_LANGS,
                         target_langs=["eng", "spa", "fra", "deu", "ita", "cmn"],
                         dynamic_params=["expressive"],
-                        description="ES-EN expressive model that supports S2S and S2T",
+                        description="multilingual expressive model that supports S2S and S2T",
                     )
                 )