Squashed commit of the following:

commit 151299bcb4803525f9a5ed243fd567bf539a54b8
Author: Justin Haaheim <[email protected]>
Date: Thu Nov 30 23:00:42 2023 +0000

Prevent the server from being unlocked if were locking it completely

commit 2a6cf503c9729a410b57f1b7e164c7b5ff01dc46
Author: Justin Haaheim <[email protected]>
Date: Thu Nov 30 22:57:47 2023 +0000

Fix member object creation and add logging if lock completely is set

commit ea97d8b4145ede78fca9c249439e10be97ba1097
Author: Justin Haaheim <[email protected]>
Date: Thu Nov 30 22:51:06 2023 +0000

Enable fully locking server using env var

commit c40b4cb84fbbb74d3b70ff09f473b9612e1bd4ac
Author: Justin Haaheim <[email protected]>
Date: Thu Nov 30 13:16:38 2023 -0800

[Streaming] Add headphones notice and echo cancellation warning (#176)

commit 94a338e9127b1a3936ea1d1c6763e6a90ee8ee9d
Author: Justin Haaheim <[email protected]>
Date: Thu Nov 30 09:31:56 2023 -0800

[Streaming] Show full language name in dropdown (#173)

Co-authored-by: Mark Duppenthaler <[email protected]>

seamless_server/app_pubsub.py

@@ -124,7 +124,22 @@ class ServerLock(TypedDict):
     member_object: Member
 
 
-server_lock: Optional[ServerLock] = None
+if os.environ.get("LOCK_SERVER_COMPLETELY"):
+    logger.info("LOCK_SERVER_COMPLETELY is set. Server will be locked on startup.")
+dummy_server_lock_member_object = Member(
+    client_id="seamless_user", session_id="dummy", name="Seamless User"
+)
+# Normally this would be an actual transcoder, but it's fine putting True here since currently we only check for the presence of the transcoder
+dummy_server_lock_member_object.transcoder = True
+server_lock: Optional[ServerLock] = (
+    {
+        "name": "Seamless User",
+        "client_id": "seamless_user",
+        "member_object": dummy_server_lock_member_object,
+    }
+    if os.environ.get("LOCK_SERVER_COMPLETELY")
+    else None
+)
 
 server_id = str(uuid4())
 
@@ -497,16 +512,17 @@ async def join_room(sid, client_id, room_id_from_client, config_dict):
     ):
         # If something goes wrong and the server gets stuck in a locked state the client can
         # force the server to remove the lock by passing the special name ESCAPE_HATCH_SERVER_LOCK_RELEASE_NAME
-        # TEMP: remove escape hatch for demo
-        # if (
-        #     server_lock is not None
-        #     and config_dict.get("lockServerName")
-        #     == ESCAPE_HATCH_SERVER_LOCK_RELEASE_NAME
-        # ):
-        #     server_lock = None
-        #     logger.info(
-        #         f"🔓 Server lock has been reset by {client_id} using the escape hatch name {ESCAPE_HATCH_SERVER_LOCK_RELEASE_NAME}"
-        #     )
+        if (
+            server_lock is not None
+            and config_dict.get("lockServerName")
+            == ESCAPE_HATCH_SERVER_LOCK_RELEASE_NAME
+            # If we are locking the server completely we don't want someone to be able to unlock it
+            and not os.environ.get("LOCK_SERVER_COMPLETELY")
+        ):
+            server_lock = None
+            logger.info(
+                f"🔓 Server lock has been reset by {client_id} using the escape hatch name {ESCAPE_HATCH_SERVER_LOCK_RELEASE_NAME}"
+            )
 
         # If the server is not locked, set a lock. If it's already locked to this client, update the lock object
         if server_lock is None or server_lock.get("client_id") == client_id:

streaming-react-app/package.json

@@ -1,11 +1,11 @@
 {
   "name": "streaming-react-app",
   "private": true,
-  "version": "0.0.13",
+  "version": "0.0.14",
   "type": "module",
   "scripts": {
     "dev": "vite --host --strictPort",
-    "build": "tsc && vite build",
+    "build": "vite build",
     "preview": "vite preview",
     "clean:node-modules": "rm -rf node_modules/",
     "ts-check": "tsc --noEmit",
@@ -24,7 +24,6 @@
     "amazon-cognito-identity-js": "^6.3.6",
     "audiobuffer-to-wav": "^1.0.0",
     "aws-sdk": "^2.1472.0",
-    "iso-639-1": "^3.1.0",
     "js-cookie": "^3.0.5",
     "lodash": "4.17.21",
     "react": "^18.2.0",

streaming-react-app/src/StreamingInterface.tsx

@@ -57,12 +57,12 @@ import {CURSOR_BLINK_INTERVAL_MS} from './cursorBlinkInterval';
 import {getURLParams} from './URLParams';
 import debug from './debug';
 import DebugSection from './DebugSection';
-import {Grid} from '@mui/material';
+import Switch from '@mui/material/Switch';
+import Grid from '@mui/material/Grid';
 import {getLanguageFromThreeLetterCode} from './languageLookup';
+import HeadphonesIcon from '@mui/icons-material/Headphones';
 
-const AUDIO_STREAM_DEFAULTS: {
-  [key in SupportedInputSource]: BrowserAudioStreamConfig;
-} = {
+const AUDIO_STREAM_DEFAULTS = {
   userMedia: {
     echoCancellation: false,
     noiseSuppression: true,
@@ -71,13 +71,10 @@ const AUDIO_STREAM_DEFAULTS: {
     echoCancellation: false,
     noiseSuppression: false,
   },
-};
+} as const;
 
 async function requestUserMediaAudioStream(
-  config: BrowserAudioStreamConfig = {
-    echoCancellation: false,
-    noiseSuppression: true,
-  },
+  config: BrowserAudioStreamConfig = AUDIO_STREAM_DEFAULTS['userMedia'],
 ) {
   const stream = await navigator.mediaDevices.getUserMedia({
     audio: {...config, channelCount: 1},
@@ -90,10 +87,7 @@ async function requestUserMediaAudioStream(
 }
 
 async function requestDisplayMediaAudioStream(
-  config: BrowserAudioStreamConfig = {
-    echoCancellation: false,
-    noiseSuppression: false,
-  },
+  config: BrowserAudioStreamConfig = AUDIO_STREAM_DEFAULTS['displayMedia'],
 ) {
   const stream = await navigator.mediaDevices.getDisplayMedia({
     audio: {...config, channelCount: 1},
@@ -962,8 +956,9 @@ export default function StreamingInterface() {
                         </RadioGroup>
                       </FormControl>
                     </Box>
-                    <Box sx={{flex: 1}}>
-                      <FormControl disabled={streamFixedConfigOptionsDisabled}>
+
+                    <Box sx={{flex: 1, flexGrow: 2}}>
+                    <FormControl disabled={streamFixedConfigOptionsDisabled}>
                         <FormLabel>Options</FormLabel>
                         <FormControlLabel
                           control={
@@ -980,9 +975,9 @@ export default function StreamingInterface() {
                               }
                             />
                           }
-                          label="Noise Suppression (Browser)"
+                          label="Noise Suppression"
                         />
-                                                <FormControlLabel
+                        <FormControlLabel
                           control={
                             <Checkbox
                               checked={
@@ -997,7 +992,7 @@ export default function StreamingInterface() {
                               }
                             />
                           }
-                          label="Echo Cancellation (Browser)"
+                          label="Echo Cancellation (not recommended)"
                         />
                         <FormControlLabel
                           control={
@@ -1008,12 +1003,34 @@ export default function StreamingInterface() {
                               ) => setServerDebugFlag(event.target.checked)}
                             />
                           }
-                          label="Server Debug Flag"
+                          label="Enable Server Debugging"
                         />
                       </FormControl>
                     </Box>
                   </Stack>
 
+                  {isSpeaker &&
+                    isListener &&
+                    inputSource === 'userMedia' &&
+                    !enableEchoCancellation &&
+                    gain !== 0 && (
+                      <div>
+                        <Alert severity="warning" icon={<HeadphonesIcon />}>
+                          Headphones required to prevent feedback.
+                        </Alert>
+                      </div>
+                    )}
+
+                  {isSpeaker && enableEchoCancellation && (
+                    <div>
+                      <Alert severity="warning">
+                        We don't recommend using echo cancellation as it may
+                        distort the input audio. If possible, use headphones and
+                        disable echo cancellation instead.
+                      </Alert>
+                    </div>
+                  )}
+
                   <Stack direction="row" spacing={2}>
                     {streamingStatus === 'stopped' ? (
                       <Button

streaming-react-app/src/languageLookup.ts

@@ -1,108 +1,110 @@
-const LANG3_FULL = {
-  eng: 'english', 
-  arb: 'arabic', 
-  asm: 'assamese', 
-  bel: 'belarusian', 
-  bul: 'bulgarian', 
-  ben: 'bengali', 
-  cat: 'catalan', 
-  ces: 'czech', 
-  cym: 'welsh', 
-  dan: 'danish', 
-  deu: 'german', 
-  ell: 'greek', 
-  spa: 'spanish', 
-  est: 'estonian', 
-  pes: 'persian', 
-  fin: 'finnish', 
-  fra: 'french', 
-  hin: 'hindi', 
-  hun: 'hungarian', 
-  ind: 'indonesian', 
-  ita: 'italian', 
-  jpn: 'japanese', 
-  kat: 'georgian', 
-  lit: 'lithuanian', 
-  lvs: 'latvian', 
-  khk: 'mongolian', 
-  mar: 'marathi', 
-  mlt: 'maltese', 
-  nld: 'dutch', 
-  pan: 'punjabi', 
-  pol: 'polish', 
-  por: 'portuguese', 
-  ron: 'romanian', 
-  rus: 'russian', 
-  slk: 'slovak', 
-  slv: 'slovenian', 
-  swe: 'swedish', 
-  swh: 'swahili', 
-  tam: 'tamil', 
-  tha: 'thai', 
-  tur: 'turkish', 
-  ukr: 'ukrainian', 
-  urd: 'urdu', 
-  uzn: 'uzbek', 
-  vie: 'vietnamese', 
-  cmn: 'chinese', 
-  afr: 'afrikaans', 
-  isl: 'icelandic', 
-  ltz: 'luxembourgish', 
-  nob: 'norwegian', 
-  glg: 'galician', 
-  bos: 'bosnian', 
-  hrv: 'croatian', 
-  mkd: 'macedonian', 
-  srp: 'serbian', 
-  hye: 'armenian', 
-  azj: 'azerbaijani', 
-  kaz: 'kazakh', 
-  kor: 'korean', 
-  guj: 'gujarati', 
-  kan: 'kannada', 
-  npi: 'nepali', 
-  snd: 'sindhi', 
-  tel: 'telugu', 
-  jav: 'javanese', 
-  zlm: 'malay', 
-  mal: 'malayalam', 
-  tgl: 'tagalog', 
-  mya: 'myanmar', 
-  khm: 'khmer', 
-  lao: 'lao', 
-  heb: 'hebrew', 
-  pbt: 'pashto', 
-  tgk: 'tajik', 
-  amh: 'amharic', 
-  lin: 'lingala', 
-  som: 'somali', 
-  yor: 'yoruba', 
-  sna: 'shona', 
-  mri: 'maori', 
-  hau: 'hausa', 
-  oci: 'occitan', 
-  bak: 'bashkir', 
-  bre: 'breton', 
-  yid: 'yiddish', 
-  hat: 'haitian creole', 
-  mlg: 'malagasy', 
-  sin: 'sinhala', 
-  sqi: 'albanian', 
-  sun: 'sundanese', 
-  eus: 'basque', 
-  nno: 'nynorsk', 
-  tat: 'tatar', 
-  bod: 'tibetan', 
-  fao: 'faroese', 
-  haw: 'hawaiian', 
-  lat: 'latin', 
-  san: 'sanskrit', 
-  tuk: 'turkmen'
+const LANG3_TO_NAME = {
+  afr: 'afrikaans',
+  amh: 'amharic',
+  arb: 'arabic',
+  asm: 'assamese',
+  azj: 'azerbaijani',
+  bak: 'bashkir',
+  bel: 'belarusian',
+  ben: 'bengali',
+  bod: 'tibetan',
+  bos: 'bosnian',
+  bre: 'breton',
+  bul: 'bulgarian',
+  cat: 'catalan',
+  ces: 'czech',
+  cmn: 'chinese',
+  cym: 'welsh',
+  dan: 'danish',
+  deu: 'german',
+  ell: 'greek',
+  eng: 'english',
+  est: 'estonian',
+  eus: 'basque',
+  fao: 'faroese',
+  fin: 'finnish',
+  fra: 'french',
+  glg: 'galician',
+  guj: 'gujarati',
+  hat: 'haitian creole',
+  hau: 'hausa',
+  haw: 'hawaiian',
+  heb: 'hebrew',
+  hin: 'hindi',
+  hrv: 'croatian',
+  hun: 'hungarian',
+  hye: 'armenian',
+  ind: 'indonesian',
+  isl: 'icelandic',
+  ita: 'italian',
+  jav: 'javanese',
+  jpn: 'japanese',
+  kan: 'kannada',
+  kat: 'georgian',
+  kaz: 'kazakh',
+  khk: 'mongolian',
+  khm: 'khmer',
+  kor: 'korean',
+  lao: 'lao',
+  lat: 'latin',
+  lin: 'lingala',
+  lit: 'lithuanian',
+  ltz: 'luxembourgish',
+  lvs: 'latvian',
+  mal: 'malayalam',
+  mar: 'marathi',
+  mkd: 'macedonian',
+  mlg: 'malagasy',
+  mlt: 'maltese',
+  mri: 'maori',
+  mya: 'myanmar',
+  nld: 'dutch',
+  nno: 'nynorsk',
+  nob: 'norwegian',
+  npi: 'nepali',
+  oci: 'occitan',
+  pan: 'punjabi',
+  pbt: 'pashto',
+  pes: 'persian',
+  pol: 'polish',
+  por: 'portuguese',
+  ron: 'romanian',
+  rus: 'russian',
+  san: 'sanskrit',
+  sin: 'sinhala',
+  slk: 'slovak',
+  slv: 'slovenian',
+  sna: 'shona',
+  snd: 'sindhi',
+  som: 'somali',
+  spa: 'spanish',
+  sqi: 'albanian',
+  srp: 'serbian',
+  sun: 'sundanese',
+  swe: 'swedish',
+  swh: 'swahili',
+  tam: 'tamil',
+  tat: 'tatar',
+  tel: 'telugu',
+  tgk: 'tajik',
+  tgl: 'tagalog',
+  tha: 'thai',
+  tuk: 'turkmen',
+  tur: 'turkish',
+  ukr: 'ukrainian',
+  urd: 'urdu',
+  uzn: 'uzbek',
+  vie: 'vietnamese',
+  yid: 'yiddish',
+  yor: 'yoruba',
+  zlm: 'malay',
 };
 
-export function getLanguageFromThreeLetterCode(code: string): string | null {
+export function getLanguageFromThreeLetterCode(
+  lang3Code: string,
+): string | null {
   try {
-    const name = LANG3_FULL[code] ?? null;
+    const name = LANG3_TO_NAME[lang3Code] ?? null;
     if (name == null) {
       return null;
     }
@@ -111,7 +113,7 @@ export function getLanguageFromThreeLetterCode(code: string): string | null {
       .map((word: string) => word[0].toUpperCase() + word.slice(1));
     return capitalizedWords.join(' ');
   } catch (e) {
-    console.warn(`Unable to get language name for code ${code}: ${e}`);
+    console.warn(`Unable to get language name for code ${lang3Code}: ${e}`);
   }
   return null;
 }

streaming-react-app/src/react-xr/TextBlocks.tsx

@@ -1,9 +1,8 @@
-import {JSX, useEffect, useRef, useState} from 'react';
+import {useEffect, useRef, useState} from 'react';
 import robotoFontFamilyJson from '../assets/RobotoMono-Regular-msdf.json?url';
 import robotoFontTexture from '../assets/RobotoMono-Regular.png';
 import ThreeMeshUIText, {ThreeMeshUITextType} from './ThreeMeshUIText';
-import {getURLParams} from '../URLParams';
-import {CURSOR_BLINK_INTERVAL_MS} from '../cursorBlinkInterval';
+import supportedCharSet from './supportedCharSet';
 
 const NUM_LINES = 3;
 
@@ -22,44 +21,80 @@ const SCROLL_Y_DELTA = 0.001;
 const OFFSET = 0.01;
 const OFFSET_WIDTH = OFFSET * 3;
 
-type Props = {
+const CHARS_PER_SECOND = 10;
+
+// The tick interval
+const RENDER_INTERVAL = 300;
+
+const CURSOR_BLINK_INTERVAL_MS = 1000;
+
+type TextBlockProps = {
   content: string;
   // The actual position or end position when animating
   y: number;
   // The start position when animating
   startY: number;
-  width: number;
-  height: number;
   textOpacity: number;
   backgroundOpacity: number;
-  // Use this to keep track of sentence + line position for animation
-  index: string;
-  enableAnimation: boolean;
+  index: number;
+  isBottomLine: boolean;
+  // key: number;
+};
+
+type TranscriptState = {
+  textBlocksProps: TextBlockProps[];
+  lastTranslationStringIndex: number;
+  lastTranslationLineStartIndex: number;
+  transcriptLines: string[];
+  lastRenderTime: number;
 };
 
 function TextBlock({
   content,
   y,
   startY,
-  width,
-  height,
   textOpacity,
   backgroundOpacity,
   index,
-  enableAnimation,
-}: Props) {
+  isBottomLine,
+}: TextBlockProps) {
   const [scrollY, setScrollY] = useState<number>(y);
-
   // We are reusing text blocks so this keeps track of when we changed rows so we can restart animation
-  const lastIndex = useRef<string>(index);
+  const lastIndex = useRef<number>(index);
   useEffect(() => {
     if (index != lastIndex.current) {
       lastIndex.current = index;
-      enableAnimation && setScrollY(startY);
+      !isBottomLine && setScrollY(startY);
     } else if (scrollY < y) {
       setScrollY((prev) => prev + SCROLL_Y_DELTA);
     }
-  }, [enableAnimation, index, scrollY, setScrollY, startY, y]);
+  }, [isBottomLine, index, scrollY, setScrollY, startY, y]);
+
+  const [cursorBlinkOn, setCursorBlinkOn] = useState(false);
+  useEffect(() => {
+    if (isBottomLine) {
+      const interval = setInterval(() => {
+        setCursorBlinkOn((prev) => !prev);
+      }, CURSOR_BLINK_INTERVAL_MS);
+
+      return () => clearInterval(interval);
+    } else {
+      setCursorBlinkOn(false);
+    }
+  }, [isBottomLine]);
+
+  const numChars = content.length;
+
+  if (cursorBlinkOn) {
+    content = content + '|';
+  }
+
+  // Accounting for potential cursor for block width (the +1)
+  const width =
+    (numChars + (isBottomLine ? 1.1 : 0) + (numChars < 10 ? 1 : 0)) *
+    CHAR_WIDTH;
+
+  const height = LINE_HEIGHT;
 
   // This is needed to update text content (doesn't work if we just update the content prop)
   const textRef = useRef<ThreeMeshUITextType>();
@@ -111,125 +146,162 @@ function TextBlock({
   );
 }
 
-// Background behind the text so it covers any missing spaces
-function TranscriptionPanel() {
-  const panelHeight = LINE_HEIGHT * NUM_LINES + 2 * BLOCK_SPACING + 2 * OFFSET;
-  const xPosition = OFFSET_WIDTH;
-  return (
-    <block
-      args={[
-        {
-          backgroundOpacity: 1,
-          width:
-            MAX_WIDTH * ((CHARS_PER_LINE + 2) / CHARS_PER_LINE) +
-            2 * OFFSET_WIDTH,
-          height: panelHeight,
-          borderRadius: 0,
-        },
-      ]}
-      position={[
-        -OFFSET + xPosition,
-        Y_COORD_START + panelHeight / 2 - 2 * OFFSET,
-        Z_COORD,
-      ]}></block>
-  );
+function initialTextBlockProps(count: number): TextBlockProps[] {
+  return Array.from({length: count}).map(() => {
+    // Push in non display blocks because mesh UI crashes if elements are add / removed from screen.
+    return {
+      y: Y_COORD_START,
+      startY: 0,
+      index: 0,
+      textOpacity: 0,
+      backgroundOpacity: 0,
+      width: MAX_WIDTH,
+      height: LINE_HEIGHT,
+      content: '',
+      isBottomLine: true,
+    };
+  });
 }
 
 export default function TextBlocks({
-  sentences,
-  blinkCursor,
+  translationText,
 }: {
-  sentences: string[][];
-  blinkCursor: boolean;
+  translationText: string;
 }) {
-  const showTranscriptionPanel =
-    getURLParams().ARTranscriptionType === 'lines_with_background';
-  const textBlocks: JSX.Element[] = [];
+  const transcriptStateRef = useRef<TranscriptState>({
+    textBlocksProps: initialTextBlockProps(NUM_LINES),
+    lastTranslationStringIndex: 0,
+    lastTranslationLineStartIndex: 0,
+    transcriptLines: [],
+    lastRenderTime: new Date().getTime(),
+  });
+
+  const transcriptState = transcriptStateRef.current;
+  const {textBlocksProps, lastTranslationStringIndex, lastRenderTime} =
+    transcriptState;
+
+  const [charsToRender, setCharsToRender] = useState<number>(0);
 
-  const [cursorBlinkOn, setCursorBlinkOn] = useState(false);
   useEffect(() => {
-    if (blinkCursor) {
-      const interval = setInterval(() => {
-        setCursorBlinkOn((prev) => !prev);
-      }, CURSOR_BLINK_INTERVAL_MS);
+    const interval = setInterval(() => {
+      const currentTime = new Date().getTime();
+      const charsToRender = Math.round(
+        ((currentTime - lastRenderTime) * CHARS_PER_SECOND) / 1000,
+      );
+      setCharsToRender(charsToRender);
+    }, RENDER_INTERVAL);
 
-      return () => clearInterval(interval);
-    } else {
-      setCursorBlinkOn(false);
-    }
-  }, [blinkCursor]);
+    return () => clearInterval(interval);
+  }, [lastRenderTime]);
 
-  // Start from bottom and populate most recent sentences by line until we fill max lines.
-  let currentY = Y_COORD_START;
-  for (let i = sentences.length - 1; i >= 0; i--) {
-    const sentenceLines = sentences[i];
-    for (let j = sentenceLines.length - 1; j >= 0; j--) {
-      if (textBlocks.length == NUM_LINES) {
-        if (showTranscriptionPanel) {
-          textBlocks.push(<TranscriptionPanel key={textBlocks.length} />);
-        }
-        return textBlocks;
-      }
+  const currentTime = new Date().getTime();
+  if (charsToRender < 1) {
+    return textBlocksProps.map((props, idx) => (
+      <TextBlock {...props} key={idx} />
+    ));
+  }
+
+  const nextTranslationStringIndex = Math.min(
+    lastTranslationStringIndex + charsToRender,
+    translationText.length,
+  );
+  const newString = translationText.substring(
+    lastTranslationStringIndex,
+    nextTranslationStringIndex,
+  );
+  if (nextTranslationStringIndex === lastTranslationStringIndex) {
+    transcriptState.lastRenderTime = currentTime;
+    return textBlocksProps.map((props, idx) => (
+      <TextBlock {...props} key={idx} />
+    ));
+  }
+
+  // Wait until more characters are accumulated if its just blankspace
+  if (/^\s*$/.test(newString)) {
+    transcriptState.lastRenderTime = currentTime;
+    return textBlocksProps.map((props, idx) => (
+      <TextBlock {...props} key={idx} />
+    ));
+  }
+
+  // Ideally we continue where we left off but this is complicated when we have mid-words. Recalculating for now
+  const runAll = true;
+  const newSentences = runAll
+    ? translationText.substring(0, nextTranslationStringIndex).split('\n')
+    : newString.split('\n');
+  const transcriptLines = runAll ? [''] : transcriptState.transcriptLines;
+  newSentences.forEach((newSentence, sentenceIdx) => {
+    const words = newSentence.split(/\s+/);
+    words.forEach((word) => {
+      const filteredWord = [...word]
+        .filter((c) => {
+          if (supportedCharSet().has(c)) {
+            return true;
+          }
+          console.error(
+            `Unsupported char ${c} - make sure this is supported in the font family msdf file`,
+          );
+          return false;
+        })
+        .join('');
 
-      const isBottomSentence = i === sentences.length - 1;
-      const isBottomLine = isBottomSentence && textBlocks.length === 0;
-      const y = currentY + LINE_HEIGHT / 2;
-      let textBlockLine = sentenceLines[j];
-      const numChars = textBlockLine.length;
+      const lastLineSoFar = transcriptLines[0];
+      const charCount = lastLineSoFar.length + filteredWord.length + 1;
 
-      if (cursorBlinkOn && isBottomLine) {
-        textBlockLine = textBlockLine + '|';
+      if (charCount <= CHARS_PER_LINE) {
+        transcriptLines[0] = lastLineSoFar + ' ' + filteredWord;
+      } else {
+        transcriptLines.unshift(filteredWord);
       }
+    });
 
-      // Accounting for potential cursor for block width (the +1)
-      const blockWidth =
-        (numChars + (isBottomLine ? 1.1 : 0) + (numChars < 10 ? 1 : 0)) *
-        CHAR_WIDTH;
-      const textOpacity = 1 - 0.1 * textBlocks.length;
-      textBlocks.push(
-        <TextBlock
-          key={textBlocks.length}
-          y={y}
-          startY={currentY}
-          index={`${sentences.length - i},${j}`}
-          textOpacity={textOpacity}
-          backgroundOpacity={0.98}
-          height={LINE_HEIGHT}
-          width={blockWidth}
-          // content={"BLOCK " + textBlocks.length + ": " + content}
-          content={textBlockLine}
-          enableAnimation={!isBottomLine}
-        />,
-      );
+    if (sentenceIdx < newSentences.length - 1) {
+      transcriptLines.unshift('\n');
+      transcriptLines.unshift('');
+    }
+  });
+
+  transcriptState.transcriptLines = transcriptLines;
+  transcriptState.lastTranslationStringIndex = nextTranslationStringIndex;
 
-      currentY = y + LINE_HEIGHT / 2;
+  const newTextBlocksProps: TextBlockProps[] = [];
+  let currentY = Y_COORD_START;
+
+  transcriptLines.forEach((line, i) => {
+    if (newTextBlocksProps.length == NUM_LINES) {
+      return;
     }
-    currentY += showTranscriptionPanel ? BLOCK_SPACING / 3 : BLOCK_SPACING;
-  }
 
-  const numRemainingBlocks = textBlocks.length - NUM_LINES;
-  if (numRemainingBlocks > 0) {
-    Array.from({length: numRemainingBlocks}).forEach(() => {
-      // Push in non display blocks because mesh UI crashes if elements are add / removed from screen.
-      textBlocks.push(
-        <TextBlock
-          key={textBlocks.length}
-          y={Y_COORD_START}
-          startY={0}
-          index="0,0"
-          textOpacity={0}
-          backgroundOpacity={0}
-          enableAnimation={false}
-          width={MAX_WIDTH}
-          height={LINE_HEIGHT}
-          content=""
-        />,
-      );
+    // const line = transcriptLines[i];
+    if (line === '\n') {
+      currentY += BLOCK_SPACING;
+      return;
+    }
+    const y = currentY + LINE_HEIGHT / 2;
+    const isBottomLine = newTextBlocksProps.length === 0;
+
+    const textOpacity = 1 - 0.1 * newTextBlocksProps.length;
+    newTextBlocksProps.push({
+      y,
+      startY: currentY,
+      index: i,
+      textOpacity,
+      backgroundOpacity: 0.98,
+      content: line,
+      isBottomLine,
     });
-  }
 
-  if (showTranscriptionPanel) {
-    textBlocks.push(<TranscriptionPanel key={textBlocks.length} />);
+    currentY = y + LINE_HEIGHT / 2;
+  });
+
+  const numRemainingBlocks = NUM_LINES - newTextBlocksProps.length;
+  if (numRemainingBlocks > 0) {
+    newTextBlocksProps.push(...initialTextBlockProps(numRemainingBlocks));
   }
-  return textBlocks;
+
+  transcriptState.textBlocksProps = newTextBlocksProps;
+  transcriptState.lastRenderTime = currentTime;
+  return newTextBlocksProps.map((props, idx) => (
+    <TextBlock {...props} key={idx} />
+  ));
 }

streaming-react-app/src/react-xr/XRConfig.tsx

@@ -25,29 +25,15 @@ import {BLACK, WHITE} from './Colors';
 import robotoFontFamilyJson from '../assets/RobotoMono-Regular-msdf.json?url';
 import robotoFontTexture from '../assets/RobotoMono-Regular.png';
 import {getURLParams} from '../URLParams';
-import TextBlocks, {CHARS_PER_LINE} from './TextBlocks';
+import TextBlocks from './TextBlocks';
 import {BufferedSpeechPlayer} from '../createBufferedSpeechPlayer';
 import {CURSOR_BLINK_INTERVAL_MS} from '../cursorBlinkInterval';
+import supportedCharSet from './supportedCharSet';
 
 // Adds on react JSX for add-on libraries to react-three-fiber
 extend(ThreeMeshUI);
 extend({TextGeometry});
 
-async function fetchSupportedCharSet(): Promise<Set<string>> {
-  try {
-    const response = await fetch(robotoFontFamilyJson);
-    const fontFamily = await response.json();
-
-    return new Set(fontFamily.info.charset);
-  } catch (e) {
-    console.error('Failed to fetch supported XR charset', e);
-    return new Set();
-  }
-}
-
-let supportedCharSet = new Set();
-fetchSupportedCharSet().then((result) => (supportedCharSet = result));
-
 // This component wraps any children so it is positioned relative to the camera, rather than from the origin
 function CameraLinkedObject({children}) {
   const camera = useThree((state) => state.camera);
@@ -76,10 +62,7 @@ function ThreeMeshUIComponents({
             translationSentences={translationSentences}
           />
         ) : (
-          <TranscriptPanelBlocks
-            animateTextDisplay={animateTextDisplay}
-            translationSentences={translationSentences}
-          />
+          <TranscriptPanelBlocks translationSentences={translationSentences} />
         )}
         {skipARIntro ? null : (
           <IntroPanel started={started} setStarted={setStarted} />
@@ -153,7 +136,7 @@ function TranscriptPanelSingleBlock({
           (wordChunks, currentWord) => {
             const filteredWord = [...currentWord]
               .filter((c) => {
-                if (supportedCharSet.has(c)) {
+                if (supportedCharSet().has(c)) {
                   return true;
                 }
                 console.error(
@@ -223,59 +206,14 @@ function TranscriptPanelSingleBlock({
 // Splits up the lines into separate blocks to treat each one separately.
 // This allows changing of opacity, animating per line, changing height / width per line etc
 function TranscriptPanelBlocks({
-  animateTextDisplay,
   translationSentences,
 }: {
-  animateTextDisplay: boolean;
   translationSentences: TranslationSentences;
 }) {
-  const [didReceiveTranslationSentences, setDidReceiveTranslationSentences] =
-    // Currently causing issues with displaying dummy text, skip over
-    useState(false);
-
-  // Normally we don't setState in render, but here we need to for computed state, and this if statement assures it won't loop infinitely
-  if (!didReceiveTranslationSentences && translationSentences.length > 0) {
-    setDidReceiveTranslationSentences(true);
-  }
-
-  const initialPrompt = 'Listening...';
-  const transcriptSentences: string[] = didReceiveTranslationSentences
-    ? translationSentences
-    : [initialPrompt];
-
-  // The transcript is an array of sentences. For each sentence we break this down into an array of words per line.
-  // This is needed so we can "scroll" through without changing the order of words in the transcript
-  const sentenceLines = transcriptSentences.map((sentence) => {
-    const words = sentence.split(/\s+/);
-    // Here we break each sentence up with newlines so all words per line fit within the panel
-    return words.reduce(
-      (wordChunks, currentWord) => {
-        const filteredWord = [...currentWord]
-          .filter((c) => {
-            if (supportedCharSet.has(c)) {
-              return true;
-            }
-            console.error(
-              `Unsupported char ${c} - make sure this is supported in the font family msdf file`,
-            );
-            return false;
-          })
-          .join('');
-        const lastLineSoFar = wordChunks[wordChunks.length - 1];
-        const charCount = lastLineSoFar.length + filteredWord.length + 1;
-        if (charCount <= CHARS_PER_LINE) {
-          wordChunks[wordChunks.length - 1] =
-            lastLineSoFar + ' ' + filteredWord;
-        } else {
-          wordChunks.push(filteredWord);
-        }
-        return wordChunks;
-      },
-      [''],
-    );
-  });
   return (
-    <TextBlocks sentences={sentenceLines} blinkCursor={animateTextDisplay} />
+    <TextBlocks
+      translationText={'Listening...\n' + translationSentences.join('\n')}
+    />
   );
 }
 
@@ -361,6 +299,8 @@ export type XRConfigProps = {
   startStreaming: () => Promise<void>;
   stopStreaming: () => Promise<void>;
   debugParam: boolean | null;
+  onARVisible?: () => void;
+  onARHidden?: () => void;
 };
 
 export default function XRConfig(props: XRConfigProps) {

streaming-react-app/src/react-xr/XRDialog.tsx

@@ -8,27 +8,12 @@ import {
   Typography,
 } from '@mui/material';
 import CloseIcon from '@mui/icons-material/Close';
+import {XRConfigProps} from './XRConfig';
 import {useEffect, useRef, useState} from 'react';
 import './XRDialog.css';
 import {getRenderer, init, updatetranslationText} from './XRRendering';
 import ARButton from './ARButton';
 import {getURLParams} from '../URLParams';
-import { BufferedSpeechPlayer } from '../createBufferedSpeechPlayer';
-import { TranslationSentences } from '../types/StreamingTypes';
-import { RoomState } from '../types/RoomState';
-
-type XRConfigProps = {
-  animateTextDisplay: boolean;
-  bufferedSpeechPlayer: BufferedSpeechPlayer;
-  translationSentences: TranslationSentences;
-  roomState: RoomState | null;
-  roomID: string | null;
-  startStreaming: () => Promise<void>;
-  stopStreaming: () => Promise<void>;
-  debugParam: boolean | null;
-  onARVisible?: () => void;
-  onARHidden?: () => void;
-};
 
 function XRContent(props: XRConfigProps) {
   const debugParam = getURLParams().debug;

streaming-react-app/src/types/StreamingTypes.ts

@@ -55,9 +55,9 @@ export const SUPPORTED_INPUT_SOURCES: Array<{
   value: SupportedInputSource;
   label: string;
 }> = [
-    { value: 'userMedia', label: 'Microphone' },
-    { value: 'displayMedia', label: 'Browser Tab' },
-  ];
+  {value: 'userMedia', label: 'Microphone'},
+  {value: 'displayMedia', label: 'Browser Tab (Chrome only)'},
+];
 
 export type StartStreamEventConfig = {
   event: 'config';
@@ -70,6 +70,7 @@ export type StartStreamEventConfig = {
 };
 
 export interface BrowserAudioStreamConfig {
+  echoCancellation: boolean;
   noiseSuppression: boolean;
   echoCancellation: boolean;
 }

streaming-react-app/vite.config.ts

@@ -1,10 +1,5 @@
 import { defineConfig } from 'vite';
 import react from '@vitejs/plugin-react';
-// import {resolve} from 'path';
-
-// const rootDir = resolve(__dirname, 'src');
-// const assetsDir = resolve(rootDir, 'assets');
-// const typesDir = resolve(__dirname, 'types');
 
 // https://vitejs.dev/config/
 export default defineConfig(({ command }) => {

streaming-react-app/yarn.lock

@@ -1853,11 +1853,6 @@ isexe@^2.0.0:
   resolved "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz"
   integrity sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==
 
-iso-639-1@^3.1.0:
-  version "3.1.0"
-  resolved "https://registry.npmjs.org/iso-639-1/-/iso-639-1-3.1.0.tgz"
-  integrity sha512-rWcHp9dcNbxa5C8jA/cxFlWNFNwy5Vup0KcFvgA8sPQs9ZeJHj/Eq0Y8Yz2eL8XlWYpxw4iwh9FfTeVxyqdRMw==
-
 isomorphic-unfetch@^3.0.0:
   version "3.1.0"
   resolved "https://registry.npmjs.org/isomorphic-unfetch/-/isomorphic-unfetch-3.1.0.tgz"