Add inital prompt mode. GITLAB #7

Add a new configuration option `vad_initial_prompt_mode` in config.json5
and the application arguments that have the following two modes:
* prepend_all_segments: Preprend the initial prompt to each VAD segment
* prepend_first_segment: Only preprend the initial prompt the first VAD segment.

This is useful if you're using the prompt to improve the accuracy of
the transcription of unusual technical terms consistently throughout a lecture. You
can add these terms to the prompt, and then set `vad_initial_prompt_mode` to
`prepend_all_segments` to include the prompt in every VAD segment.

Note that this will have no effect if you're not using a VAD.

app.py

@@ -12,7 +12,7 @@ import numpy as np
 
 import torch
 
-from src.config import ApplicationConfig
+from src.config import ApplicationConfig, VadInitialPromptMode
 from src.hooks.progressListener import ProgressListener
 from src.hooks.subTaskProgressListener import SubTaskProgressListener
 from src.hooks.whisperProgressHook import create_progress_listener_handle
@@ -43,6 +43,17 @@ MAX_AUTO_CPU_CORES = 8
 
 WHISPER_MODELS = ["tiny", "base", "small", "medium", "large", "large-v1", "large-v2"]
 
+class VadOptions:
+    def __init__(self, vad: str = None, vadMergeWindow: float = 5, vadMaxMergeSize: float = 150, vadPadding: float = 1, vadPromptWindow: float = 1, 
+                                        vadInitialPromptMode: Union[VadInitialPromptMode, str] = VadInitialPromptMode.PREPREND_FIRST_SEGMENT):
+        self.vad = vad
+        self.vadMergeWindow = vadMergeWindow
+        self.vadMaxMergeSize = vadMaxMergeSize
+        self.vadPadding = vadPadding
+        self.vadPromptWindow = vadPromptWindow
+        self.vadInitialPromptMode = vadInitialPromptMode if isinstance(vadInitialPromptMode, VadInitialPromptMode) \
+                                        else VadInitialPromptMode.from_string(vadInitialPromptMode)
+
 class WhisperTranscriber:
     def __init__(self, input_audio_max_duration: float = None, vad_process_timeout: float = None, 
                  vad_cpu_cores: int = 1, delete_uploaded_files: bool = False, output_dir: str = None, 
@@ -75,11 +86,14 @@ class WhisperTranscriber:
     # Entry function for the simple tab
     def transcribe_webui_simple(self, modelName, languageName, urlData, multipleFiles, microphoneData, task, vad, vadMergeWindow, vadMaxMergeSize, vadPadding, vadPromptWindow, 
                                 progress=gr.Progress()):
-        return self.transcribe_webui(modelName, languageName, urlData, multipleFiles, microphoneData, task, vad, vadMergeWindow, vadMaxMergeSize, vadPadding, vadPromptWindow, 
-                                     progress=progress)
+        
+        vadOptions = VadOptions(vad, vadMergeWindow, vadMaxMergeSize, vadPadding, vadPromptWindow, self.app_config.vad_initial_prompt_mode)
+
+        return self.transcribe_webui(modelName, languageName, urlData, multipleFiles, microphoneData, task, vadOptions, progress=progress)
 
     # Entry function for the full tab
-    def transcribe_webui_full(self, modelName, languageName, urlData, multipleFiles, microphoneData, task, vad, vadMergeWindow, vadMaxMergeSize, vadPadding, vadPromptWindow, 
+    def transcribe_webui_full(self, modelName, languageName, urlData, multipleFiles, microphoneData, task, 
+                                    vad, vadMergeWindow, vadMaxMergeSize, vadPadding, vadPromptWindow, vadInitialPromptMode, 
                                     initial_prompt: str, temperature: float, best_of: int, beam_size: int, patience: float, length_penalty: float, suppress_tokens: str, 
                                     condition_on_previous_text: bool, fp16: bool, temperature_increment_on_fallback: float, 
                                     compression_ratio_threshold: float, logprob_threshold: float, no_speech_threshold: float, 
@@ -91,14 +105,16 @@ class WhisperTranscriber:
         else:
             temperature = [temperature]
 
-        return self.transcribe_webui(modelName, languageName, urlData, multipleFiles, microphoneData, task, vad, vadMergeWindow, vadMaxMergeSize, vadPadding, vadPromptWindow, 
+        vadOptions = VadOptions(vad, vadMergeWindow, vadMaxMergeSize, vadPadding, vadPromptWindow, vadInitialPromptMode)
+
+        return self.transcribe_webui(modelName, languageName, urlData, multipleFiles, microphoneData, task, vadOptions,
                                      initial_prompt=initial_prompt, temperature=temperature, best_of=best_of, beam_size=beam_size, patience=patience, length_penalty=length_penalty, suppress_tokens=suppress_tokens,
                                      condition_on_previous_text=condition_on_previous_text, fp16=fp16,
                                      compression_ratio_threshold=compression_ratio_threshold, logprob_threshold=logprob_threshold, no_speech_threshold=no_speech_threshold, 
                                      progress=progress)
 
-    def transcribe_webui(self, modelName, languageName, urlData, multipleFiles, microphoneData, task, vad, vadMergeWindow, vadMaxMergeSize, vadPadding, vadPromptWindow, 
-                         progress: gr.Progress = None, **decodeOptions: dict):
+    def transcribe_webui(self, modelName, languageName, urlData, multipleFiles, microphoneData, task, 
+                         vadOptions: VadOptions, progress: gr.Progress = None, **decodeOptions: dict):
         try:
             sources = self.__get_source(urlData, multipleFiles, microphoneData)
             
@@ -146,7 +162,7 @@ class WhisperTranscriber:
                                                    sub_task_total=source_audio_duration)
 
                     # Transcribe
-                    result = self.transcribe_file(model, source.source_path, selectedLanguage, task, vad, vadMergeWindow, vadMaxMergeSize, vadPadding, vadPromptWindow, scaled_progress_listener, **decodeOptions)
+                    result = self.transcribe_file(model, source.source_path, selectedLanguage, task, vadOptions, scaled_progress_listener, **decodeOptions)
                     filePrefix = slugify(source_prefix + source.get_short_name(), allow_unicode=True)
 
                     # Update progress
@@ -210,8 +226,8 @@ class WhisperTranscriber:
         except ExceededMaximumDuration as e:
             return [], ("[ERROR]: Maximum remote video length is " + str(e.maxDuration) + "s, file was " + str(e.videoDuration) + "s"), "[ERROR]"
 
-    def transcribe_file(self, model: AbstractWhisperContainer, audio_path: str, language: str, task: str = None, vad: str = None, 
-                        vadMergeWindow: float = 5, vadMaxMergeSize: float = 150, vadPadding: float = 1, vadPromptWindow: float = 1, 
+    def transcribe_file(self, model: AbstractWhisperContainer, audio_path: str, language: str, task: str = None, 
+                        vadOptions: VadOptions = VadOptions(), 
                         progressListener: ProgressListener = None, **decodeOptions: dict):
         
         initial_prompt = decodeOptions.pop('initial_prompt', None)
@@ -224,26 +240,26 @@ class WhisperTranscriber:
             task = decodeOptions.pop('task')
 
         # Callable for processing an audio file
-        whisperCallable = model.create_callback(language, task, initial_prompt, **decodeOptions)
+        whisperCallable = model.create_callback(language, task, initial_prompt, initial_prompt_mode=vadOptions.vadInitialPromptMode, **decodeOptions)
 
         # The results
-        if (vad == 'silero-vad'):
+        if (vadOptions.vad == 'silero-vad'):
             # Silero VAD where non-speech gaps are transcribed
-            process_gaps = self._create_silero_config(NonSpeechStrategy.CREATE_SEGMENT, vadMergeWindow, vadMaxMergeSize, vadPadding, vadPromptWindow)
+            process_gaps = self._create_silero_config(NonSpeechStrategy.CREATE_SEGMENT, vadOptions)
             result = self.process_vad(audio_path, whisperCallable, self.vad_model, process_gaps, progressListener=progressListener)
-        elif (vad == 'silero-vad-skip-gaps'):
+        elif (vadOptions.vad == 'silero-vad-skip-gaps'):
             # Silero VAD where non-speech gaps are simply ignored
-            skip_gaps = self._create_silero_config(NonSpeechStrategy.SKIP, vadMergeWindow, vadMaxMergeSize, vadPadding, vadPromptWindow)
+            skip_gaps = self._create_silero_config(NonSpeechStrategy.SKIP, vadOptions)
             result = self.process_vad(audio_path, whisperCallable, self.vad_model, skip_gaps, progressListener=progressListener)
-        elif (vad == 'silero-vad-expand-into-gaps'):
+        elif (vadOptions.vad == 'silero-vad-expand-into-gaps'):
             # Use Silero VAD where speech-segments are expanded into non-speech gaps
-            expand_gaps = self._create_silero_config(NonSpeechStrategy.EXPAND_SEGMENT, vadMergeWindow, vadMaxMergeSize, vadPadding, vadPromptWindow)
+            expand_gaps = self._create_silero_config(NonSpeechStrategy.EXPAND_SEGMENT, vadOptions)
             result = self.process_vad(audio_path, whisperCallable, self.vad_model, expand_gaps, progressListener=progressListener)
-        elif (vad == 'periodic-vad'):
+        elif (vadOptions.vad == 'periodic-vad'):
             # Very simple VAD - mark every 5 minutes as speech. This makes it less likely that Whisper enters an infinite loop, but
             # it may create a break in the middle of a sentence, causing some artifacts.
             periodic_vad = VadPeriodicTranscription()
-            period_config = PeriodicTranscriptionConfig(periodic_duration=vadMaxMergeSize, max_prompt_window=vadPromptWindow)
+            period_config = PeriodicTranscriptionConfig(periodic_duration=vadOptions.vadMaxMergeSize, max_prompt_window=vadOptions.vadPromptWindow)
             result = self.process_vad(audio_path, whisperCallable, periodic_vad, period_config, progressListener=progressListener)
 
         else:
@@ -314,15 +330,15 @@ class WhisperTranscriber:
         else:
             return prompt1 + " " + prompt2
 
-    def _create_silero_config(self, non_speech_strategy: NonSpeechStrategy, vadMergeWindow: float = 5, vadMaxMergeSize: float = 150, vadPadding: float = 1, vadPromptWindow: float = 1):
+    def _create_silero_config(self, non_speech_strategy: NonSpeechStrategy, vadOptions: VadOptions):
         # Use Silero VAD 
         if (self.vad_model is None):
             self.vad_model = VadSileroTranscription()
 
         config = TranscriptionConfig(non_speech_strategy = non_speech_strategy, 
-                max_silent_period=vadMergeWindow, max_merge_size=vadMaxMergeSize, 
-                segment_padding_left=vadPadding, segment_padding_right=vadPadding, 
-                max_prompt_window=vadPromptWindow)
+                max_silent_period=vadOptions.vadMergeWindow, max_merge_size=vadOptions.vadMaxMergeSize, 
+                segment_padding_left=vadOptions.vadPadding, segment_padding_right=vadOptions.vadPadding, 
+                max_prompt_window=vadOptions.vadPromptWindow)
 
         return config
 
@@ -451,6 +467,7 @@ def create_ui(app_config: ApplicationConfig):
 
     full_transcribe = gr.Interface(fn=ui.transcribe_webui_full, description=full_description, article=ui_article, inputs=[
         *simple_inputs(),
+        gr.Dropdown(choices=["prepend_first_segment", "prepend_all_segments"], value=app_config.vad_initial_prompt_mode, label="VAD - Initial Prompt Mode"),
         gr.TextArea(label="Initial Prompt"),
         gr.Number(label="Temperature", value=app_config.temperature),
         gr.Number(label="Best Of - Non-zero temperature", value=app_config.best_of, precision=0),
@@ -503,6 +520,8 @@ if __name__ == '__main__':
                         help="The default model name.") # medium
     parser.add_argument("--default_vad", type=str, default=default_app_config.default_vad, \
                         help="The default VAD.") # silero-vad
+    parser.add_argument("--vad_initial_prompt_mode", type=str, default=default_app_config.vad_initial_prompt_mode, choices=["prepend_all_segments", "prepend_first_segment"], \
+                        help="Whether or not to prepend the initial prompt to each VAD segment (prepend_all_segments), or just the first segment (prepend_first_segment)") # prepend_first_segment
     parser.add_argument("--vad_parallel_devices", type=str, default=default_app_config.vad_parallel_devices, \
                         help="A commma delimited list of CUDA devices to use for parallel processing. If None, disable parallel processing.") # ""
     parser.add_argument("--vad_cpu_cores", type=int, default=default_app_config.vad_cpu_cores, \

cli.py

@@ -6,8 +6,8 @@ import warnings
 import numpy as np
 
 import torch
-from app import WhisperTranscriber
-from src.config import ApplicationConfig
+from app import VadOptions, WhisperTranscriber
+from src.config import ApplicationConfig, VadInitialPromptMode
 from src.download import download_url
 from src.languages import get_language_names
 
@@ -47,6 +47,8 @@ def cli():
 
     parser.add_argument("--vad", type=str, default=app_config.default_vad, choices=["none", "silero-vad", "silero-vad-skip-gaps", "silero-vad-expand-into-gaps", "periodic-vad"], \
                         help="The voice activity detection algorithm to use") # silero-vad
+    parser.add_argument("--vad_initial_prompt_mode", type=str, default=app_config.vad_initial_prompt_mode, choices=["prepend_all_segments", "prepend_first_segment"], \
+                        help="Whether or not to prepend the initial prompt to each VAD segment (prepend_all_segments), or just the first segment (prepend_first_segment)") # prepend_first_segment
     parser.add_argument("--vad_merge_window", type=optional_float, default=app_config.vad_merge_window, \
                         help="The window size (in seconds) to merge voice segments")
     parser.add_argument("--vad_max_merge_size", type=optional_float, default=app_config.vad_max_merge_size,\
@@ -115,6 +117,7 @@ def cli():
         temperature = [temperature]
 
     vad = args.pop("vad")
+    vad_initial_prompt_mode = args.pop("vad_initial_prompt_mode")
     vad_merge_window = args.pop("vad_merge_window")
     vad_max_merge_size = args.pop("vad_max_merge_size")
     vad_padding = args.pop("vad_padding")
@@ -150,9 +153,10 @@ def cli():
             source_path = source["path"]
             source_name = source["name"]
 
-            result = transcriber.transcribe_file(model, source_path, temperature=temperature, 
-                                                vad=vad, vadMergeWindow=vad_merge_window, vadMaxMergeSize=vad_max_merge_size, 
-                                                vadPadding=vad_padding, vadPromptWindow=vad_prompt_window, **args)
+            vadOptions = VadOptions(vad, vad_merge_window, vad_max_merge_size, vad_padding, vad_prompt_window, 
+                                    VadInitialPromptMode.from_string(vad_initial_prompt_mode))
+
+            result = transcriber.transcribe_file(model, source_path, temperature=temperature, vadOptions=vadOptions, **args)
             
             transcriber.write_result(result, source_name, output_dir)
 

config.json5

@@ -97,6 +97,8 @@
     "vad_max_merge_size": 30,
     // The padding (in seconds) to add to each voice segment
     "vad_padding": 1,
+    // Whether or not to prepend the initial prompt to each VAD segment (prepend_all_segments), or just the first segment (prepend_first_segment)
+    "vad_initial_prompt_mode": "prepend_first_segment",
     // The window size of the prompt to pass to Whisper
     "vad_prompt_window": 3,
     // Temperature to use for sampling

src/config.py

@@ -1,3 +1,4 @@
+from enum import Enum
 import urllib
 
 import os
@@ -23,6 +24,21 @@ class ModelConfig:
         self.path = path
         self.type = type
 
+class VadInitialPromptMode(Enum):
+    PREPEND_ALL_SEGMENTS = 1
+    PREPREND_FIRST_SEGMENT = 2
+
+    @staticmethod
+    def from_string(s: str):
+        normalized = s.lower() if s is not None else None
+
+        if normalized == "prepend_all_segments":
+            return VadInitialPromptMode.PREPEND_ALL_SEGMENTS
+        elif normalized == "prepend_first_segment":
+            return VadInitialPromptMode.PREPREND_FIRST_SEGMENT
+        else:
+            raise ValueError(f"Invalid value for VadInitialPromptMode: {s}")
+
 class ApplicationConfig:
     def __init__(self, models: List[ModelConfig] = [], input_audio_max_duration: int = 600, 
                  share: bool = False, server_name: str = None, server_port: int = 7860, 
@@ -33,6 +49,7 @@ class ApplicationConfig:
                  auto_parallel: bool = False, output_dir: str = None,
                  model_dir: str = None, device: str = None, 
                  verbose: bool = True, task: str = "transcribe", language: str = None,
+                 vad_initial_prompt_mode: str = "prepend_first_segment ", 
                  vad_merge_window: float = 5, vad_max_merge_size: float = 30,
                  vad_padding: float = 1, vad_prompt_window: float = 3,
                  temperature: float = 0, best_of: int = 5, beam_size: int = 5,
@@ -67,6 +84,7 @@ class ApplicationConfig:
         self.verbose = verbose
         self.task = task
         self.language = language
+        self.vad_initial_prompt_mode = vad_initial_prompt_mode
         self.vad_merge_window = vad_merge_window
         self.vad_max_merge_size = vad_max_merge_size
         self.vad_padding = vad_padding

src/whisper/abstractWhisperContainer.py

@@ -1,6 +1,6 @@
 import abc
 from typing import List
-from src.config import ModelConfig
+from src.config import ModelConfig, VadInitialPromptMode
 
 from src.hooks.progressListener import ProgressListener
 from src.modelCache import GLOBAL_MODEL_CACHE, ModelCache
@@ -24,6 +24,15 @@ class AbstractWhisperCallback:
         """
         raise NotImplementedError()
 
+    def _get_initial_prompt(self, initial_prompt: str, initial_prompt_mode: VadInitialPromptMode, 
+                               prompt: str, segment_index: int):
+        if (initial_prompt_mode == VadInitialPromptMode.PREPEND_ALL_SEGMENTS):
+            return self._concat_prompt(initial_prompt, prompt)
+        elif (initial_prompt_mode == VadInitialPromptMode.PREPREND_FIRST_SEGMENT):
+            return self._concat_prompt(initial_prompt, prompt) if segment_index == 0 else prompt
+        else:
+            raise ValueError(f"Unknown initial prompt mode {initial_prompt_mode}")
+
     def _concat_prompt(self, prompt1, prompt2):
         if (prompt1 is None):
             return prompt2
@@ -66,7 +75,9 @@ class AbstractWhisperContainer:
         pass
 
     @abc.abstractmethod
-    def create_callback(self, language: str = None, task: str = None, initial_prompt: str = None, **decodeOptions: dict) -> AbstractWhisperCallback:
+    def create_callback(self, language: str = None, task: str = None, initial_prompt: str = None, 
+                        initial_prompt_mode: VadInitialPromptMode = VadInitialPromptMode.PREPREND_FIRST_SEGMENT, 
+                        **decodeOptions: dict) -> AbstractWhisperCallback:
         """
         Create a WhisperCallback object that can be used to transcript audio files.
 
@@ -78,6 +89,9 @@ class AbstractWhisperContainer:
             The task - either translate or transcribe.
         initial_prompt: str
             The initial prompt to use for the transcription.
+        initial_prompt_mode: VadInitialPromptMode
+            The mode to use for the initial prompt. If set to PREPEND_FIRST_SEGMENT, the initial prompt will be prepended to the first segment of audio.
+            If set to PREPEND_ALL_SEGMENTS, the initial prompt will be prepended to all segments of audio.
         decodeOptions: dict
             Additional options to pass to the decoder. Must be pickleable.
 

src/whisper/fasterWhisperContainer.py

@@ -2,7 +2,7 @@ import os
 from typing import List, Union
 
 from faster_whisper import WhisperModel, download_model
-from src.config import ModelConfig
+from src.config import ModelConfig, VadInitialPromptMode
 from src.hooks.progressListener import ProgressListener
 from src.languages import get_language_from_name
 from src.modelCache import ModelCache
@@ -51,7 +51,9 @@ class FasterWhisperContainer(AbstractWhisperContainer):
         model = WhisperModel(model_config.url, device=device, compute_type=self.compute_type)
         return model
 
-    def create_callback(self, language: str = None, task: str = None, initial_prompt: str = None, **decodeOptions: dict):
+    def create_callback(self, language: str = None, task: str = None, initial_prompt: str = None, 
+                        initial_prompt_mode: VadInitialPromptMode = VadInitialPromptMode.PREPREND_FIRST_SEGMENT, 
+                        **decodeOptions: dict) -> AbstractWhisperCallback:
         """
         Create a WhisperCallback object that can be used to transcript audio files.
 
@@ -63,6 +65,9 @@ class FasterWhisperContainer(AbstractWhisperContainer):
             The task - either translate or transcribe.
         initial_prompt: str
             The initial prompt to use for the transcription.
+        initial_prompt_mode: VadInitialPromptMode
+            The mode to use for the initial prompt. If set to PREPEND_FIRST_SEGMENT, the initial prompt will be prepended to the first segment of audio.
+            If set to PREPEND_ALL_SEGMENTS, the initial prompt will be prepended to all segments of audio.
         decodeOptions: dict
             Additional options to pass to the decoder. Must be pickleable.
 
@@ -70,14 +75,17 @@ class FasterWhisperContainer(AbstractWhisperContainer):
         -------
         A WhisperCallback object.
         """
-        return FasterWhisperCallback(self, language=language, task=task, initial_prompt=initial_prompt, **decodeOptions)
+        return FasterWhisperCallback(self, language=language, task=task, initial_prompt=initial_prompt, initial_prompt_mode=initial_prompt_mode, **decodeOptions)
 
 class FasterWhisperCallback(AbstractWhisperCallback):
-    def __init__(self, model_container: FasterWhisperContainer, language: str = None, task: str = None, initial_prompt: str = None, **decodeOptions: dict):
+    def __init__(self, model_container: FasterWhisperContainer, language: str = None, task: str = None, 
+                 initial_prompt: str = None, initial_prompt_mode: VadInitialPromptMode=VadInitialPromptMode.PREPREND_FIRST_SEGMENT, 
+                 **decodeOptions: dict):
         self.model_container = model_container
         self.language = language
         self.task = task
         self.initial_prompt = initial_prompt
+        self.initial_prompt_mode = initial_prompt_mode
         self.decodeOptions = decodeOptions
 
         self._printed_warning = False
@@ -125,9 +133,11 @@ class FasterWhisperCallback(AbstractWhisperCallback):
         # See if supress_tokens is a string - if so, convert it to a list of ints
         decodeOptions["suppress_tokens"] = self._split_suppress_tokens(suppress_tokens)
 
+        initial_prompt = self._get_initial_prompt(self.initial_prompt, self.initial_prompt_mode, prompt, segment_index)
+
         segments_generator, info = model.transcribe(audio, \
             language=language_code if language_code else detected_language, task=self.task, \
-            initial_prompt=self._concat_prompt(self.initial_prompt, prompt) if segment_index == 0 else prompt, \
+            initial_prompt=initial_prompt, \
             **decodeOptions
         )
 

src/whisper/whisperContainer.py

@@ -11,7 +11,7 @@ from src.hooks.progressListener import ProgressListener
 import whisper
 from whisper import Whisper
 
-from src.config import ModelConfig
+from src.config import ModelConfig, VadInitialPromptMode
 from src.hooks.whisperProgressHook import create_progress_listener_handle
 
 from src.modelCache import GLOBAL_MODEL_CACHE, ModelCache
@@ -69,7 +69,9 @@ class WhisperContainer(AbstractWhisperContainer):
 
         return whisper.load_model(model_path, device=self.device, download_root=self.download_root)
 
-    def create_callback(self, language: str = None, task: str = None, initial_prompt: str = None, **decodeOptions: dict):
+    def create_callback(self, language: str = None, task: str = None, initial_prompt: str = None, 
+                        initial_prompt_mode: VadInitialPromptMode = VadInitialPromptMode.PREPREND_FIRST_SEGMENT, 
+                        **decodeOptions: dict) -> AbstractWhisperCallback:
         """
         Create a WhisperCallback object that can be used to transcript audio files.
 
@@ -81,6 +83,9 @@ class WhisperContainer(AbstractWhisperContainer):
             The task - either translate or transcribe.
         initial_prompt: str
             The initial prompt to use for the transcription.
+        initial_prompt_mode: VadInitialPromptMode
+            The mode to use for the initial prompt. If set to PREPEND_FIRST_SEGMENT, the initial prompt will be prepended to the first segment of audio.
+            If set to PREPEND_ALL_SEGMENTS, the initial prompt will be prepended to all segments of audio.
         decodeOptions: dict
             Additional options to pass to the decoder. Must be pickleable.
 
@@ -88,7 +93,7 @@ class WhisperContainer(AbstractWhisperContainer):
         -------
         A WhisperCallback object.
         """
-        return WhisperCallback(self, language=language, task=task, initial_prompt=initial_prompt, **decodeOptions)
+        return WhisperCallback(self, language=language, task=task, initial_prompt=initial_prompt, initial_prompt_mode=initial_prompt_mode, **decodeOptions)
 
     def _get_model_path(self, model_config: ModelConfig, root_dir: str = None):
         from src.conversion.hf_converter import convert_hf_whisper
@@ -157,11 +162,13 @@ class WhisperContainer(AbstractWhisperContainer):
         return model_config.path
 
 class WhisperCallback(AbstractWhisperCallback):
-    def __init__(self, model_container: WhisperContainer, language: str = None, task: str = None, initial_prompt: str = None, **decodeOptions: dict):
+    def __init__(self, model_container: WhisperContainer, language: str = None, task: str = None, initial_prompt: str = None, 
+                 initial_prompt_mode: VadInitialPromptMode=VadInitialPromptMode.PREPREND_FIRST_SEGMENT, **decodeOptions: dict):
         self.model_container = model_container
         self.language = language
         self.task = task
         self.initial_prompt = initial_prompt
+        self.initial_prompt_mode = initial_prompt_mode
         self.decodeOptions = decodeOptions
         
     def invoke(self, audio, segment_index: int, prompt: str, detected_language: str, progress_listener: ProgressListener = None):
@@ -194,8 +201,10 @@ class WhisperCallback(AbstractWhisperCallback):
         if self.model_container.compute_type in ["fp16", "float16"]:
             decodeOptions["fp16"] = True
 
+        initial_prompt = self._get_initial_prompt(self.initial_prompt, self.initial_prompt_mode, prompt, segment_index)
+
         return model.transcribe(audio, \
             language=self.language if self.language else detected_language, task=self.task, \
-            initial_prompt=self._concat_prompt(self.initial_prompt, prompt) if segment_index == 0 else prompt, \
+            initial_prompt=initial_prompt, \
             **decodeOptions
         )