Upload 127 files

App_Function_Libraries/Audio/Audio_Transcription_Lib.py

@@ -1,284 +1,335 @@
-# Audio_Transcription_Lib.py
-#########################################
-# Transcription Library
-# This library is used to perform transcription of audio files.
-# Currently, uses faster_whisper for transcription.
-#
-####################
-# Function List
-#
-# 1. convert_to_wav(video_file_path, offset=0, overwrite=False)
-# 2. speech_to_text(audio_file_path, selected_source_lang='en', whisper_model='small.en', vad_filter=False)
-#
-####################
-#
-# Import necessary libraries to run solo for testing
-import gc
-import json
-import logging
-import multiprocessing
-import os
-import queue
-import sys
-import subprocess
-import tempfile
-import threading
-import time
-# DEBUG Imports
-#from memory_profiler import profile
-#import pyaudio
-from faster_whisper import WhisperModel as OriginalWhisperModel
-from typing import Optional, Union, List, Dict, Any
-#
-# Import Local
-from App_Function_Libraries.Utils.Utils import load_comprehensive_config
-from App_Function_Libraries.Metrics.metrics_logger import log_counter, log_histogram
-#
-#######################################################################################################################
-# Function Definitions
-#
-
-# Convert video .m4a into .wav using ffmpeg
-#   ffmpeg -i "example.mp4" -ar 16000 -ac 1 -c:a pcm_s16le "output.wav"
-#       https://www.gyan.dev/ffmpeg/builds/
-#
-
-
-whisper_model_instance = None
-config = load_comprehensive_config()
-processing_choice = config.get('Processing', 'processing_choice', fallback='cpu')
-total_thread_count = multiprocessing.cpu_count()
-
-
-class WhisperModel(OriginalWhisperModel):
-    tldw_dir = os.path.dirname(os.path.dirname(__file__))
-    default_download_root = os.path.join(tldw_dir, 'models', 'Whisper')
-
-    valid_model_sizes = [
-        "tiny.en", "tiny", "base.en", "base", "small.en", "small", "medium.en", "medium",
-        "large-v1", "large-v2", "large-v3", "large", "distil-large-v2", "distil-medium.en",
-        "distil-small.en", "distil-large-v3",
-    ]
-
-    def __init__(
-        self,
-        model_size_or_path: str,
-        device: str = processing_choice,
-        device_index: Union[int, List[int]] = 0,
-        compute_type: str = "default",
-        cpu_threads: int = 0,#total_thread_count, FIXME - I think this should be 0
-        num_workers: int = 1,
-        download_root: Optional[str] = None,
-        local_files_only: bool = False,
-        files: Optional[Dict[str, Any]] = None,
-        **model_kwargs: Any
-    ):
-        if download_root is None:
-            download_root = self.default_download_root
-
-        os.makedirs(download_root, exist_ok=True)
-
-        # FIXME - validate....
-        # Also write an integration test...
-        # Check if model_size_or_path is a valid model size
-        if model_size_or_path in self.valid_model_sizes:
-            # It's a model size, so we'll use the download_root
-            model_path = os.path.join(download_root, model_size_or_path)
-            if not os.path.isdir(model_path):
-                # If it doesn't exist, we'll let the parent class download it
-                model_size_or_path = model_size_or_path  # Keep the original model size
-            else:
-                # If it exists, use the full path
-                model_size_or_path = model_path
-        else:
-            # It's not a valid model size, so assume it's a path
-            model_size_or_path = os.path.abspath(model_size_or_path)
-
-        super().__init__(
-            model_size_or_path,
-            device=device,
-            device_index=device_index,
-            compute_type=compute_type,
-            cpu_threads=cpu_threads,
-            num_workers=num_workers,
-            download_root=download_root,
-            local_files_only=local_files_only,
-# Maybe? idk, FIXME
-#            files=files,
-#            **model_kwargs
-        )
-
-def get_whisper_model(model_name, device):
-    global whisper_model_instance
-    if whisper_model_instance is None:
-        logging.info(f"Initializing new WhisperModel with size {model_name} on device {device}")
-        whisper_model_instance = WhisperModel(model_name, device=device)
-    return whisper_model_instance
-
-# os.system(r'.\Bin\ffmpeg.exe -ss 00:00:00 -i "{video_file_path}" -ar 16000 -ac 1 -c:a pcm_s16le "{out_path}"')
-#DEBUG
-#@profile
-def convert_to_wav(video_file_path, offset=0, overwrite=False):
-    log_counter("convert_to_wav_attempt", labels={"file_path": video_file_path})
-    start_time = time.time()
-
-    out_path = os.path.splitext(video_file_path)[0] + ".wav"
-
-    if os.path.exists(out_path) and not overwrite:
-        print(f"File '{out_path}' already exists. Skipping conversion.")
-        logging.info(f"Skipping conversion as file already exists: {out_path}")
-        log_counter("convert_to_wav_skipped", labels={"file_path": video_file_path})
-        return out_path
-
-    print("Starting conversion process of .m4a to .WAV")
-    out_path = os.path.splitext(video_file_path)[0] + ".wav"
-
-    try:
-        if os.name == "nt":
-            logging.debug("ffmpeg being ran on windows")
-
-            if sys.platform.startswith('win'):
-                ffmpeg_cmd = ".\\Bin\\ffmpeg.exe"
-                logging.debug(f"ffmpeg_cmd: {ffmpeg_cmd}")
-            else:
-                ffmpeg_cmd = 'ffmpeg'  # Assume 'ffmpeg' is in PATH for non-Windows systems
-
-            command = [
-                ffmpeg_cmd,  # Assuming the working directory is correctly set where .\Bin exists
-                "-ss", "00:00:00",  # Start at the beginning of the video
-                "-i", video_file_path,
-                "-ar", "16000",  # Audio sample rate
-                "-ac", "1",  # Number of audio channels
-                "-c:a", "pcm_s16le",  # Audio codec
-                out_path
-            ]
-            try:
-                # Redirect stdin from null device to prevent ffmpeg from waiting for input
-                with open(os.devnull, 'rb') as null_file:
-                    result = subprocess.run(command, stdin=null_file, text=True, capture_output=True)
-                if result.returncode == 0:
-                    logging.info("FFmpeg executed successfully")
-                    logging.debug("FFmpeg output: %s", result.stdout)
-                else:
-                    logging.error("Error in running FFmpeg")
-                    logging.error("FFmpeg stderr: %s", result.stderr)
-                    raise RuntimeError(f"FFmpeg error: {result.stderr}")
-            except Exception as e:
-                logging.error("Error occurred - ffmpeg doesn't like windows")
-                raise RuntimeError("ffmpeg failed")
-        elif os.name == "posix":
-            os.system(f'ffmpeg -ss 00:00:00 -i "{video_file_path}" -ar 16000 -ac 1 -c:a pcm_s16le "{out_path}"')
-        else:
-            raise RuntimeError("Unsupported operating system")
-        logging.info("Conversion to WAV completed: %s", out_path)
-        log_counter("convert_to_wav_success", labels={"file_path": video_file_path})
-    except Exception as e:
-        logging.error("speech-to-text: Error transcribing audio: %s", str(e))
-        log_counter("convert_to_wav_error", labels={"file_path": video_file_path, "error": str(e)})
-        return {"error": str(e)}
-
-    conversion_time = time.time() - start_time
-    log_histogram("convert_to_wav_duration", conversion_time, labels={"file_path": video_file_path})
-
-    gc.collect()
-    return out_path
-
-
-# Transcribe .wav into .segments.json
-#DEBUG
-#@profile
-# FIXME - I feel like the `vad_filter` shoudl be enabled by default....
-def speech_to_text(audio_file_path, selected_source_lang='en', whisper_model='medium.en', vad_filter=False, diarize=False):
-    log_counter("speech_to_text_attempt", labels={"file_path": audio_file_path, "model": whisper_model})
-    time_start = time.time()
-
-    if audio_file_path is None:
-        log_counter("speech_to_text_error", labels={"error": "No audio file provided"})
-        raise ValueError("speech-to-text: No audio file provided")
-    logging.info("speech-to-text: Audio file path: %s", audio_file_path)
-
-    try:
-        _, file_ending = os.path.splitext(audio_file_path)
-        out_file = audio_file_path.replace(file_ending, "-whisper_model-"+whisper_model+".segments.json")
-        prettified_out_file = audio_file_path.replace(file_ending, "-whisper_model-"+whisper_model+".segments_pretty.json")
-        if os.path.exists(out_file):
-            logging.info("speech-to-text: Segments file already exists: %s", out_file)
-            with open(out_file) as f:
-                global segments
-                segments = json.load(f)
-            return segments
-
-        logging.info('speech-to-text: Starting transcription...')
-        # FIXME - revisit this
-        options = dict(language=selected_source_lang, beam_size=10, best_of=10, vad_filter=vad_filter)
-        transcribe_options = dict(task="transcribe", **options)
-        # use function and config at top of file
-        logging.debug("speech-to-text: Using whisper model: %s", whisper_model)
-        whisper_model_instance = get_whisper_model(whisper_model, processing_choice)
-        # faster_whisper transcription right here - FIXME -test batching - ha
-        segments_raw, info = whisper_model_instance.transcribe(audio_file_path, **transcribe_options)
-
-        segments = []
-        for segment_chunk in segments_raw:
-            chunk = {
-                "Time_Start": segment_chunk.start,
-                "Time_End": segment_chunk.end,
-                "Text": segment_chunk.text
-            }
-            logging.debug("Segment: %s", chunk)
-            segments.append(chunk)
-            # Print to verify its working
-            logging.info(f"{segment_chunk.start:.2f}s - {segment_chunk.end:.2f}s | {segment_chunk.text}")
-
-            # Log it as well.
-            logging.debug(
-                f"Transcribed Segment: {segment_chunk.start:.2f}s - {segment_chunk.end:.2f}s | {segment_chunk.text}")
-
-        if segments:
-            segments[0]["Text"] = f"This text was transcribed using whisper model: {whisper_model}\n\n" + segments[0]["Text"]
-
-        if not segments:
-            log_counter("speech_to_text_error", labels={"error": "No transcription produced"})
-            raise RuntimeError("No transcription produced. The audio file may be invalid or empty.")
-
-        transcription_time = time.time() - time_start
-        logging.info("speech-to-text: Transcription completed in %.2f seconds", transcription_time)
-        log_histogram("speech_to_text_duration", transcription_time, labels={"file_path": audio_file_path, "model": whisper_model})
-        log_counter("speech_to_text_success", labels={"file_path": audio_file_path, "model": whisper_model})
-        # Save the segments to a JSON file - prettified and non-prettified
-        # FIXME refactor so this is an optional flag to save either the prettified json file or the normal one
-        save_json = True
-        if save_json:
-            logging.info("speech-to-text: Saving segments to JSON file")
-            output_data = {'segments': segments}
-            logging.info("speech-to-text: Saving prettified JSON to %s", prettified_out_file)
-            with open(prettified_out_file, 'w') as f:
-                json.dump(output_data, f, indent=2)
-
-            logging.info("speech-to-text: Saving JSON to %s", out_file)
-            with open(out_file, 'w') as f:
-                json.dump(output_data, f)
-
-        logging.debug(f"speech-to-text: returning {segments[:500]}")
-        gc.collect()
-        return segments
-
-    except Exception as e:
-        logging.error("speech-to-text: Error transcribing audio: %s", str(e))
-        log_counter("speech_to_text_error", labels={"file_path": audio_file_path, "model": whisper_model, "error": str(e)})
-        raise RuntimeError("speech-to-text: Error transcribing audio")
-
-
-def record_audio(duration, sample_rate=16000, chunk_size=1024):
-    pass
-
-
-def stop_recording(p, stream, audio_queue, stop_recording_event, audio_thread):
-    pass
-
-def save_audio_temp(audio_data, sample_rate=16000):
-    pass
-
-#
-#
+# Audio_Transcription_Lib.py
+#########################################
+# Transcription Library
+# This library is used to perform transcription of audio files.
+# Currently, uses faster_whisper for transcription.
+#
+####################
+# Function List
+#
+# 1. convert_to_wav(video_file_path, offset=0, overwrite=False)
+# 2. speech_to_text(audio_file_path, selected_source_lang='en', whisper_model='small.en', vad_filter=False)
+#
+####################
+#
+# Import necessary libraries to run solo for testing
+import gc
+import json
+import logging
+import multiprocessing
+import os
+import queue
+import sys
+import subprocess
+import tempfile
+import threading
+import time
+# DEBUG Imports
+#from memory_profiler import profile
+import pyaudio
+from faster_whisper import WhisperModel as OriginalWhisperModel
+from typing import Optional, Union, List, Dict, Any
+#
+# Import Local
+from App_Function_Libraries.Utils.Utils import load_comprehensive_config
+from App_Function_Libraries.Metrics.metrics_logger import log_counter, log_histogram
+#
+#######################################################################################################################
+# Function Definitions
+#
+
+# Convert video .m4a into .wav using ffmpeg
+#   ffmpeg -i "example.mp4" -ar 16000 -ac 1 -c:a pcm_s16le "output.wav"
+#       https://www.gyan.dev/ffmpeg/builds/
+#
+
+
+whisper_model_instance = None
+config = load_comprehensive_config()
+processing_choice = config.get('Processing', 'processing_choice', fallback='cpu')
+total_thread_count = multiprocessing.cpu_count()
+
+
+class WhisperModel(OriginalWhisperModel):
+    tldw_dir = os.path.dirname(os.path.dirname(__file__))
+    default_download_root = os.path.join(tldw_dir, 'models', 'Whisper')
+
+    valid_model_sizes = [
+        "tiny.en", "tiny", "base.en", "base", "small.en", "small", "medium.en", "medium",
+        "large-v1", "large-v2", "large-v3", "large", "distil-large-v2", "distil-medium.en",
+        "distil-small.en", "distil-large-v3",
+    ]
+
+    def __init__(
+        self,
+        model_size_or_path: str,
+        device: str = processing_choice,
+        device_index: Union[int, List[int]] = 0,
+        compute_type: str = "default",
+        cpu_threads: int = 0,#total_thread_count, FIXME - I think this should be 0
+        num_workers: int = 1,
+        download_root: Optional[str] = None,
+        local_files_only: bool = False,
+        files: Optional[Dict[str, Any]] = None,
+        **model_kwargs: Any
+    ):
+        if download_root is None:
+            download_root = self.default_download_root
+
+        os.makedirs(download_root, exist_ok=True)
+
+        # FIXME - validate....
+        # Also write an integration test...
+        # Check if model_size_or_path is a valid model size
+        if model_size_or_path in self.valid_model_sizes:
+            # It's a model size, so we'll use the download_root
+            model_path = os.path.join(download_root, model_size_or_path)
+            if not os.path.isdir(model_path):
+                # If it doesn't exist, we'll let the parent class download it
+                model_size_or_path = model_size_or_path  # Keep the original model size
+            else:
+                # If it exists, use the full path
+                model_size_or_path = model_path
+        else:
+            # It's not a valid model size, so assume it's a path
+            model_size_or_path = os.path.abspath(model_size_or_path)
+
+        super().__init__(
+            model_size_or_path,
+            device=device,
+            device_index=device_index,
+            compute_type=compute_type,
+            cpu_threads=cpu_threads,
+            num_workers=num_workers,
+            download_root=download_root,
+            local_files_only=local_files_only,
+# Maybe? idk, FIXME
+#            files=files,
+#            **model_kwargs
+        )
+
+def get_whisper_model(model_name, device):
+    global whisper_model_instance
+    if whisper_model_instance is None:
+        logging.info(f"Initializing new WhisperModel with size {model_name} on device {device}")
+        whisper_model_instance = WhisperModel(model_name, device=device)
+    return whisper_model_instance
+
+# os.system(r'.\Bin\ffmpeg.exe -ss 00:00:00 -i "{video_file_path}" -ar 16000 -ac 1 -c:a pcm_s16le "{out_path}"')
+#DEBUG
+#@profile
+def convert_to_wav(video_file_path, offset=0, overwrite=False):
+    log_counter("convert_to_wav_attempt", labels={"file_path": video_file_path})
+    start_time = time.time()
+
+    out_path = os.path.splitext(video_file_path)[0] + ".wav"
+
+    if os.path.exists(out_path) and not overwrite:
+        print(f"File '{out_path}' already exists. Skipping conversion.")
+        logging.info(f"Skipping conversion as file already exists: {out_path}")
+        log_counter("convert_to_wav_skipped", labels={"file_path": video_file_path})
+        return out_path
+
+    print("Starting conversion process of .m4a to .WAV")
+    out_path = os.path.splitext(video_file_path)[0] + ".wav"
+
+    try:
+        if os.name == "nt":
+            logging.debug("ffmpeg being ran on windows")
+
+            if sys.platform.startswith('win'):
+                ffmpeg_cmd = ".\\Bin\\ffmpeg.exe"
+                logging.debug(f"ffmpeg_cmd: {ffmpeg_cmd}")
+            else:
+                ffmpeg_cmd = 'ffmpeg'  # Assume 'ffmpeg' is in PATH for non-Windows systems
+
+            command = [
+                ffmpeg_cmd,  # Assuming the working directory is correctly set where .\Bin exists
+                "-ss", "00:00:00",  # Start at the beginning of the video
+                "-i", video_file_path,
+                "-ar", "16000",  # Audio sample rate
+                "-ac", "1",  # Number of audio channels
+                "-c:a", "pcm_s16le",  # Audio codec
+                out_path
+            ]
+            try:
+                # Redirect stdin from null device to prevent ffmpeg from waiting for input
+                with open(os.devnull, 'rb') as null_file:
+                    result = subprocess.run(command, stdin=null_file, text=True, capture_output=True)
+                if result.returncode == 0:
+                    logging.info("FFmpeg executed successfully")
+                    logging.debug("FFmpeg output: %s", result.stdout)
+                else:
+                    logging.error("Error in running FFmpeg")
+                    logging.error("FFmpeg stderr: %s", result.stderr)
+                    raise RuntimeError(f"FFmpeg error: {result.stderr}")
+            except Exception as e:
+                logging.error("Error occurred - ffmpeg doesn't like windows")
+                raise RuntimeError("ffmpeg failed")
+        elif os.name == "posix":
+            os.system(f'ffmpeg -ss 00:00:00 -i "{video_file_path}" -ar 16000 -ac 1 -c:a pcm_s16le "{out_path}"')
+        else:
+            raise RuntimeError("Unsupported operating system")
+        logging.info("Conversion to WAV completed: %s", out_path)
+        log_counter("convert_to_wav_success", labels={"file_path": video_file_path})
+    except Exception as e:
+        logging.error("speech-to-text: Error transcribing audio: %s", str(e))
+        log_counter("convert_to_wav_error", labels={"file_path": video_file_path, "error": str(e)})
+        return {"error": str(e)}
+
+    conversion_time = time.time() - start_time
+    log_histogram("convert_to_wav_duration", conversion_time, labels={"file_path": video_file_path})
+
+    gc.collect()
+    return out_path
+
+
+# Transcribe .wav into .segments.json
+#DEBUG
+#@profile
+# FIXME - I feel like the `vad_filter` shoudl be enabled by default....
+def speech_to_text(audio_file_path, selected_source_lang='en', whisper_model='medium.en', vad_filter=False, diarize=False):
+    log_counter("speech_to_text_attempt", labels={"file_path": audio_file_path, "model": whisper_model})
+    time_start = time.time()
+
+    if audio_file_path is None:
+        log_counter("speech_to_text_error", labels={"error": "No audio file provided"})
+        raise ValueError("speech-to-text: No audio file provided")
+    logging.info("speech-to-text: Audio file path: %s", audio_file_path)
+
+    try:
+        _, file_ending = os.path.splitext(audio_file_path)
+        out_file = audio_file_path.replace(file_ending, "-whisper_model-"+whisper_model+".segments.json")
+        prettified_out_file = audio_file_path.replace(file_ending, "-whisper_model-"+whisper_model+".segments_pretty.json")
+        if os.path.exists(out_file):
+            logging.info("speech-to-text: Segments file already exists: %s", out_file)
+            with open(out_file) as f:
+                global segments
+                segments = json.load(f)
+            return segments
+
+        logging.info('speech-to-text: Starting transcription...')
+        # FIXME - revisit this
+        options = dict(language=selected_source_lang, beam_size=10, best_of=10, vad_filter=vad_filter)
+        transcribe_options = dict(task="transcribe", **options)
+        # use function and config at top of file
+        logging.debug("speech-to-text: Using whisper model: %s", whisper_model)
+        whisper_model_instance = get_whisper_model(whisper_model, processing_choice)
+        # faster_whisper transcription right here - FIXME -test batching - ha
+        segments_raw, info = whisper_model_instance.transcribe(audio_file_path, **transcribe_options)
+
+        segments = []
+        for segment_chunk in segments_raw:
+            chunk = {
+                "Time_Start": segment_chunk.start,
+                "Time_End": segment_chunk.end,
+                "Text": segment_chunk.text
+            }
+            logging.debug("Segment: %s", chunk)
+            segments.append(chunk)
+            # Print to verify its working
+            logging.info(f"{segment_chunk.start:.2f}s - {segment_chunk.end:.2f}s | {segment_chunk.text}")
+
+            # Log it as well.
+            logging.debug(
+                f"Transcribed Segment: {segment_chunk.start:.2f}s - {segment_chunk.end:.2f}s | {segment_chunk.text}")
+
+        if segments:
+            segments[0]["Text"] = f"This text was transcribed using whisper model: {whisper_model}\n\n" + segments[0]["Text"]
+
+        if not segments:
+            log_counter("speech_to_text_error", labels={"error": "No transcription produced"})
+            raise RuntimeError("No transcription produced. The audio file may be invalid or empty.")
+
+        transcription_time = time.time() - time_start
+        logging.info("speech-to-text: Transcription completed in %.2f seconds", transcription_time)
+        log_histogram("speech_to_text_duration", transcription_time, labels={"file_path": audio_file_path, "model": whisper_model})
+        log_counter("speech_to_text_success", labels={"file_path": audio_file_path, "model": whisper_model})
+        # Save the segments to a JSON file - prettified and non-prettified
+        # FIXME refactor so this is an optional flag to save either the prettified json file or the normal one
+        save_json = True
+        if save_json:
+            logging.info("speech-to-text: Saving segments to JSON file")
+            output_data = {'segments': segments}
+            logging.info("speech-to-text: Saving prettified JSON to %s", prettified_out_file)
+            with open(prettified_out_file, 'w') as f:
+                json.dump(output_data, f, indent=2)
+
+            logging.info("speech-to-text: Saving JSON to %s", out_file)
+            with open(out_file, 'w') as f:
+                json.dump(output_data, f)
+
+        logging.debug(f"speech-to-text: returning {segments[:500]}")
+        gc.collect()
+        return segments
+
+    except Exception as e:
+        logging.error("speech-to-text: Error transcribing audio: %s", str(e))
+        log_counter("speech_to_text_error", labels={"file_path": audio_file_path, "model": whisper_model, "error": str(e)})
+        raise RuntimeError("speech-to-text: Error transcribing audio")
+
+
+def record_audio(duration, sample_rate=16000, chunk_size=1024):
+    log_counter("record_audio_attempt", labels={"duration": duration})
+    p = pyaudio.PyAudio()
+    stream = p.open(format=pyaudio.paInt16,
+                    channels=1,
+                    rate=sample_rate,
+                    input=True,
+                    frames_per_buffer=chunk_size)
+
+    print("Recording...")
+    frames = []
+    stop_recording = threading.Event()
+    audio_queue = queue.Queue()
+
+    def audio_callback():
+        for _ in range(0, int(sample_rate / chunk_size * duration)):
+            if stop_recording.is_set():
+                break
+            data = stream.read(chunk_size)
+            audio_queue.put(data)
+
+    audio_thread = threading.Thread(target=audio_callback)
+    audio_thread.start()
+
+    return p, stream, audio_queue, stop_recording, audio_thread
+
+
+def stop_recording(p, stream, audio_queue, stop_recording_event, audio_thread):
+    log_counter("stop_recording_attempt")
+    start_time = time.time()
+    stop_recording_event.set()
+    audio_thread.join()
+
+    frames = []
+    while not audio_queue.empty():
+        frames.append(audio_queue.get())
+
+    print("Recording finished.")
+
+    stream.stop_stream()
+    stream.close()
+    p.terminate()
+
+    stop_time = time.time() - start_time
+    log_histogram("stop_recording_duration", stop_time)
+    log_counter("stop_recording_success")
+    return b''.join(frames)
+
+def save_audio_temp(audio_data, sample_rate=16000):
+    log_counter("save_audio_temp_attempt")
+    with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as temp_file:
+        import wave
+        wf = wave.open(temp_file.name, 'wb')
+        wf.setnchannels(1)
+        wf.setsampwidth(2)
+        wf.setframerate(sample_rate)
+        wf.writeframes(audio_data)
+        wf.close()
+        log_counter("save_audio_temp_success")
+        return temp_file.name
+
+#
+#
 #######################################################################################################################

App_Function_Libraries/Benchmarks_Evaluations/ms_g_eval.py

@@ -259,7 +259,7 @@ def run_geval(transcript: str, summary: str, api_key: str, api_name: str = None,
 
 
 def create_geval_tab():
-    with gr.Tab("G-Eval"):
+    with gr.Tab("G-Eval", id="g-eval"):
         gr.Markdown("# G-Eval Summarization Evaluation")
         with gr.Row():
             with gr.Column():

App_Function_Libraries/Books/Book_Ingestion_Lib.py

@@ -11,32 +11,42 @@
 #
 ####################
 #
-# Import necessary libraries
+# Imports
 import os
 import re
 import tempfile
 import zipfile
 from datetime import datetime
 import logging
-
+#
+# External Imports
 import ebooklib
 from bs4 import BeautifulSoup
 from ebooklib import epub
-
-from App_Function_Libraries.Chunk_Lib import chunk_ebook_by_chapters
 #
 # Import Local
 from App_Function_Libraries.DB.DB_Manager import add_media_with_keywords, add_media_to_database
 from App_Function_Libraries.Summarization.Summarization_General_Lib import perform_summarization
-
-
+from App_Function_Libraries.Chunk_Lib import chunk_ebook_by_chapters
+from App_Function_Libraries.Metrics.metrics_logger import log_counter, log_histogram
 #
 #######################################################################################################################
 # Function Definitions
 #
 
-def import_epub(file_path, title=None, author=None, keywords=None, custom_prompt=None, system_prompt=None, summary=None,
-               auto_summarize=False, api_name=None, api_key=None, chunk_options=None, custom_chapter_pattern=None):
+def import_epub(file_path,
+                title=None,
+                author=None,
+                keywords=None,
+                custom_prompt=None,
+                system_prompt=None,
+                summary=None,
+                auto_summarize=False,
+                api_name=None,
+                api_key=None,
+                chunk_options=None,
+                custom_chapter_pattern=None
+                ):
     """
     Imports an EPUB file, extracts its content, chunks it, optionally summarizes it, and adds it to the database.
 
@@ -58,6 +68,9 @@ def import_epub(file_path, title=None, author=None, keywords=None, custom_prompt
     """
     try:
         logging.info(f"Importing EPUB file from {file_path}")
+        log_counter("epub_import_attempt", labels={"file_path": file_path})
+
+        start_time = datetime.now()
 
         # Convert EPUB to Markdown
         markdown_content = epub_to_markdown(file_path)
@@ -90,10 +103,11 @@ def import_epub(file_path, title=None, author=None, keywords=None, custom_prompt
         # Chunk the content by chapters
         chunks = chunk_ebook_by_chapters(markdown_content, chunk_options)
         logging.info(f"Total chunks created: {len(chunks)}")
+        log_histogram("epub_chunks_created", len(chunks), labels={"file_path": file_path})
+
         if chunks:
             logging.debug(f"Structure of first chunk: {chunks[0].keys()}")
 
-
         # Handle summarization if enabled
         if auto_summarize and api_name and api_key:
             logging.info("Auto-summarization is enabled.")
@@ -101,11 +115,15 @@ def import_epub(file_path, title=None, author=None, keywords=None, custom_prompt
             for chunk in chunks:
                 chunk_text = chunk.get('text', '')
                 if chunk_text:
-                    summary_text = perform_summarization(api_name, chunk_text, custom_prompt, api_key, recursive_summarization=False, temp=None, system_message=system_prompt)
+                    summary_text = perform_summarization(api_name, chunk_text, custom_prompt, api_key,
+                                                            recursive_summarization=False, temp=None,
+                                                            system_message=system_prompt
+                                                            )
                     chunk['metadata']['summary'] = summary_text
                     summarized_chunks.append(chunk)
             chunks = summarized_chunks
             logging.info("Summarization of chunks completed.")
+            log_counter("epub_chunks_summarized", value=len(chunks), labels={"file_path": file_path})
         else:
             # If not summarizing, set a default summary or use provided summary
             if summary:
@@ -137,15 +155,33 @@ def import_epub(file_path, title=None, author=None, keywords=None, custom_prompt
             overwrite=False
         )
 
+        end_time = datetime.now()
+        processing_time = (end_time - start_time).total_seconds()
+        log_histogram("epub_import_duration", processing_time, labels={"file_path": file_path})
+
         logging.info(f"Ebook '{title}' by {author} imported successfully. Database result: {result}")
+        log_counter("epub ingested into the DB successfully", labels={"file_path": file_path})
         return f"Ebook '{title}' by {author} imported successfully. Database result: {result}"
 
     except Exception as e:
         logging.exception(f"Error importing ebook: {str(e)}")
+        log_counter("epub_import_error", labels={"file_path": file_path, "error": str(e)})
         return f"Error importing ebook: {str(e)}"
 
+
 # FIXME
-def process_zip_file(zip_file, title, author, keywords, custom_prompt, system_prompt, summary, auto_summarize, api_name, api_key, chunk_options):
+def process_zip_file(zip_file,
+                     title,
+                     author,
+                     keywords,
+                     custom_prompt,
+                     system_prompt,
+                     summary,
+                     auto_summarize,
+                     api_name,
+                     api_key,
+                     chunk_options
+                     ):
     """
     Processes a ZIP file containing multiple EPUB files and imports each one.
 
@@ -169,38 +205,58 @@ def process_zip_file(zip_file, title, author, keywords, custom_prompt, system_pr
         with tempfile.TemporaryDirectory() as temp_dir:
             zip_path = zip_file.name if hasattr(zip_file, 'name') else zip_file.path
             logging.info(f"Extracting ZIP file {zip_path} to temporary directory {temp_dir}")
+            log_counter("zip_processing_attempt", labels={"zip_path": zip_path})
+
             with zipfile.ZipFile(zip_path, 'r') as zip_ref:
                 zip_ref.extractall(temp_dir)
 
-            for filename in os.listdir(temp_dir):
-                if filename.lower().endswith('.epub'):
-                    file_path = os.path.join(temp_dir, filename)
-                    logging.info(f"Processing EPUB file {filename} from ZIP.")
-                    result = import_epub(
-                        file_path=file_path,
-                        title=title,
-                        author=author,
-                        keywords=keywords,
-                        custom_prompt=custom_prompt,
-                        summary=summary,
-                        auto_summarize=auto_summarize,
-                        api_name=api_name,
-                        api_key=api_key,
-                        chunk_options=chunk_options,
-                        custom_chapter_pattern=chunk_options.get('custom_chapter_pattern') if chunk_options else None
-                    )
-                    results.append(f"File: {filename} - {result}")
+            epub_files = [f for f in os.listdir(temp_dir) if f.lower().endswith('.epub')]
+            log_histogram("epub_files_in_zip", len(epub_files), labels={"zip_path": zip_path})
+
+            for filename in epub_files:
+                file_path = os.path.join(temp_dir, filename)
+                logging.info(f"Processing EPUB file {filename} from ZIP.")
+                result = import_epub(
+                    file_path=file_path,
+                    title=title,
+                    author=author,
+                    keywords=keywords,
+                    custom_prompt=custom_prompt,
+                    summary=summary,
+                    auto_summarize=auto_summarize,
+                    api_name=api_name,
+                    api_key=api_key,
+                    chunk_options=chunk_options,
+                    custom_chapter_pattern=chunk_options.get('custom_chapter_pattern') if chunk_options else None
+                )
+                results.append(f"File: {filename} - {result}")
+
             logging.info("Completed processing all EPUB files in the ZIP.")
+            log_counter("zip_processing_success", labels={"zip_path": zip_path})
     except Exception as e:
         logging.exception(f"Error processing ZIP file: {str(e)}")
+        log_counter("zip_processing_error", labels={"zip_path": zip_path, "error": str(e)})
         return f"Error processing ZIP file: {str(e)}"
 
     return "\n".join(results)
 
 
-def import_file_handler(file, title, author, keywords, system_prompt, custom_prompt, auto_summarize, api_name,
-                        api_key, max_chunk_size, chunk_overlap, custom_chapter_pattern):
+def import_file_handler(file,
+                        title,
+                        author,
+                        keywords,
+                        system_prompt,
+                        custom_prompt,
+                        auto_summarize,
+                        api_name,
+                        api_key,
+                        max_chunk_size,
+                        chunk_overlap,
+                        custom_chapter_pattern
+                        ):
     try:
+        log_counter("file_import_attempt", labels={"file_name": file.name})
+
         # Handle max_chunk_size
         if isinstance(max_chunk_size, str):
             max_chunk_size = int(max_chunk_size) if max_chunk_size.strip() else 4000
@@ -221,12 +277,16 @@ def import_file_handler(file, title, author, keywords, system_prompt, custom_pro
         }
 
         if file is None:
+            log_counter("file_import_error", labels={"error": "No file uploaded"})
             return "No file uploaded."
 
         file_path = file.name
         if not os.path.exists(file_path):
+            log_counter("file_import_error", labels={"error": "File not found", "file_name": file.name})
             return "Uploaded file not found."
 
+        start_time = datetime.now()
+
         if file_path.lower().endswith('.epub'):
             status = import_epub(
                 file_path,
@@ -242,7 +302,8 @@ def import_file_handler(file, title, author, keywords, system_prompt, custom_pro
                 chunk_options=chunk_options,
                 custom_chapter_pattern=custom_chapter_pattern
             )
-            return f"📚 EPUB Imported Successfully:\n{status}"
+            log_counter("epub_import_success", labels={"file_name": file.name})
+            result = f"📚 EPUB Imported Successfully:\n{status}"
         elif file.name.lower().endswith('.zip'):
             status = process_zip_file(
                 zip_file=file,
@@ -251,26 +312,38 @@ def import_file_handler(file, title, author, keywords, system_prompt, custom_pro
                 keywords=keywords,
                 custom_prompt=custom_prompt,
                 system_prompt=system_prompt,
-                summary=None,  # Let the library handle summarization
+                summary=None,
                 auto_summarize=auto_summarize,
                 api_name=api_name,
                 api_key=api_key,
                 chunk_options=chunk_options
             )
-            return f"📦 ZIP Processed Successfully:\n{status}"
+            log_counter("zip_import_success", labels={"file_name": file.name})
+            result = f"📦 ZIP Processed Successfully:\n{status}"
         elif file.name.lower().endswith(('.chm', '.html', '.pdf', '.xml', '.opml')):
             file_type = file.name.split('.')[-1].upper()
-            return f"{file_type} file import is not yet supported."
+            log_counter("unsupported_file_type", labels={"file_type": file_type})
+            result = f"{file_type} file import is not yet supported."
         else:
-            return "❌ Unsupported file type. Please upload an `.epub` file or a `.zip` file containing `.epub` files."
+            log_counter("unsupported_file_type", labels={"file_type": file.name.split('.')[-1]})
+            result = "❌ Unsupported file type. Please upload an `.epub` file or a `.zip` file containing `.epub` files."
+
+        end_time = datetime.now()
+        processing_time = (end_time - start_time).total_seconds()
+        log_histogram("file_import_duration", processing_time, labels={"file_name": file.name})
+
+        return result
 
     except ValueError as ve:
         logging.exception(f"Error parsing input values: {str(ve)}")
+        log_counter("file_import_error", labels={"error": "Invalid input", "file_name": file.name})
         return f"❌ Error: Invalid input for chunk size or overlap. Please enter valid numbers."
     except Exception as e:
         logging.exception(f"Error during file import: {str(e)}")
+        log_counter("file_import_error", labels={"error": str(e), "file_name": file.name})
         return f"❌ Error during import: {str(e)}"
 
+
 def read_epub(file_path):
     """
     Reads and extracts text from an EPUB file.

App_Function_Libraries/Character_Chat/Character_Chat_Lib.py

@@ -6,6 +6,7 @@ import json
 import logging
 import io
 import base64
+import time
 from typing import Dict, Any, Optional, List, Tuple
 #
 # External Imports
@@ -13,6 +14,7 @@ from PIL import Image
 #
 # Local imports
 from App_Function_Libraries.DB.DB_Manager import get_character_card_by_id, get_character_chat_by_id
+from App_Function_Libraries.Metrics.metrics_logger import log_counter, log_histogram
 #
 # Constants
 ####################################################################################################
@@ -79,16 +81,32 @@ def replace_user_placeholder(history, user_name):
 
 #################################################################################
 #
-# f
+# Functions for character card processing:
 
 def extract_character_id(choice: str) -> int:
     """Extract the character ID from the dropdown selection string."""
-    return int(choice.split('(ID: ')[1].rstrip(')'))
+    log_counter("extract_character_id_attempt")
+    try:
+        character_id = int(choice.split('(ID: ')[1].rstrip(')'))
+        log_counter("extract_character_id_success")
+        return character_id
+    except Exception as e:
+        log_counter("extract_character_id_error", labels={"error": str(e)})
+        raise
 
 def load_character_wrapper(character_id: int, user_name: str) -> Tuple[Dict[str, Any], List[Tuple[Optional[str], str]], Optional[Image.Image]]:
     """Wrapper function to load character and image using the extracted ID."""
-    char_data, chat_history, img = load_character_and_image(character_id, user_name)
-    return char_data, chat_history, img
+    log_counter("load_character_wrapper_attempt")
+    start_time = time.time()
+    try:
+        char_data, chat_history, img = load_character_and_image(character_id, user_name)
+        load_duration = time.time() - start_time
+        log_histogram("load_character_wrapper_duration", load_duration)
+        log_counter("load_character_wrapper_success")
+        return char_data, chat_history, img
+    except Exception as e:
+        log_counter("load_character_wrapper_error", labels={"error": str(e)})
+        raise
 
 def parse_character_book(book_data: Dict[str, Any]) -> Dict[str, Any]:
     """
@@ -143,9 +161,12 @@ def load_character_and_image(character_id: int, user_name: str) -> Tuple[Optiona
         Tuple[Optional[Dict[str, Any]], List[Tuple[Optional[str], str]], Optional[Image.Image]]:
         A tuple containing the character data, chat history, and character image (if available).
     """
+    log_counter("load_character_and_image_attempt")
+    start_time = time.time()
     try:
         char_data = get_character_card_by_id(character_id)
         if not char_data:
+            log_counter("load_character_and_image_no_data")
             logging.warning(f"No character data found for ID: {character_id}")
             return None, [], None
 
@@ -165,12 +186,18 @@ def load_character_and_image(character_id: int, user_name: str) -> Tuple[Optiona
             try:
                 image_data = base64.b64decode(char_data['image'])
                 img = Image.open(io.BytesIO(image_data)).convert("RGBA")
+                log_counter("load_character_image_success")
             except Exception as e:
+                log_counter("load_character_image_error", labels={"error": str(e)})
                 logging.error(f"Error processing image for character '{char_data['name']}': {e}")
 
+        load_duration = time.time() - start_time
+        log_histogram("load_character_and_image_duration", load_duration)
+        log_counter("load_character_and_image_success")
         return char_data, chat_history, img
 
     except Exception as e:
+        log_counter("load_character_and_image_error", labels={"error": str(e)})
         logging.error(f"Error in load_character_and_image: {e}")
         return None, [], None
 
@@ -186,10 +213,13 @@ def load_chat_and_character(chat_id: int, user_name: str) -> Tuple[Optional[Dict
         Tuple[Optional[Dict[str, Any]], List[Tuple[str, str]], Optional[Image.Image]]:
         A tuple containing the character data, processed chat history, and character image (if available).
     """
+    log_counter("load_chat_and_character_attempt")
+    start_time = time.time()
     try:
         # Load the chat
         chat = get_character_chat_by_id(chat_id)
         if not chat:
+            log_counter("load_chat_and_character_no_chat")
             logging.warning(f"No chat found with ID: {chat_id}")
             return None, [], None
 
@@ -197,6 +227,7 @@ def load_chat_and_character(chat_id: int, user_name: str) -> Tuple[Optional[Dict
         character_id = chat['character_id']
         char_data = get_character_card_by_id(character_id)
         if not char_data:
+            log_counter("load_chat_and_character_no_character")
             logging.warning(f"No character found for chat ID: {chat_id}")
             return None, chat['chat_history'], None
 
@@ -209,7 +240,9 @@ def load_chat_and_character(chat_id: int, user_name: str) -> Tuple[Optional[Dict
             try:
                 image_data = base64.b64decode(char_data['image'])
                 img = Image.open(io.BytesIO(image_data)).convert("RGBA")
+                log_counter("load_chat_character_image_success")
             except Exception as e:
+                log_counter("load_chat_character_image_error", labels={"error": str(e)})
                 logging.error(f"Error processing image for character '{char_data['name']}': {e}")
 
         # Process character data templates
@@ -217,14 +250,21 @@ def load_chat_and_character(chat_id: int, user_name: str) -> Tuple[Optional[Dict
             if field in char_data:
                 char_data[field] = replace_placeholders(char_data[field], char_data['name'], user_name)
 
+        load_duration = time.time() - start_time
+        log_histogram("load_chat_and_character_duration", load_duration)
+        log_counter("load_chat_and_character_success")
         return char_data, processed_history, img
 
     except Exception as e:
+        log_counter("load_chat_and_character_error", labels={"error": str(e)})
         logging.error(f"Error in load_chat_and_character: {e}")
         return None, [], None
 
+
 def extract_json_from_image(image_file):
     logging.debug(f"Attempting to extract JSON from image: {image_file.name}")
+    log_counter("extract_json_from_image_attempt")
+    start_time = time.time()
     try:
         with Image.open(image_file) as img:
             logging.debug("Image opened successfully")
@@ -236,16 +276,18 @@ def extract_json_from_image(image_file):
                 try:
                     decoded_content = base64.b64decode(chara_content).decode('utf-8')
                     logging.debug(f"Decoded content (first 100 chars): {decoded_content[:100]}...")
+                    log_counter("extract_json_from_image_metadata_success")
                     return decoded_content
                 except Exception as e:
                     logging.error(f"Error decoding base64 content: {e}")
+                    log_counter("extract_json_from_image_decode_error", labels={"error": str(e)})
 
             logging.warning("'chara' not found in metadata, attempting to find JSON data in image bytes")
             # Alternative method to extract embedded JSON from image bytes if metadata is not available
             img_byte_arr = io.BytesIO()
             img.save(img_byte_arr, format='PNG')
             img_bytes = img_byte_arr.getvalue()
-            img_str = img_bytes.decode('latin1')  # Use 'latin1' to preserve byte values
+            img_str = img_bytes.decode('latin1')
 
             # Search for JSON-like structures in the image bytes
             json_start = img_str.find('{')
@@ -255,18 +297,26 @@ def extract_json_from_image(image_file):
                 try:
                     json.loads(possible_json)
                     logging.debug("Found JSON data in image bytes")
+                    log_counter("extract_json_from_image_bytes_success")
                     return possible_json
                 except json.JSONDecodeError:
                     logging.debug("No valid JSON found in image bytes")
+                    log_counter("extract_json_from_image_invalid_json")
 
             logging.warning("No JSON data found in the image")
+            log_counter("extract_json_from_image_no_json_found")
     except Exception as e:
+        log_counter("extract_json_from_image_error", labels={"error": str(e)})
         logging.error(f"Error extracting JSON from image: {e}")
-    return None
 
+    extract_duration = time.time() - start_time
+    log_histogram("extract_json_from_image_duration", extract_duration)
+    return None
 
 
 def load_chat_history(file):
+    log_counter("load_chat_history_attempt")
+    start_time = time.time()
     try:
         content = file.read().decode('utf-8')
         chat_data = json.loads(content)
@@ -276,11 +326,16 @@ def load_chat_history(file):
         character_name = chat_data.get('character') or chat_data.get('character_name')
 
         if not history or not character_name:
+            log_counter("load_chat_history_incomplete_data")
             logging.error("Chat history or character name missing in the imported file.")
             return None, None
 
+        load_duration = time.time() - start_time
+        log_histogram("load_chat_history_duration", load_duration)
+        log_counter("load_chat_history_success")
         return history, character_name
     except Exception as e:
+        log_counter("load_chat_history_error", labels={"error": str(e)})
         logging.error(f"Error loading chat history: {e}")
         return None, None
 
@@ -297,14 +352,25 @@ def process_chat_history(chat_history: List[Tuple[str, str]], char_name: str, us
     Returns:
         List[Tuple[str, str]]: The processed chat history.
     """
-    processed_history = []
-    for user_msg, char_msg in chat_history:
-        if user_msg:
-            user_msg = replace_placeholders(user_msg, char_name, user_name)
-        if char_msg:
-            char_msg = replace_placeholders(char_msg, char_name, user_name)
-        processed_history.append((user_msg, char_msg))
-    return processed_history
+    log_counter("process_chat_history_attempt")
+    start_time = time.time()
+    try:
+        processed_history = []
+        for user_msg, char_msg in chat_history:
+            if user_msg:
+                user_msg = replace_placeholders(user_msg, char_name, user_name)
+            if char_msg:
+                char_msg = replace_placeholders(char_msg, char_name, user_name)
+            processed_history.append((user_msg, char_msg))
+
+        process_duration = time.time() - start_time
+        log_histogram("process_chat_history_duration", process_duration)
+        log_counter("process_chat_history_success", labels={"message_count": len(chat_history)})
+        return processed_history
+    except Exception as e:
+        log_counter("process_chat_history_error", labels={"error": str(e)})
+        logging.error(f"Error processing chat history: {e}")
+        raise
 
 def validate_character_book(book_data):
     """

App_Function_Libraries/Chat.py

@@ -7,6 +7,7 @@ import logging
 import os
 import re
 import tempfile
+import time
 from datetime import datetime
 from pathlib import Path
 #
@@ -15,20 +16,20 @@ from pathlib import Path
 # Local Imports
 from App_Function_Libraries.DB.DB_Manager import get_conversation_name, save_chat_history_to_database
 from App_Function_Libraries.LLM_API_Calls import chat_with_openai, chat_with_anthropic, chat_with_cohere, \
-    chat_with_groq, chat_with_openrouter, chat_with_deepseek, chat_with_mistral, chat_with_huggingface  #, chat_with_vllm
+    chat_with_groq, chat_with_openrouter, chat_with_deepseek, chat_with_mistral, chat_with_huggingface
 from App_Function_Libraries.LLM_API_Calls_Local import chat_with_aphrodite, chat_with_local_llm, chat_with_ollama, \
     chat_with_kobold, chat_with_llama, chat_with_oobabooga, chat_with_tabbyapi, chat_with_vllm, chat_with_custom_openai
 from App_Function_Libraries.DB.SQLite_DB import load_media_content
 from App_Function_Libraries.Utils.Utils import generate_unique_filename, load_and_log_configs
-
-
+from App_Function_Libraries.Metrics.metrics_logger import log_counter, log_histogram
 #
 ####################################################################################################
 #
 # Functions:
 
-
 def chat_api_call(api_endpoint, api_key, input_data, prompt, temp, system_message=None):
+    log_counter("chat_api_call_attempt", labels={"api_endpoint": api_endpoint})
+    start_time = time.time()
     if not api_key:
         api_key = None
     model = None
@@ -105,14 +106,21 @@ def chat_api_call(api_endpoint, api_key, input_data, prompt, temp, system_messag
         else:
             raise ValueError(f"Unsupported API endpoint: {api_endpoint}")
 
+        call_duration = time.time() - start_time
+        log_histogram("chat_api_call_duration", call_duration, labels={"api_endpoint": api_endpoint})
+        log_counter("chat_api_call_success", labels={"api_endpoint": api_endpoint})
         return response
 
     except Exception as e:
+        log_counter("chat_api_call_error", labels={"api_endpoint": api_endpoint, "error": str(e)})
         logging.error(f"Error in chat function: {str(e)}")
         return f"An error occurred: {str(e)}"
 
+
 def chat(message, history, media_content, selected_parts, api_endpoint, api_key, prompt, temperature,
          system_message=None):
+    log_counter("chat_attempt", labels={"api_endpoint": api_endpoint})
+    start_time = time.time()
     try:
         logging.info(f"Debug - Chat Function - Message: {message}")
         logging.info(f"Debug - Chat Function - Media Content: {media_content}")
@@ -151,14 +159,19 @@ def chat(message, history, media_content, selected_parts, api_endpoint, api_key,
         # Use the existing API request code based on the selected endpoint
         response = chat_api_call(api_endpoint, api_key, input_data, prompt, temp, system_message)
 
+        chat_duration = time.time() - start_time
+        log_histogram("chat_duration", chat_duration, labels={"api_endpoint": api_endpoint})
+        log_counter("chat_success", labels={"api_endpoint": api_endpoint})
         return response
     except Exception as e:
+        log_counter("chat_error", labels={"api_endpoint": api_endpoint, "error": str(e)})
         logging.error(f"Error in chat function: {str(e)}")
         return f"An error occurred: {str(e)}"
 
 
-
 def save_chat_history_to_db_wrapper(chatbot, conversation_id, media_content, media_name=None):
+    log_counter("save_chat_history_to_db_attempt")
+    start_time = time.time()
     logging.info(f"Attempting to save chat history. Media content type: {type(media_content)}")
     try:
         # Extract the media_id and media_name from the media_content
@@ -206,14 +219,20 @@ def save_chat_history_to_db_wrapper(chatbot, conversation_id, media_content, med
 
         new_conversation_id = save_chat_history_to_database(chatbot, conversation_id, media_id, media_name,
                                                             conversation_name)
+        save_duration = time.time() - start_time
+        log_histogram("save_chat_history_to_db_duration", save_duration)
+        log_counter("save_chat_history_to_db_success")
         return new_conversation_id, f"Chat history saved successfully as {conversation_name}!"
     except Exception as e:
+        log_counter("save_chat_history_to_db_error", labels={"error": str(e)})
         error_message = f"Failed to save chat history: {str(e)}"
         logging.error(error_message, exc_info=True)
         return conversation_id, error_message
 
 
 def save_chat_history(history, conversation_id, media_content):
+    log_counter("save_chat_history_attempt")
+    start_time = time.time()
     try:
         content, conversation_name = generate_chat_history_content(history, conversation_id, media_content)
 
@@ -233,8 +252,12 @@ def save_chat_history(history, conversation_id, media_content):
         # Rename the temporary file to the unique filename
         os.rename(temp_file_path, final_path)
 
+        save_duration = time.time() - start_time
+        log_histogram("save_chat_history_duration", save_duration)
+        log_counter("save_chat_history_success")
         return final_path
     except Exception as e:
+        log_counter("save_chat_history_error", labels={"error": str(e)})
         logging.error(f"Error saving chat history: {str(e)}")
         return None
 
@@ -286,6 +309,8 @@ def extract_media_name(media_content):
 
 
 def update_chat_content(selected_item, use_content, use_summary, use_prompt, item_mapping):
+    log_counter("update_chat_content_attempt")
+    start_time = time.time()
     logging.debug(f"Debug - Update Chat Content - Selected Item: {selected_item}\n")
     logging.debug(f"Debug - Update Chat Content - Use Content: {use_content}\n\n\n\n")
     logging.debug(f"Debug - Update Chat Content - Use Summary: {use_summary}\n\n")
@@ -312,17 +337,21 @@ def update_chat_content(selected_item, use_content, use_summary, use_prompt, ite
             print(f"Debug - Update Chat Content - Content(first 500 char): {str(content)[:500]}\n\n\n\n")
 
         print(f"Debug - Update Chat Content - Selected Parts: {selected_parts}")
+        update_duration = time.time() - start_time
+        log_histogram("update_chat_content_duration", update_duration)
+        log_counter("update_chat_content_success")
         return content, selected_parts
     else:
+        log_counter("update_chat_content_error", labels={"error": str("No item selected or item not in mapping")})
         print(f"Debug - Update Chat Content - No item selected or item not in mapping")
         return {}, []
 
 #
 # End of Chat functions
-##########################################################################################################################
+#######################################################################################################################
 
 
-##########################################################################################################################
+#######################################################################################################################
 #
 # Character Card Functions
 
@@ -330,6 +359,8 @@ CHARACTERS_FILE = Path('.', 'Helper_Scripts', 'Character_Cards', 'Characters.jso
 
 
 def save_character(character_data):
+    log_counter("save_character_attempt")
+    start_time = time.time()
     characters_file = os.path.join(os.path.dirname(__file__), '..', 'Helper_Scripts', 'Character_Cards', 'Characters.json')
     characters_dir = os.path.dirname(characters_file)
 
@@ -357,28 +388,52 @@ def save_character(character_data):
         with open(characters_file, 'w') as f:
             json.dump(characters, f, indent=2)
 
+        save_duration = time.time() - start_time
+        log_histogram("save_character_duration", save_duration)
+        log_counter("save_character_success")
         logging.info(f"Character '{char_name}' saved successfully.")
     except Exception as e:
+        log_counter("save_character_error", labels={"error": str(e)})
         logging.error(f"Error saving character: {str(e)}")
 
 
-
 def load_characters():
-    characters_file = os.path.join(os.path.dirname(__file__), '..', 'Helper_Scripts', 'Character_Cards', 'Characters.json')
-    if os.path.exists(characters_file):
-        with open(characters_file, 'r') as f:
-            characters = json.load(f)
-        logging.debug(f"Loaded {len(characters)} characters from {characters_file}")
-        return characters
-    logging.warning(f"Characters file not found: {characters_file}")
-    return {}
+    log_counter("load_characters_attempt")
+    start_time = time.time()
+    try:
+        characters_file = os.path.join(os.path.dirname(__file__), '..', 'Helper_Scripts', 'Character_Cards', 'Characters.json')
+        if os.path.exists(characters_file):
+            with open(characters_file, 'r') as f:
+                characters = json.load(f)
+            logging.debug(f"Loaded {len(characters)} characters from {characters_file}")
+            load_duration = time.time() - start_time
+            log_histogram("load_characters_duration", load_duration)
+            log_counter("load_characters_success", labels={"character_count": len(characters)})
+            return characters
+        else:
+            logging.warning(f"Characters file not found: {characters_file}")
+            return {}
+    except Exception as e:
+        log_counter("load_characters_error", labels={"error": str(e)})
+        return {}
 
 
-def get_character_names():
-    characters = load_characters()
-    return list(characters.keys())
 
+def get_character_names():
+    log_counter("get_character_names_attempt")
+    start_time = time.time()
+    try:
+        characters = load_characters()
+        names = list(characters.keys())
+        get_names_duration = time.time() - start_time
+        log_histogram("get_character_names_duration", get_names_duration)
+        log_counter("get_character_names_success", labels={"name_count": len(names)})
+        return names
+    except Exception as e:
+        log_counter("get_character_names_error", labels={"error": str(e)})
+        logging.error(f"Error getting character names: {str(e)}")
+        return []
 
 #
 # End of Chat.py
-##########################################################################################################################
+##########################################################################################################################

App_Function_Libraries/DB/Character_Chat_DB.py

@@ -1,701 +1,701 @@
-# character_chat_db.py
-# Database functions for managing character cards and chat histories.
-# #
-# Imports
-import configparser
-import sqlite3
-import logging
-import json
-import os
-import sys
-from typing import List, Dict, Optional, Tuple, Any, Union
-
-from App_Function_Libraries.Utils.Utils import get_database_dir, get_project_relative_path, get_database_path
-
-#
-#######################################################################################################################
-#
-#
-
-def ensure_database_directory():
-    os.makedirs(get_database_dir(), exist_ok=True)
-
-ensure_database_directory()
-
-
-# Construct the path to the config file
-config_path = get_project_relative_path('Config_Files/config.txt')
-
-# Read the config file
-config = configparser.ConfigParser()
-config.read(config_path)
-
-# Get the chat db path from the config, or use the default if not specified
-chat_DB_PATH = config.get('Database', 'chatDB_path', fallback=get_database_path('chatDB.db'))
-print(f"Chat Database path: {chat_DB_PATH}")
-
-########################################################################################################
-#
-# Functions
-
-# FIXME - Setup properly and test/add documentation for its existence...
-def initialize_database():
-    """Initialize the SQLite database with required tables and FTS5 virtual tables."""
-    conn = None
-    try:
-        conn = sqlite3.connect(chat_DB_PATH)
-        cursor = conn.cursor()
-
-        # Enable foreign key constraints
-        cursor.execute("PRAGMA foreign_keys = ON;")
-
-        # Create CharacterCards table with V2 fields
-        cursor.execute("""
-        CREATE TABLE IF NOT EXISTS CharacterCards (
-            id INTEGER PRIMARY KEY AUTOINCREMENT,
-            name TEXT UNIQUE NOT NULL,
-            description TEXT,
-            personality TEXT,
-            scenario TEXT,
-            image BLOB,
-            post_history_instructions TEXT,
-            first_mes TEXT,
-            mes_example TEXT,
-            creator_notes TEXT,
-            system_prompt TEXT,
-            alternate_greetings TEXT,
-            tags TEXT,
-            creator TEXT,
-            character_version TEXT,
-            extensions TEXT,
-            created_at DATETIME DEFAULT CURRENT_TIMESTAMP
-        );
-        """)
-
-        # Create CharacterChats table
-        cursor.execute("""
-        CREATE TABLE IF NOT EXISTS CharacterChats (
-            id INTEGER PRIMARY KEY AUTOINCREMENT,
-            character_id INTEGER NOT NULL,
-            conversation_name TEXT,
-            chat_history TEXT,
-            is_snapshot BOOLEAN DEFAULT FALSE,
-            created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
-            FOREIGN KEY (character_id) REFERENCES CharacterCards(id) ON DELETE CASCADE
-        );
-        """)
-
-        # Create FTS5 virtual table for CharacterChats
-        cursor.execute("""
-        CREATE VIRTUAL TABLE IF NOT EXISTS CharacterChats_fts USING fts5(
-            conversation_name,
-            chat_history,
-            content='CharacterChats',
-            content_rowid='id'
-        );
-        """)
-
-        # Create triggers to keep FTS5 table in sync with CharacterChats
-        cursor.executescript("""
-        CREATE TRIGGER IF NOT EXISTS CharacterChats_ai AFTER INSERT ON CharacterChats BEGIN
-            INSERT INTO CharacterChats_fts(rowid, conversation_name, chat_history)
-            VALUES (new.id, new.conversation_name, new.chat_history);
-        END;
-
-        CREATE TRIGGER IF NOT EXISTS CharacterChats_ad AFTER DELETE ON CharacterChats BEGIN
-            DELETE FROM CharacterChats_fts WHERE rowid = old.id;
-        END;
-
-        CREATE TRIGGER IF NOT EXISTS CharacterChats_au AFTER UPDATE ON CharacterChats BEGIN
-            UPDATE CharacterChats_fts SET conversation_name = new.conversation_name, chat_history = new.chat_history
-            WHERE rowid = new.id;
-        END;
-        """)
-
-        # Create ChatKeywords table
-        cursor.execute("""
-        CREATE TABLE IF NOT EXISTS ChatKeywords (
-            chat_id INTEGER NOT NULL,
-            keyword TEXT NOT NULL,
-            FOREIGN KEY (chat_id) REFERENCES CharacterChats(id) ON DELETE CASCADE
-        );
-        """)
-
-        # Create indexes for faster searches
-        cursor.execute("""
-        CREATE INDEX IF NOT EXISTS idx_chatkeywords_keyword ON ChatKeywords(keyword);
-        """)
-        cursor.execute("""
-        CREATE INDEX IF NOT EXISTS idx_chatkeywords_chat_id ON ChatKeywords(chat_id);
-        """)
-
-        conn.commit()
-        logging.info("Database initialized successfully.")
-    except sqlite3.Error as e:
-        logging.error(f"SQLite error occurred during database initialization: {e}")
-        if conn:
-            conn.rollback()
-        raise
-    except Exception as e:
-        logging.error(f"Unexpected error occurred during database initialization: {e}")
-        if conn:
-            conn.rollback()
-        raise
-    finally:
-        if conn:
-            conn.close()
-
-# Call initialize_database() at the start of your application
-def setup_chat_database():
-    try:
-        initialize_database()
-    except Exception as e:
-        logging.critical(f"Failed to initialize database: {e}")
-        sys.exit(1)
-
-setup_chat_database()
-
-########################################################################################################
-#
-# Character Card handling
-
-def parse_character_card(card_data: Dict[str, Any]) -> Dict[str, Any]:
-    """Parse and validate a character card according to V2 specification."""
-    v2_data = {
-        'name': card_data.get('name', ''),
-        'description': card_data.get('description', ''),
-        'personality': card_data.get('personality', ''),
-        'scenario': card_data.get('scenario', ''),
-        'first_mes': card_data.get('first_mes', ''),
-        'mes_example': card_data.get('mes_example', ''),
-        'creator_notes': card_data.get('creator_notes', ''),
-        'system_prompt': card_data.get('system_prompt', ''),
-        'post_history_instructions': card_data.get('post_history_instructions', ''),
-        'alternate_greetings': json.dumps(card_data.get('alternate_greetings', [])),
-        'tags': json.dumps(card_data.get('tags', [])),
-        'creator': card_data.get('creator', ''),
-        'character_version': card_data.get('character_version', ''),
-        'extensions': json.dumps(card_data.get('extensions', {}))
-    }
-
-    # Handle 'image' separately as it might be binary data
-    if 'image' in card_data:
-        v2_data['image'] = card_data['image']
-
-    return v2_data
-
-
-def add_character_card(card_data: Dict[str, Any]) -> Optional[int]:
-    """Add or update a character card in the database."""
-    conn = sqlite3.connect(chat_DB_PATH)
-    cursor = conn.cursor()
-    try:
-        parsed_card = parse_character_card(card_data)
-
-        # Check if character already exists
-        cursor.execute("SELECT id FROM CharacterCards WHERE name = ?", (parsed_card['name'],))
-        row = cursor.fetchone()
-
-        if row:
-            # Update existing character
-            character_id = row[0]
-            update_query = """
-                UPDATE CharacterCards
-                SET description = ?, personality = ?, scenario = ?, image = ?, 
-                    post_history_instructions = ?, first_mes = ?, mes_example = ?,
-                    creator_notes = ?, system_prompt = ?, alternate_greetings = ?,
-                    tags = ?, creator = ?, character_version = ?, extensions = ?
-                WHERE id = ?
-            """
-            cursor.execute(update_query, (
-                parsed_card['description'], parsed_card['personality'], parsed_card['scenario'],
-                parsed_card['image'], parsed_card['post_history_instructions'], parsed_card['first_mes'],
-                parsed_card['mes_example'], parsed_card['creator_notes'], parsed_card['system_prompt'],
-                parsed_card['alternate_greetings'], parsed_card['tags'], parsed_card['creator'],
-                parsed_card['character_version'], parsed_card['extensions'], character_id
-            ))
-        else:
-            # Insert new character
-            insert_query = """
-                INSERT INTO CharacterCards (name, description, personality, scenario, image, 
-                post_history_instructions, first_mes, mes_example, creator_notes, system_prompt, 
-                alternate_greetings, tags, creator, character_version, extensions)
-                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-            """
-            cursor.execute(insert_query, (
-                parsed_card['name'], parsed_card['description'], parsed_card['personality'],
-                parsed_card['scenario'], parsed_card['image'], parsed_card['post_history_instructions'],
-                parsed_card['first_mes'], parsed_card['mes_example'], parsed_card['creator_notes'],
-                parsed_card['system_prompt'], parsed_card['alternate_greetings'], parsed_card['tags'],
-                parsed_card['creator'], parsed_card['character_version'], parsed_card['extensions']
-            ))
-            character_id = cursor.lastrowid
-
-        conn.commit()
-        return character_id
-    except sqlite3.IntegrityError as e:
-        logging.error(f"Error adding character card: {e}")
-        return None
-    except Exception as e:
-        logging.error(f"Unexpected error adding character card: {e}")
-        return None
-    finally:
-        conn.close()
-
-# def add_character_card(card_data: Dict) -> Optional[int]:
-#     """Add or update a character card in the database.
-#
-#     Returns the ID of the inserted character or None if failed.
-#     """
-#     conn = sqlite3.connect(chat_DB_PATH)
-#     cursor = conn.cursor()
-#     try:
-#         # Ensure all required fields are present
-#         required_fields = ['name', 'description', 'personality', 'scenario', 'image', 'post_history_instructions', 'first_message']
-#         for field in required_fields:
-#             if field not in card_data:
-#                 card_data[field] = ''  # Assign empty string if field is missing
-#
-#         # Check if character already exists
-#         cursor.execute("SELECT id FROM CharacterCards WHERE name = ?", (card_data['name'],))
-#         row = cursor.fetchone()
-#
-#         if row:
-#             # Update existing character
-#             character_id = row[0]
-#             cursor.execute("""
-#                 UPDATE CharacterCards
-#                 SET description = ?, personality = ?, scenario = ?, image = ?, post_history_instructions = ?, first_message = ?
-#                 WHERE id = ?
-#             """, (
-#                 card_data['description'],
-#                 card_data['personality'],
-#                 card_data['scenario'],
-#                 card_data['image'],
-#                 card_data['post_history_instructions'],
-#                 card_data['first_message'],
-#                 character_id
-#             ))
-#         else:
-#             # Insert new character
-#             cursor.execute("""
-#                 INSERT INTO CharacterCards (name, description, personality, scenario, image, post_history_instructions, first_message)
-#                 VALUES (?, ?, ?, ?, ?, ?, ?)
-#             """, (
-#                 card_data['name'],
-#                 card_data['description'],
-#                 card_data['personality'],
-#                 card_data['scenario'],
-#                 card_data['image'],
-#                 card_data['post_history_instructions'],
-#                 card_data['first_message']
-#             ))
-#             character_id = cursor.lastrowid
-#
-#         conn.commit()
-#         return cursor.lastrowid
-#     except sqlite3.IntegrityError as e:
-#         logging.error(f"Error adding character card: {e}")
-#         return None
-#     except Exception as e:
-#         logging.error(f"Unexpected error adding character card: {e}")
-#         return None
-#     finally:
-#         conn.close()
-
-
-def get_character_cards() -> List[Dict]:
-    """Retrieve all character cards from the database."""
-    logging.debug(f"Fetching characters from DB: {chat_DB_PATH}")
-    conn = sqlite3.connect(chat_DB_PATH)
-    cursor = conn.cursor()
-    cursor.execute("SELECT * FROM CharacterCards")
-    rows = cursor.fetchall()
-    columns = [description[0] for description in cursor.description]
-    conn.close()
-    characters = [dict(zip(columns, row)) for row in rows]
-    #logging.debug(f"Characters fetched from DB: {characters}")
-    return characters
-
-
-def get_character_card_by_id(character_id: Union[int, Dict[str, Any]]) -> Optional[Dict[str, Any]]:
-    """
-    Retrieve a single character card by its ID.
-
-    Args:
-        character_id: Can be either an integer ID or a dictionary containing character data.
-
-    Returns:
-        A dictionary containing the character card data, or None if not found.
-    """
-    conn = sqlite3.connect(chat_DB_PATH)
-    cursor = conn.cursor()
-    try:
-        if isinstance(character_id, dict):
-            # If a dictionary is passed, assume it's already a character card
-            return character_id
-        elif isinstance(character_id, int):
-            # If an integer is passed, fetch the character from the database
-            cursor.execute("SELECT * FROM CharacterCards WHERE id = ?", (character_id,))
-            row = cursor.fetchone()
-            if row:
-                columns = [description[0] for description in cursor.description]
-                return dict(zip(columns, row))
-        else:
-            logging.warning(f"Invalid type for character_id: {type(character_id)}")
-        return None
-    except Exception as e:
-        logging.error(f"Error in get_character_card_by_id: {e}")
-        return None
-    finally:
-        conn.close()
-
-
-def update_character_card(character_id: int, card_data: Dict) -> bool:
-    """Update an existing character card."""
-    conn = sqlite3.connect(chat_DB_PATH)
-    cursor = conn.cursor()
-    try:
-        cursor.execute("""
-            UPDATE CharacterCards
-            SET name = ?, description = ?, personality = ?, scenario = ?, image = ?, post_history_instructions = ?, first_message = ?
-            WHERE id = ?
-        """, (
-            card_data.get('name'),
-            card_data.get('description'),
-            card_data.get('personality'),
-            card_data.get('scenario'),
-            card_data.get('image'),
-            card_data.get('post_history_instructions', ''),
-            card_data.get('first_message', "Hello! I'm ready to chat."),
-            character_id
-        ))
-        conn.commit()
-        return cursor.rowcount > 0
-    except sqlite3.IntegrityError as e:
-        logging.error(f"Error updating character card: {e}")
-        return False
-    finally:
-        conn.close()
-
-
-def delete_character_card(character_id: int) -> bool:
-    """Delete a character card and its associated chats."""
-    conn = sqlite3.connect(chat_DB_PATH)
-    cursor = conn.cursor()
-    try:
-        # Delete associated chats first due to foreign key constraint
-        cursor.execute("DELETE FROM CharacterChats WHERE character_id = ?", (character_id,))
-        cursor.execute("DELETE FROM CharacterCards WHERE id = ?", (character_id,))
-        conn.commit()
-        return cursor.rowcount > 0
-    except sqlite3.Error as e:
-        logging.error(f"Error deleting character card: {e}")
-        return False
-    finally:
-        conn.close()
-
-
-def add_character_chat(character_id: int, conversation_name: str, chat_history: List[Tuple[str, str]], keywords: Optional[List[str]] = None, is_snapshot: bool = False) -> Optional[int]:
-    """
-    Add a new chat history for a character, optionally associating keywords.
-
-    Args:
-        character_id (int): The ID of the character.
-        conversation_name (str): Name of the conversation.
-        chat_history (List[Tuple[str, str]]): List of (user, bot) message tuples.
-        keywords (Optional[List[str]]): List of keywords to associate with this chat.
-        is_snapshot (bool, optional): Whether this chat is a snapshot.
-
-    Returns:
-        Optional[int]: The ID of the inserted chat or None if failed.
-    """
-    conn = sqlite3.connect(chat_DB_PATH)
-    cursor = conn.cursor()
-    try:
-        chat_history_json = json.dumps(chat_history)
-        cursor.execute("""
-            INSERT INTO CharacterChats (character_id, conversation_name, chat_history, is_snapshot)
-            VALUES (?, ?, ?, ?)
-        """, (
-            character_id,
-            conversation_name,
-            chat_history_json,
-            is_snapshot
-        ))
-        chat_id = cursor.lastrowid
-
-        if keywords:
-            # Insert keywords into ChatKeywords table
-            keyword_records = [(chat_id, keyword.strip().lower()) for keyword in keywords]
-            cursor.executemany("""
-                INSERT INTO ChatKeywords (chat_id, keyword)
-                VALUES (?, ?)
-            """, keyword_records)
-
-        conn.commit()
-        return chat_id
-    except sqlite3.Error as e:
-        logging.error(f"Error adding character chat: {e}")
-        return None
-    finally:
-        conn.close()
-
-
-def get_character_chats(character_id: Optional[int] = None) -> List[Dict]:
-    """Retrieve all chats, or chats for a specific character if character_id is provided."""
-    conn = sqlite3.connect(chat_DB_PATH)
-    cursor = conn.cursor()
-    if character_id is not None:
-        cursor.execute("SELECT * FROM CharacterChats WHERE character_id = ?", (character_id,))
-    else:
-        cursor.execute("SELECT * FROM CharacterChats")
-    rows = cursor.fetchall()
-    columns = [description[0] for description in cursor.description]
-    conn.close()
-    return [dict(zip(columns, row)) for row in rows]
-
-
-def get_character_chat_by_id(chat_id: int) -> Optional[Dict]:
-    """Retrieve a single chat by its ID."""
-    conn = sqlite3.connect(chat_DB_PATH)
-    cursor = conn.cursor()
-    cursor.execute("SELECT * FROM CharacterChats WHERE id = ?", (chat_id,))
-    row = cursor.fetchone()
-    conn.close()
-    if row:
-        columns = [description[0] for description in cursor.description]
-        chat = dict(zip(columns, row))
-        chat['chat_history'] = json.loads(chat['chat_history'])
-        return chat
-    return None
-
-
-def search_character_chats(query: str, character_id: Optional[int] = None) -> Tuple[List[Dict], str]:
-    """
-    Search for character chats using FTS5, optionally filtered by character_id.
-
-    Args:
-        query (str): The search query.
-        character_id (Optional[int]): The ID of the character to filter chats by.
-
-    Returns:
-        Tuple[List[Dict], str]: A list of matching chats and a status message.
-    """
-    if not query.strip():
-        return [], "Please enter a search query."
-
-    conn = sqlite3.connect(chat_DB_PATH)
-    cursor = conn.cursor()
-    try:
-        if character_id is not None:
-            # Search with character_id filter
-            cursor.execute("""
-                SELECT CharacterChats.id, CharacterChats.conversation_name, CharacterChats.chat_history
-                FROM CharacterChats_fts
-                JOIN CharacterChats ON CharacterChats_fts.rowid = CharacterChats.id
-                WHERE CharacterChats_fts MATCH ? AND CharacterChats.character_id = ?
-                ORDER BY rank
-            """, (query, character_id))
-        else:
-            # Search without character_id filter
-            cursor.execute("""
-                SELECT CharacterChats.id, CharacterChats.conversation_name, CharacterChats.chat_history
-                FROM CharacterChats_fts
-                JOIN CharacterChats ON CharacterChats_fts.rowid = CharacterChats.id
-                WHERE CharacterChats_fts MATCH ?
-                ORDER BY rank
-            """, (query,))
-
-        rows = cursor.fetchall()
-        columns = [description[0] for description in cursor.description]
-        results = [dict(zip(columns, row)) for row in rows]
-
-        if character_id is not None:
-            status_message = f"Found {len(results)} chat(s) matching '{query}' for the selected character."
-        else:
-            status_message = f"Found {len(results)} chat(s) matching '{query}' across all characters."
-
-        return results, status_message
-    except Exception as e:
-        logging.error(f"Error searching chats with FTS5: {e}")
-        return [], f"Error occurred during search: {e}"
-    finally:
-        conn.close()
-
-def update_character_chat(chat_id: int, chat_history: List[Tuple[str, str]]) -> bool:
-    """Update an existing chat history."""
-    conn = sqlite3.connect(chat_DB_PATH)
-    cursor = conn.cursor()
-    try:
-        chat_history_json = json.dumps(chat_history)
-        cursor.execute("""
-            UPDATE CharacterChats
-            SET chat_history = ?
-            WHERE id = ?
-        """, (
-            chat_history_json,
-            chat_id
-        ))
-        conn.commit()
-        return cursor.rowcount > 0
-    except sqlite3.Error as e:
-        logging.error(f"Error updating character chat: {e}")
-        return False
-    finally:
-        conn.close()
-
-
-def delete_character_chat(chat_id: int) -> bool:
-    """Delete a specific chat."""
-    conn = sqlite3.connect(chat_DB_PATH)
-    cursor = conn.cursor()
-    try:
-        cursor.execute("DELETE FROM CharacterChats WHERE id = ?", (chat_id,))
-        conn.commit()
-        return cursor.rowcount > 0
-    except sqlite3.Error as e:
-        logging.error(f"Error deleting character chat: {e}")
-        return False
-    finally:
-        conn.close()
-
-def fetch_keywords_for_chats(keywords: List[str]) -> List[int]:
-    """
-    Fetch chat IDs associated with any of the specified keywords.
-
-    Args:
-        keywords (List[str]): List of keywords to search for.
-
-    Returns:
-        List[int]: List of chat IDs associated with the keywords.
-    """
-    if not keywords:
-        return []
-
-    conn = sqlite3.connect(chat_DB_PATH)
-    cursor = conn.cursor()
-    try:
-        # Construct the WHERE clause to search for each keyword
-        keyword_clauses = " OR ".join(["keyword = ?"] * len(keywords))
-        sql_query = f"SELECT DISTINCT chat_id FROM ChatKeywords WHERE {keyword_clauses}"
-        cursor.execute(sql_query, keywords)
-        rows = cursor.fetchall()
-        chat_ids = [row[0] for row in rows]
-        return chat_ids
-    except Exception as e:
-        logging.error(f"Error in fetch_keywords_for_chats: {e}")
-        return []
-    finally:
-        conn.close()
-
-def save_chat_history_to_character_db(character_id: int, conversation_name: str, chat_history: List[Tuple[str, str]]) -> Optional[int]:
-    """Save chat history to the CharacterChats table.
-
-    Returns the ID of the inserted chat or None if failed.
-    """
-    return add_character_chat(character_id, conversation_name, chat_history)
-
-def migrate_chat_to_media_db():
-    pass
-
-
-def search_db(query: str, fields: List[str], where_clause: str = "", page: int = 1, results_per_page: int = 5) -> List[Dict[str, Any]]:
-    """
-    Perform a full-text search on specified fields with optional filtering and pagination.
-
-    Args:
-        query (str): The search query.
-        fields (List[str]): List of fields to search in.
-        where_clause (str, optional): Additional SQL WHERE clause to filter results.
-        page (int, optional): Page number for pagination.
-        results_per_page (int, optional): Number of results per page.
-
-    Returns:
-        List[Dict[str, Any]]: List of matching chat records with content and metadata.
-    """
-    if not query.strip():
-        return []
-
-    conn = sqlite3.connect(chat_DB_PATH)
-    cursor = conn.cursor()
-    try:
-        # Construct the MATCH query for FTS5
-        match_query = " AND ".join(fields) + f" MATCH ?"
-        # Adjust the query with the fields
-        fts_query = f"""
-            SELECT CharacterChats.id, CharacterChats.conversation_name, CharacterChats.chat_history
-            FROM CharacterChats_fts
-            JOIN CharacterChats ON CharacterChats_fts.rowid = CharacterChats.id
-            WHERE {match_query}
-        """
-        if where_clause:
-            fts_query += f" AND ({where_clause})"
-        fts_query += " ORDER BY rank LIMIT ? OFFSET ?"
-        offset = (page - 1) * results_per_page
-        cursor.execute(fts_query, (query, results_per_page, offset))
-        rows = cursor.fetchall()
-        columns = [description[0] for description in cursor.description]
-        results = [dict(zip(columns, row)) for row in rows]
-        return results
-    except Exception as e:
-        logging.error(f"Error in search_db: {e}")
-        return []
-    finally:
-        conn.close()
-
-
-def perform_full_text_search_chat(query: str, relevant_chat_ids: List[int], page: int = 1, results_per_page: int = 5) -> \
-List[Dict[str, Any]]:
-    """
-    Perform a full-text search within the specified chat IDs using FTS5.
-
-    Args:
-        query (str): The user's query.
-        relevant_chat_ids (List[int]): List of chat IDs to search within.
-        page (int): Pagination page number.
-        results_per_page (int): Number of results per page.
-
-    Returns:
-        List[Dict[str, Any]]: List of search results with content and metadata.
-    """
-    try:
-        # Construct a WHERE clause to limit the search to relevant chat IDs
-        where_clause = " OR ".join([f"media_id = {chat_id}" for chat_id in relevant_chat_ids])
-        if not where_clause:
-            where_clause = "1"  # No restriction if no chat IDs
-
-        # Perform full-text search using FTS5
-        fts_results = search_db(query, ["content"], where_clause, page=page, results_per_page=results_per_page)
-
-        filtered_fts_results = [
-            {
-                "content": result['content'],
-                "metadata": {"media_id": result['id']}
-            }
-            for result in fts_results
-            if result['id'] in relevant_chat_ids
-        ]
-        return filtered_fts_results
-    except Exception as e:
-        logging.error(f"Error in perform_full_text_search_chat: {str(e)}")
-        return []
-
-
-def fetch_all_chats() -> List[Dict[str, Any]]:
-    """
-    Fetch all chat messages from the database.
-
-    Returns:
-        List[Dict[str, Any]]: List of chat messages with relevant metadata.
-    """
-    try:
-        chats = get_character_chats()  # Modify this function to retrieve all chats
-        return chats
-    except Exception as e:
-        logging.error(f"Error fetching all chats: {str(e)}")
-        return []
-
-#
-# End of Character_Chat_DB.py
-#######################################################################################################################
+# character_chat_db.py
+# Database functions for managing character cards and chat histories.
+# #
+# Imports
+import configparser
+import sqlite3
+import json
+import os
+import sys
+from typing import List, Dict, Optional, Tuple, Any, Union
+
+from App_Function_Libraries.Utils.Utils import get_database_dir, get_project_relative_path, get_database_path
+from Tests.Chat_APIs.Chat_APIs_Integration_test import logging
+
+#
+#######################################################################################################################
+#
+#
+
+def ensure_database_directory():
+    os.makedirs(get_database_dir(), exist_ok=True)
+
+ensure_database_directory()
+
+
+# Construct the path to the config file
+config_path = get_project_relative_path('Config_Files/config.txt')
+
+# Read the config file
+config = configparser.ConfigParser()
+config.read(config_path)
+
+# Get the chat db path from the config, or use the default if not specified
+chat_DB_PATH = config.get('Database', 'chatDB_path', fallback=get_database_path('chatDB.db'))
+print(f"Chat Database path: {chat_DB_PATH}")
+
+########################################################################################################
+#
+# Functions
+
+# FIXME - Setup properly and test/add documentation for its existence...
+def initialize_database():
+    """Initialize the SQLite database with required tables and FTS5 virtual tables."""
+    conn = None
+    try:
+        conn = sqlite3.connect(chat_DB_PATH)
+        cursor = conn.cursor()
+
+        # Enable foreign key constraints
+        cursor.execute("PRAGMA foreign_keys = ON;")
+
+        # Create CharacterCards table with V2 fields
+        cursor.execute("""
+        CREATE TABLE IF NOT EXISTS CharacterCards (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            name TEXT UNIQUE NOT NULL,
+            description TEXT,
+            personality TEXT,
+            scenario TEXT,
+            image BLOB,
+            post_history_instructions TEXT,
+            first_mes TEXT,
+            mes_example TEXT,
+            creator_notes TEXT,
+            system_prompt TEXT,
+            alternate_greetings TEXT,
+            tags TEXT,
+            creator TEXT,
+            character_version TEXT,
+            extensions TEXT,
+            created_at DATETIME DEFAULT CURRENT_TIMESTAMP
+        );
+        """)
+
+        # Create CharacterChats table
+        cursor.execute("""
+        CREATE TABLE IF NOT EXISTS CharacterChats (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            character_id INTEGER NOT NULL,
+            conversation_name TEXT,
+            chat_history TEXT,
+            is_snapshot BOOLEAN DEFAULT FALSE,
+            created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+            FOREIGN KEY (character_id) REFERENCES CharacterCards(id) ON DELETE CASCADE
+        );
+        """)
+
+        # Create FTS5 virtual table for CharacterChats
+        cursor.execute("""
+        CREATE VIRTUAL TABLE IF NOT EXISTS CharacterChats_fts USING fts5(
+            conversation_name,
+            chat_history,
+            content='CharacterChats',
+            content_rowid='id'
+        );
+        """)
+
+        # Create triggers to keep FTS5 table in sync with CharacterChats
+        cursor.executescript("""
+        CREATE TRIGGER IF NOT EXISTS CharacterChats_ai AFTER INSERT ON CharacterChats BEGIN
+            INSERT INTO CharacterChats_fts(rowid, conversation_name, chat_history)
+            VALUES (new.id, new.conversation_name, new.chat_history);
+        END;
+
+        CREATE TRIGGER IF NOT EXISTS CharacterChats_ad AFTER DELETE ON CharacterChats BEGIN
+            DELETE FROM CharacterChats_fts WHERE rowid = old.id;
+        END;
+
+        CREATE TRIGGER IF NOT EXISTS CharacterChats_au AFTER UPDATE ON CharacterChats BEGIN
+            UPDATE CharacterChats_fts SET conversation_name = new.conversation_name, chat_history = new.chat_history
+            WHERE rowid = new.id;
+        END;
+        """)
+
+        # Create ChatKeywords table
+        cursor.execute("""
+        CREATE TABLE IF NOT EXISTS ChatKeywords (
+            chat_id INTEGER NOT NULL,
+            keyword TEXT NOT NULL,
+            FOREIGN KEY (chat_id) REFERENCES CharacterChats(id) ON DELETE CASCADE
+        );
+        """)
+
+        # Create indexes for faster searches
+        cursor.execute("""
+        CREATE INDEX IF NOT EXISTS idx_chatkeywords_keyword ON ChatKeywords(keyword);
+        """)
+        cursor.execute("""
+        CREATE INDEX IF NOT EXISTS idx_chatkeywords_chat_id ON ChatKeywords(chat_id);
+        """)
+
+        conn.commit()
+        logging.info("Database initialized successfully.")
+    except sqlite3.Error as e:
+        logging.error(f"SQLite error occurred during database initialization: {e}")
+        if conn:
+            conn.rollback()
+        raise
+    except Exception as e:
+        logging.error(f"Unexpected error occurred during database initialization: {e}")
+        if conn:
+            conn.rollback()
+        raise
+    finally:
+        if conn:
+            conn.close()
+
+# Call initialize_database() at the start of your application
+def setup_chat_database():
+    try:
+        initialize_database()
+    except Exception as e:
+        logging.critical(f"Failed to initialize database: {e}")
+        sys.exit(1)
+
+setup_chat_database()
+
+########################################################################################################
+#
+# Character Card handling
+
+def parse_character_card(card_data: Dict[str, Any]) -> Dict[str, Any]:
+    """Parse and validate a character card according to V2 specification."""
+    v2_data = {
+        'name': card_data.get('name', ''),
+        'description': card_data.get('description', ''),
+        'personality': card_data.get('personality', ''),
+        'scenario': card_data.get('scenario', ''),
+        'first_mes': card_data.get('first_mes', ''),
+        'mes_example': card_data.get('mes_example', ''),
+        'creator_notes': card_data.get('creator_notes', ''),
+        'system_prompt': card_data.get('system_prompt', ''),
+        'post_history_instructions': card_data.get('post_history_instructions', ''),
+        'alternate_greetings': json.dumps(card_data.get('alternate_greetings', [])),
+        'tags': json.dumps(card_data.get('tags', [])),
+        'creator': card_data.get('creator', ''),
+        'character_version': card_data.get('character_version', ''),
+        'extensions': json.dumps(card_data.get('extensions', {}))
+    }
+
+    # Handle 'image' separately as it might be binary data
+    if 'image' in card_data:
+        v2_data['image'] = card_data['image']
+
+    return v2_data
+
+
+def add_character_card(card_data: Dict[str, Any]) -> Optional[int]:
+    """Add or update a character card in the database."""
+    conn = sqlite3.connect(chat_DB_PATH)
+    cursor = conn.cursor()
+    try:
+        parsed_card = parse_character_card(card_data)
+
+        # Check if character already exists
+        cursor.execute("SELECT id FROM CharacterCards WHERE name = ?", (parsed_card['name'],))
+        row = cursor.fetchone()
+
+        if row:
+            # Update existing character
+            character_id = row[0]
+            update_query = """
+                UPDATE CharacterCards
+                SET description = ?, personality = ?, scenario = ?, image = ?, 
+                    post_history_instructions = ?, first_mes = ?, mes_example = ?,
+                    creator_notes = ?, system_prompt = ?, alternate_greetings = ?,
+                    tags = ?, creator = ?, character_version = ?, extensions = ?
+                WHERE id = ?
+            """
+            cursor.execute(update_query, (
+                parsed_card['description'], parsed_card['personality'], parsed_card['scenario'],
+                parsed_card['image'], parsed_card['post_history_instructions'], parsed_card['first_mes'],
+                parsed_card['mes_example'], parsed_card['creator_notes'], parsed_card['system_prompt'],
+                parsed_card['alternate_greetings'], parsed_card['tags'], parsed_card['creator'],
+                parsed_card['character_version'], parsed_card['extensions'], character_id
+            ))
+        else:
+            # Insert new character
+            insert_query = """
+                INSERT INTO CharacterCards (name, description, personality, scenario, image, 
+                post_history_instructions, first_mes, mes_example, creator_notes, system_prompt, 
+                alternate_greetings, tags, creator, character_version, extensions)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            """
+            cursor.execute(insert_query, (
+                parsed_card['name'], parsed_card['description'], parsed_card['personality'],
+                parsed_card['scenario'], parsed_card['image'], parsed_card['post_history_instructions'],
+                parsed_card['first_mes'], parsed_card['mes_example'], parsed_card['creator_notes'],
+                parsed_card['system_prompt'], parsed_card['alternate_greetings'], parsed_card['tags'],
+                parsed_card['creator'], parsed_card['character_version'], parsed_card['extensions']
+            ))
+            character_id = cursor.lastrowid
+
+        conn.commit()
+        return character_id
+    except sqlite3.IntegrityError as e:
+        logging.error(f"Error adding character card: {e}")
+        return None
+    except Exception as e:
+        logging.error(f"Unexpected error adding character card: {e}")
+        return None
+    finally:
+        conn.close()
+
+# def add_character_card(card_data: Dict) -> Optional[int]:
+#     """Add or update a character card in the database.
+#
+#     Returns the ID of the inserted character or None if failed.
+#     """
+#     conn = sqlite3.connect(chat_DB_PATH)
+#     cursor = conn.cursor()
+#     try:
+#         # Ensure all required fields are present
+#         required_fields = ['name', 'description', 'personality', 'scenario', 'image', 'post_history_instructions', 'first_message']
+#         for field in required_fields:
+#             if field not in card_data:
+#                 card_data[field] = ''  # Assign empty string if field is missing
+#
+#         # Check if character already exists
+#         cursor.execute("SELECT id FROM CharacterCards WHERE name = ?", (card_data['name'],))
+#         row = cursor.fetchone()
+#
+#         if row:
+#             # Update existing character
+#             character_id = row[0]
+#             cursor.execute("""
+#                 UPDATE CharacterCards
+#                 SET description = ?, personality = ?, scenario = ?, image = ?, post_history_instructions = ?, first_message = ?
+#                 WHERE id = ?
+#             """, (
+#                 card_data['description'],
+#                 card_data['personality'],
+#                 card_data['scenario'],
+#                 card_data['image'],
+#                 card_data['post_history_instructions'],
+#                 card_data['first_message'],
+#                 character_id
+#             ))
+#         else:
+#             # Insert new character
+#             cursor.execute("""
+#                 INSERT INTO CharacterCards (name, description, personality, scenario, image, post_history_instructions, first_message)
+#                 VALUES (?, ?, ?, ?, ?, ?, ?)
+#             """, (
+#                 card_data['name'],
+#                 card_data['description'],
+#                 card_data['personality'],
+#                 card_data['scenario'],
+#                 card_data['image'],
+#                 card_data['post_history_instructions'],
+#                 card_data['first_message']
+#             ))
+#             character_id = cursor.lastrowid
+#
+#         conn.commit()
+#         return cursor.lastrowid
+#     except sqlite3.IntegrityError as e:
+#         logging.error(f"Error adding character card: {e}")
+#         return None
+#     except Exception as e:
+#         logging.error(f"Unexpected error adding character card: {e}")
+#         return None
+#     finally:
+#         conn.close()
+
+
+def get_character_cards() -> List[Dict]:
+    """Retrieve all character cards from the database."""
+    logging.debug(f"Fetching characters from DB: {chat_DB_PATH}")
+    conn = sqlite3.connect(chat_DB_PATH)
+    cursor = conn.cursor()
+    cursor.execute("SELECT * FROM CharacterCards")
+    rows = cursor.fetchall()
+    columns = [description[0] for description in cursor.description]
+    conn.close()
+    characters = [dict(zip(columns, row)) for row in rows]
+    #logging.debug(f"Characters fetched from DB: {characters}")
+    return characters
+
+
+def get_character_card_by_id(character_id: Union[int, Dict[str, Any]]) -> Optional[Dict[str, Any]]:
+    """
+    Retrieve a single character card by its ID.
+
+    Args:
+        character_id: Can be either an integer ID or a dictionary containing character data.
+
+    Returns:
+        A dictionary containing the character card data, or None if not found.
+    """
+    conn = sqlite3.connect(chat_DB_PATH)
+    cursor = conn.cursor()
+    try:
+        if isinstance(character_id, dict):
+            # If a dictionary is passed, assume it's already a character card
+            return character_id
+        elif isinstance(character_id, int):
+            # If an integer is passed, fetch the character from the database
+            cursor.execute("SELECT * FROM CharacterCards WHERE id = ?", (character_id,))
+            row = cursor.fetchone()
+            if row:
+                columns = [description[0] for description in cursor.description]
+                return dict(zip(columns, row))
+        else:
+            logging.warning(f"Invalid type for character_id: {type(character_id)}")
+        return None
+    except Exception as e:
+        logging.error(f"Error in get_character_card_by_id: {e}")
+        return None
+    finally:
+        conn.close()
+
+
+def update_character_card(character_id: int, card_data: Dict) -> bool:
+    """Update an existing character card."""
+    conn = sqlite3.connect(chat_DB_PATH)
+    cursor = conn.cursor()
+    try:
+        cursor.execute("""
+            UPDATE CharacterCards
+            SET name = ?, description = ?, personality = ?, scenario = ?, image = ?, post_history_instructions = ?, first_message = ?
+            WHERE id = ?
+        """, (
+            card_data.get('name'),
+            card_data.get('description'),
+            card_data.get('personality'),
+            card_data.get('scenario'),
+            card_data.get('image'),
+            card_data.get('post_history_instructions', ''),
+            card_data.get('first_message', "Hello! I'm ready to chat."),
+            character_id
+        ))
+        conn.commit()
+        return cursor.rowcount > 0
+    except sqlite3.IntegrityError as e:
+        logging.error(f"Error updating character card: {e}")
+        return False
+    finally:
+        conn.close()
+
+
+def delete_character_card(character_id: int) -> bool:
+    """Delete a character card and its associated chats."""
+    conn = sqlite3.connect(chat_DB_PATH)
+    cursor = conn.cursor()
+    try:
+        # Delete associated chats first due to foreign key constraint
+        cursor.execute("DELETE FROM CharacterChats WHERE character_id = ?", (character_id,))
+        cursor.execute("DELETE FROM CharacterCards WHERE id = ?", (character_id,))
+        conn.commit()
+        return cursor.rowcount > 0
+    except sqlite3.Error as e:
+        logging.error(f"Error deleting character card: {e}")
+        return False
+    finally:
+        conn.close()
+
+
+def add_character_chat(character_id: int, conversation_name: str, chat_history: List[Tuple[str, str]], keywords: Optional[List[str]] = None, is_snapshot: bool = False) -> Optional[int]:
+    """
+    Add a new chat history for a character, optionally associating keywords.
+
+    Args:
+        character_id (int): The ID of the character.
+        conversation_name (str): Name of the conversation.
+        chat_history (List[Tuple[str, str]]): List of (user, bot) message tuples.
+        keywords (Optional[List[str]]): List of keywords to associate with this chat.
+        is_snapshot (bool, optional): Whether this chat is a snapshot.
+
+    Returns:
+        Optional[int]: The ID of the inserted chat or None if failed.
+    """
+    conn = sqlite3.connect(chat_DB_PATH)
+    cursor = conn.cursor()
+    try:
+        chat_history_json = json.dumps(chat_history)
+        cursor.execute("""
+            INSERT INTO CharacterChats (character_id, conversation_name, chat_history, is_snapshot)
+            VALUES (?, ?, ?, ?)
+        """, (
+            character_id,
+            conversation_name,
+            chat_history_json,
+            is_snapshot
+        ))
+        chat_id = cursor.lastrowid
+
+        if keywords:
+            # Insert keywords into ChatKeywords table
+            keyword_records = [(chat_id, keyword.strip().lower()) for keyword in keywords]
+            cursor.executemany("""
+                INSERT INTO ChatKeywords (chat_id, keyword)
+                VALUES (?, ?)
+            """, keyword_records)
+
+        conn.commit()
+        return chat_id
+    except sqlite3.Error as e:
+        logging.error(f"Error adding character chat: {e}")
+        return None
+    finally:
+        conn.close()
+
+
+def get_character_chats(character_id: Optional[int] = None) -> List[Dict]:
+    """Retrieve all chats, or chats for a specific character if character_id is provided."""
+    conn = sqlite3.connect(chat_DB_PATH)
+    cursor = conn.cursor()
+    if character_id is not None:
+        cursor.execute("SELECT * FROM CharacterChats WHERE character_id = ?", (character_id,))
+    else:
+        cursor.execute("SELECT * FROM CharacterChats")
+    rows = cursor.fetchall()
+    columns = [description[0] for description in cursor.description]
+    conn.close()
+    return [dict(zip(columns, row)) for row in rows]
+
+
+def get_character_chat_by_id(chat_id: int) -> Optional[Dict]:
+    """Retrieve a single chat by its ID."""
+    conn = sqlite3.connect(chat_DB_PATH)
+    cursor = conn.cursor()
+    cursor.execute("SELECT * FROM CharacterChats WHERE id = ?", (chat_id,))
+    row = cursor.fetchone()
+    conn.close()
+    if row:
+        columns = [description[0] for description in cursor.description]
+        chat = dict(zip(columns, row))
+        chat['chat_history'] = json.loads(chat['chat_history'])
+        return chat
+    return None
+
+
+def search_character_chats(query: str, character_id: Optional[int] = None) -> Tuple[List[Dict], str]:
+    """
+    Search for character chats using FTS5, optionally filtered by character_id.
+
+    Args:
+        query (str): The search query.
+        character_id (Optional[int]): The ID of the character to filter chats by.
+
+    Returns:
+        Tuple[List[Dict], str]: A list of matching chats and a status message.
+    """
+    if not query.strip():
+        return [], "Please enter a search query."
+
+    conn = sqlite3.connect(chat_DB_PATH)
+    cursor = conn.cursor()
+    try:
+        if character_id is not None:
+            # Search with character_id filter
+            cursor.execute("""
+                SELECT CharacterChats.id, CharacterChats.conversation_name, CharacterChats.chat_history
+                FROM CharacterChats_fts
+                JOIN CharacterChats ON CharacterChats_fts.rowid = CharacterChats.id
+                WHERE CharacterChats_fts MATCH ? AND CharacterChats.character_id = ?
+                ORDER BY rank
+            """, (query, character_id))
+        else:
+            # Search without character_id filter
+            cursor.execute("""
+                SELECT CharacterChats.id, CharacterChats.conversation_name, CharacterChats.chat_history
+                FROM CharacterChats_fts
+                JOIN CharacterChats ON CharacterChats_fts.rowid = CharacterChats.id
+                WHERE CharacterChats_fts MATCH ?
+                ORDER BY rank
+            """, (query,))
+
+        rows = cursor.fetchall()
+        columns = [description[0] for description in cursor.description]
+        results = [dict(zip(columns, row)) for row in rows]
+
+        if character_id is not None:
+            status_message = f"Found {len(results)} chat(s) matching '{query}' for the selected character."
+        else:
+            status_message = f"Found {len(results)} chat(s) matching '{query}' across all characters."
+
+        return results, status_message
+    except Exception as e:
+        logging.error(f"Error searching chats with FTS5: {e}")
+        return [], f"Error occurred during search: {e}"
+    finally:
+        conn.close()
+
+def update_character_chat(chat_id: int, chat_history: List[Tuple[str, str]]) -> bool:
+    """Update an existing chat history."""
+    conn = sqlite3.connect(chat_DB_PATH)
+    cursor = conn.cursor()
+    try:
+        chat_history_json = json.dumps(chat_history)
+        cursor.execute("""
+            UPDATE CharacterChats
+            SET chat_history = ?
+            WHERE id = ?
+        """, (
+            chat_history_json,
+            chat_id
+        ))
+        conn.commit()
+        return cursor.rowcount > 0
+    except sqlite3.Error as e:
+        logging.error(f"Error updating character chat: {e}")
+        return False
+    finally:
+        conn.close()
+
+
+def delete_character_chat(chat_id: int) -> bool:
+    """Delete a specific chat."""
+    conn = sqlite3.connect(chat_DB_PATH)
+    cursor = conn.cursor()
+    try:
+        cursor.execute("DELETE FROM CharacterChats WHERE id = ?", (chat_id,))
+        conn.commit()
+        return cursor.rowcount > 0
+    except sqlite3.Error as e:
+        logging.error(f"Error deleting character chat: {e}")
+        return False
+    finally:
+        conn.close()
+
+def fetch_keywords_for_chats(keywords: List[str]) -> List[int]:
+    """
+    Fetch chat IDs associated with any of the specified keywords.
+
+    Args:
+        keywords (List[str]): List of keywords to search for.
+
+    Returns:
+        List[int]: List of chat IDs associated with the keywords.
+    """
+    if not keywords:
+        return []
+
+    conn = sqlite3.connect(chat_DB_PATH)
+    cursor = conn.cursor()
+    try:
+        # Construct the WHERE clause to search for each keyword
+        keyword_clauses = " OR ".join(["keyword = ?"] * len(keywords))
+        sql_query = f"SELECT DISTINCT chat_id FROM ChatKeywords WHERE {keyword_clauses}"
+        cursor.execute(sql_query, keywords)
+        rows = cursor.fetchall()
+        chat_ids = [row[0] for row in rows]
+        return chat_ids
+    except Exception as e:
+        logging.error(f"Error in fetch_keywords_for_chats: {e}")
+        return []
+    finally:
+        conn.close()
+
+def save_chat_history_to_character_db(character_id: int, conversation_name: str, chat_history: List[Tuple[str, str]]) -> Optional[int]:
+    """Save chat history to the CharacterChats table.
+
+    Returns the ID of the inserted chat or None if failed.
+    """
+    return add_character_chat(character_id, conversation_name, chat_history)
+
+def migrate_chat_to_media_db():
+    pass
+
+
+def search_db(query: str, fields: List[str], where_clause: str = "", page: int = 1, results_per_page: int = 5) -> List[Dict[str, Any]]:
+    """
+    Perform a full-text search on specified fields with optional filtering and pagination.
+
+    Args:
+        query (str): The search query.
+        fields (List[str]): List of fields to search in.
+        where_clause (str, optional): Additional SQL WHERE clause to filter results.
+        page (int, optional): Page number for pagination.
+        results_per_page (int, optional): Number of results per page.
+
+    Returns:
+        List[Dict[str, Any]]: List of matching chat records with content and metadata.
+    """
+    if not query.strip():
+        return []
+
+    conn = sqlite3.connect(chat_DB_PATH)
+    cursor = conn.cursor()
+    try:
+        # Construct the MATCH query for FTS5
+        match_query = " AND ".join(fields) + f" MATCH ?"
+        # Adjust the query with the fields
+        fts_query = f"""
+            SELECT CharacterChats.id, CharacterChats.conversation_name, CharacterChats.chat_history
+            FROM CharacterChats_fts
+            JOIN CharacterChats ON CharacterChats_fts.rowid = CharacterChats.id
+            WHERE {match_query}
+        """
+        if where_clause:
+            fts_query += f" AND ({where_clause})"
+        fts_query += " ORDER BY rank LIMIT ? OFFSET ?"
+        offset = (page - 1) * results_per_page
+        cursor.execute(fts_query, (query, results_per_page, offset))
+        rows = cursor.fetchall()
+        columns = [description[0] for description in cursor.description]
+        results = [dict(zip(columns, row)) for row in rows]
+        return results
+    except Exception as e:
+        logging.error(f"Error in search_db: {e}")
+        return []
+    finally:
+        conn.close()
+
+
+def perform_full_text_search_chat(query: str, relevant_chat_ids: List[int], page: int = 1, results_per_page: int = 5) -> \
+List[Dict[str, Any]]:
+    """
+    Perform a full-text search within the specified chat IDs using FTS5.
+
+    Args:
+        query (str): The user's query.
+        relevant_chat_ids (List[int]): List of chat IDs to search within.
+        page (int): Pagination page number.
+        results_per_page (int): Number of results per page.
+
+    Returns:
+        List[Dict[str, Any]]: List of search results with content and metadata.
+    """
+    try:
+        # Construct a WHERE clause to limit the search to relevant chat IDs
+        where_clause = " OR ".join([f"media_id = {chat_id}" for chat_id in relevant_chat_ids])
+        if not where_clause:
+            where_clause = "1"  # No restriction if no chat IDs
+
+        # Perform full-text search using FTS5
+        fts_results = search_db(query, ["content"], where_clause, page=page, results_per_page=results_per_page)
+
+        filtered_fts_results = [
+            {
+                "content": result['content'],
+                "metadata": {"media_id": result['id']}
+            }
+            for result in fts_results
+            if result['id'] in relevant_chat_ids
+        ]
+        return filtered_fts_results
+    except Exception as e:
+        logging.error(f"Error in perform_full_text_search_chat: {str(e)}")
+        return []
+
+
+def fetch_all_chats() -> List[Dict[str, Any]]:
+    """
+    Fetch all chat messages from the database.
+
+    Returns:
+        List[Dict[str, Any]]: List of chat messages with relevant metadata.
+    """
+    try:
+        chats = get_character_chats()  # Modify this function to retrieve all chats
+        return chats
+    except Exception as e:
+        logging.error(f"Error fetching all chats: {str(e)}")
+        return []
+
+#
+# End of Character_Chat_DB.py
+#######################################################################################################################

App_Function_Libraries/DB/RAG_QA_Chat_DB.py

@@ -0,0 +1,461 @@
+# RAG_QA_Chat_DB.py
+# Description: This file contains the database operations for the RAG QA Chat + Notes system.
+#
+# Imports
+import logging
+import re
+import sqlite3
+from contextlib import contextmanager
+from datetime import datetime
+#
+# External Imports
+#
+# Local Imports
+#
+########################################################################################################################
+#
+# Functions:
+
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Database schema
+SCHEMA_SQL = '''
+-- Table for storing chat messages
+CREATE TABLE IF NOT EXISTS rag_qa_chats (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    conversation_id TEXT NOT NULL,
+    timestamp DATETIME NOT NULL,
+    role TEXT NOT NULL,
+    content TEXT NOT NULL
+);
+
+-- Table for storing conversation metadata
+CREATE TABLE IF NOT EXISTS conversation_metadata (
+    conversation_id TEXT PRIMARY KEY,
+    created_at DATETIME NOT NULL,
+    last_updated DATETIME NOT NULL,
+    title TEXT NOT NULL
+);
+
+-- Table for storing keywords
+CREATE TABLE IF NOT EXISTS rag_qa_keywords (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    keyword TEXT NOT NULL UNIQUE
+);
+
+-- Table for linking keywords to conversations
+CREATE TABLE IF NOT EXISTS rag_qa_conversation_keywords (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    conversation_id TEXT NOT NULL,
+    keyword_id INTEGER NOT NULL,
+    FOREIGN KEY (conversation_id) REFERENCES conversation_metadata(conversation_id),
+    FOREIGN KEY (keyword_id) REFERENCES rag_qa_keywords(id)
+);
+
+-- Table for storing keyword collections
+CREATE TABLE IF NOT EXISTS rag_qa_keyword_collections (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    name TEXT NOT NULL UNIQUE,
+    parent_id INTEGER,
+    FOREIGN KEY (parent_id) REFERENCES rag_qa_keyword_collections(id)
+);
+
+-- Table for linking keywords to collections
+CREATE TABLE IF NOT EXISTS rag_qa_collection_keywords (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    collection_id INTEGER NOT NULL,
+    keyword_id INTEGER NOT NULL,
+    FOREIGN KEY (collection_id) REFERENCES rag_qa_keyword_collections(id),
+    FOREIGN KEY (keyword_id) REFERENCES rag_qa_keywords(id)
+);
+
+-- Table for storing notes
+CREATE TABLE IF NOT EXISTS rag_qa_notes (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    conversation_id TEXT NOT NULL,
+    content TEXT NOT NULL,
+    timestamp DATETIME NOT NULL,
+    FOREIGN KEY (conversation_id) REFERENCES conversation_metadata(conversation_id)
+);
+
+-- Table for linking notes to keywords
+CREATE TABLE IF NOT EXISTS rag_qa_note_keywords (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    note_id INTEGER NOT NULL,
+    keyword_id INTEGER NOT NULL,
+    FOREIGN KEY (note_id) REFERENCES rag_qa_notes(id),
+    FOREIGN KEY (keyword_id) REFERENCES rag_qa_keywords(id)
+);
+
+-- Indexes for improved query performance
+CREATE INDEX IF NOT EXISTS idx_rag_qa_chats_conversation_id ON rag_qa_chats(conversation_id);
+CREATE INDEX IF NOT EXISTS idx_rag_qa_chats_timestamp ON rag_qa_chats(timestamp);
+CREATE INDEX IF NOT EXISTS idx_rag_qa_keywords_keyword ON rag_qa_keywords(keyword);
+CREATE INDEX IF NOT EXISTS idx_rag_qa_conversation_keywords_conversation_id ON rag_qa_conversation_keywords(conversation_id);
+CREATE INDEX IF NOT EXISTS idx_rag_qa_conversation_keywords_keyword_id ON rag_qa_conversation_keywords(keyword_id);
+CREATE INDEX IF NOT EXISTS idx_rag_qa_keyword_collections_parent_id ON rag_qa_keyword_collections(parent_id);
+CREATE INDEX IF NOT EXISTS idx_rag_qa_collection_keywords_collection_id ON rag_qa_collection_keywords(collection_id);
+CREATE INDEX IF NOT EXISTS idx_rag_qa_collection_keywords_keyword_id ON rag_qa_collection_keywords(keyword_id);
+
+-- Full-text search virtual table for chat content
+CREATE VIRTUAL TABLE IF NOT EXISTS rag_qa_chats_fts USING fts5(conversation_id, timestamp, role, content);
+
+-- Trigger to keep the FTS table up to date
+CREATE TRIGGER IF NOT EXISTS rag_qa_chats_ai AFTER INSERT ON rag_qa_chats BEGIN
+  INSERT INTO rag_qa_chats_fts(conversation_id, timestamp, role, content) VALUES (new.conversation_id, new.timestamp, new.role, new.content);
+END;
+'''
+
+# Database connection management
+@contextmanager
+def get_db_connection():
+    conn = sqlite3.connect('rag_qa_chat.db')
+    try:
+        yield conn
+    finally:
+        conn.close()
+
+@contextmanager
+def transaction():
+    with get_db_connection() as conn:
+        try:
+            conn.execute('BEGIN TRANSACTION')
+            yield conn
+            conn.commit()
+        except Exception:
+            conn.rollback()
+            raise
+
+def execute_query(query, params=None, transaction_conn=None):
+    if transaction_conn:
+        cursor = transaction_conn.cursor()
+        if params:
+            cursor.execute(query, params)
+        else:
+            cursor.execute(query)
+        return cursor.fetchall()
+    else:
+        with get_db_connection() as conn:
+            cursor = conn.cursor()
+            if params:
+                cursor.execute(query, params)
+            else:
+                cursor.execute(query)
+            conn.commit()
+            return cursor.fetchall()
+
+def create_tables():
+    with get_db_connection() as conn:
+        conn.executescript(SCHEMA_SQL)
+    logger.info("All RAG QA Chat tables created successfully")
+
+# Initialize the database
+create_tables()
+
+# Input validation
+def validate_keyword(keyword):
+    if not isinstance(keyword, str):
+        raise ValueError("Keyword must be a string")
+    if not keyword.strip():
+        raise ValueError("Keyword cannot be empty or just whitespace")
+    if len(keyword) > 100:
+        raise ValueError("Keyword is too long (max 100 characters)")
+    if not re.match(r'^[a-zA-Z0-9\s\-_]+$', keyword):
+        raise ValueError("Keyword contains invalid characters")
+    return keyword.strip()
+
+def validate_collection_name(name):
+    if not isinstance(name, str):
+        raise ValueError("Collection name must be a string")
+    if not name.strip():
+        raise ValueError("Collection name cannot be empty or just whitespace")
+    if len(name) > 100:
+        raise ValueError("Collection name is too long (max 100 characters)")
+    if not re.match(r'^[a-zA-Z0-9\s\-_]+$', name):
+        raise ValueError("Collection name contains invalid characters")
+    return name.strip()
+
+# Core functions
+def add_keyword(keyword):
+    try:
+        validated_keyword = validate_keyword(keyword)
+        query = "INSERT OR IGNORE INTO rag_qa_keywords (keyword) VALUES (?)"
+        execute_query(query, (validated_keyword,))
+        logger.info(f"Keyword '{validated_keyword}' added successfully")
+    except ValueError as e:
+        logger.error(f"Invalid keyword: {e}")
+        raise
+    except Exception as e:
+        logger.error(f"Error adding keyword '{keyword}': {e}")
+        raise
+
+def create_keyword_collection(name, parent_id=None):
+    try:
+        validated_name = validate_collection_name(name)
+        query = "INSERT INTO rag_qa_keyword_collections (name, parent_id) VALUES (?, ?)"
+        execute_query(query, (validated_name, parent_id))
+        logger.info(f"Keyword collection '{validated_name}' created successfully")
+    except ValueError as e:
+        logger.error(f"Invalid collection name: {e}")
+        raise
+    except Exception as e:
+        logger.error(f"Error creating keyword collection '{name}': {e}")
+        raise
+
+def add_keyword_to_collection(collection_name, keyword):
+    try:
+        validated_collection_name = validate_collection_name(collection_name)
+        validated_keyword = validate_keyword(keyword)
+
+        with transaction() as conn:
+            add_keyword(validated_keyword)
+
+            query = '''
+            INSERT INTO rag_qa_collection_keywords (collection_id, keyword_id)
+            SELECT c.id, k.id
+            FROM rag_qa_keyword_collections c, rag_qa_keywords k
+            WHERE c.name = ? AND k.keyword = ?
+            '''
+            execute_query(query, (validated_collection_name, validated_keyword), conn)
+
+        logger.info(f"Keyword '{validated_keyword}' added to collection '{validated_collection_name}' successfully")
+    except ValueError as e:
+        logger.error(f"Invalid input: {e}")
+        raise
+    except Exception as e:
+        logger.error(f"Error adding keyword '{keyword}' to collection '{collection_name}': {e}")
+        raise
+
+def add_keywords_to_conversation(conversation_id, keywords):
+    if not isinstance(keywords, (list, tuple)):
+        raise ValueError("Keywords must be a list or tuple")
+    try:
+        with transaction() as conn:
+            for keyword in keywords:
+                validated_keyword = validate_keyword(keyword)
+
+                query = "INSERT OR IGNORE INTO rag_qa_keywords (keyword) VALUES (?)"
+                execute_query(query, (validated_keyword,), conn)
+
+                query = '''
+                INSERT INTO rag_qa_conversation_keywords (conversation_id, keyword_id)
+                SELECT ?, id FROM rag_qa_keywords WHERE keyword = ?
+                '''
+                execute_query(query, (conversation_id, validated_keyword), conn)
+
+        logger.info(f"Keywords added to conversation '{conversation_id}' successfully")
+    except ValueError as e:
+        logger.error(f"Invalid keyword: {e}")
+        raise
+    except Exception as e:
+        logger.error(f"Error adding keywords to conversation '{conversation_id}': {e}")
+        raise
+
+def get_keywords_for_conversation(conversation_id):
+    try:
+        query = '''
+        SELECT k.keyword
+        FROM rag_qa_keywords k
+        JOIN rag_qa_conversation_keywords ck ON k.id = ck.keyword_id
+        WHERE ck.conversation_id = ?
+        '''
+        result = execute_query(query, (conversation_id,))
+        keywords = [row[0] for row in result]
+        logger.info(f"Retrieved {len(keywords)} keywords for conversation '{conversation_id}'")
+        return keywords
+    except Exception as e:
+        logger.error(f"Error getting keywords for conversation '{conversation_id}': {e}")
+        raise
+
+def get_keywords_for_collection(collection_name):
+    try:
+        query = '''
+        SELECT k.keyword
+        FROM rag_qa_keywords k
+        JOIN rag_qa_collection_keywords ck ON k.id = ck.keyword_id
+        JOIN rag_qa_keyword_collections c ON ck.collection_id = c.id
+        WHERE c.name = ?
+        '''
+        result = execute_query(query, (collection_name,))
+        keywords = [row[0] for row in result]
+        logger.info(f"Retrieved {len(keywords)} keywords for collection '{collection_name}'")
+        return keywords
+    except Exception as e:
+        logger.error(f"Error getting keywords for collection '{collection_name}': {e}")
+        raise
+
+def save_notes(conversation_id, content):
+    """Save notes to the database."""
+    try:
+        query = "INSERT INTO rag_qa_notes (conversation_id, content, timestamp) VALUES (?, ?, ?)"
+        timestamp = datetime.now().isoformat()
+        execute_query(query, (conversation_id, content, timestamp))
+        logger.info(f"Notes saved for conversation '{conversation_id}'")
+    except Exception as e:
+        logger.error(f"Error saving notes for conversation '{conversation_id}': {e}")
+        raise
+
+def get_notes(conversation_id):
+    """Retrieve notes for a given conversation."""
+    try:
+        query = "SELECT content FROM rag_qa_notes WHERE conversation_id = ?"
+        result = execute_query(query, (conversation_id,))
+        notes = [row[0] for row in result]
+        logger.info(f"Retrieved {len(notes)} notes for conversation '{conversation_id}'")
+        return notes
+    except Exception as e:
+        logger.error(f"Error getting notes for conversation '{conversation_id}': {e}")
+        raise
+
+def clear_notes(conversation_id):
+    """Clear all notes for a given conversation."""
+    try:
+        query = "DELETE FROM rag_qa_notes WHERE conversation_id = ?"
+        execute_query(query, (conversation_id,))
+        logger.info(f"Cleared notes for conversation '{conversation_id}'")
+    except Exception as e:
+        logger.error(f"Error clearing notes for conversation '{conversation_id}': {e}")
+        raise
+
+def add_keywords_to_note(note_id, keywords):
+    """Associate keywords with a note."""
+    try:
+        with transaction() as conn:
+            for keyword in keywords:
+                validated_keyword = validate_keyword(keyword)
+
+                # Insert the keyword into the rag_qa_keywords table if it doesn't exist
+                query = "INSERT OR IGNORE INTO rag_qa_keywords (keyword) VALUES (?)"
+                execute_query(query, (validated_keyword,), conn)
+
+                # Retrieve the keyword ID
+                query = "SELECT id FROM rag_qa_keywords WHERE keyword = ?"
+                keyword_id = execute_query(query, (validated_keyword,), conn)[0][0]
+
+                # Link the note and keyword
+                query = "INSERT INTO rag_qa_note_keywords (note_id, keyword_id) VALUES (?, ?)"
+                execute_query(query, (note_id, keyword_id), conn)
+
+        logger.info(f"Keywords added to note ID '{note_id}' successfully")
+    except Exception as e:
+        logger.error(f"Error adding keywords to note ID '{note_id}': {e}")
+        raise
+
+def get_keywords_for_note(note_id):
+    """Retrieve keywords associated with a given note."""
+    try:
+        query = '''
+        SELECT k.keyword
+        FROM rag_qa_keywords k
+        JOIN rag_qa_note_keywords nk ON k.id = nk.keyword_id
+        WHERE nk.note_id = ?
+        '''
+        result = execute_query(query, (note_id,))
+        keywords = [row[0] for row in result]
+        logger.info(f"Retrieved {len(keywords)} keywords for note ID '{note_id}'")
+        return keywords
+    except Exception as e:
+        logger.error(f"Error getting keywords for note ID '{note_id}': {e}")
+        raise
+
+def clear_keywords_from_note(note_id):
+    """Clear all keywords from a given note."""
+    try:
+        query = "DELETE FROM rag_qa_note_keywords WHERE note_id = ?"
+        execute_query(query, (note_id,))
+        logger.info(f"Cleared keywords for note ID '{note_id}'")
+    except Exception as e:
+        logger.error(f"Error clearing keywords for note ID '{note_id}': {e}")
+        raise
+
+def save_message(conversation_id, role, content):
+    try:
+        query = "INSERT INTO rag_qa_chats (conversation_id, timestamp, role, content) VALUES (?, ?, ?, ?)"
+        timestamp = datetime.now().isoformat()
+        execute_query(query, (conversation_id, timestamp, role, content))
+        logger.info(f"Message saved for conversation '{conversation_id}'")
+    except Exception as e:
+        logger.error(f"Error saving message for conversation '{conversation_id}': {e}")
+        raise
+
+def start_new_conversation(title="Untitled Conversation"):
+    try:
+        conversation_id = datetime.now().isoformat()
+        query = "INSERT INTO conversation_metadata (conversation_id, created_at, last_updated, title) VALUES (?, ?, ?, ?)"
+        now = datetime.now()
+        execute_query(query, (conversation_id, now, now, title))
+        logger.info(f"New conversation '{conversation_id}' started with title '{title}'")
+        return conversation_id
+    except Exception as e:
+        logger.error(f"Error starting new conversation: {e}")
+        raise
+
+# Pagination helper function
+def get_paginated_results(query, params=None, page=1, page_size=20):
+    try:
+        offset = (page - 1) * page_size
+        paginated_query = f"{query} LIMIT ? OFFSET ?"
+        if params:
+            params = tuple(params) + (page_size, offset)
+        else:
+            params = (page_size, offset)
+
+        result = execute_query(paginated_query, params)
+
+        count_query = f"SELECT COUNT(*) FROM ({query})"
+        total_count = execute_query(count_query, params[:-2] if params else None)[0][0]
+
+        total_pages = (total_count + page_size - 1) // page_size
+
+        logger.info(f"Retrieved page {page} of {total_pages} (total items: {total_count})")
+        return result, total_pages, total_count
+    except Exception as e:
+        logger.error(f"Error retrieving paginated results: {e}")
+        raise
+
+def get_all_collections(page=1, page_size=20):
+    try:
+        query = "SELECT name FROM rag_qa_keyword_collections"
+        results, total_pages, total_count = get_paginated_results(query, page=page, page_size=page_size)
+        collections = [row[0] for row in results]
+        logger.info(f"Retrieved {len(collections)} keyword collections (page {page} of {total_pages})")
+        return collections, total_pages, total_count
+    except Exception as e:
+        logger.error(f"Error getting collections: {e}")
+        raise
+
+def search_conversations_by_keywords(keywords, page=1, page_size=20):
+    try:
+        placeholders = ','.join(['?' for _ in keywords])
+        query = f'''
+        SELECT DISTINCT cm.conversation_id, cm.title
+        FROM conversation_metadata cm
+        JOIN rag_qa_conversation_keywords ck ON cm.conversation_id = ck.conversation_id
+        JOIN rag_qa_keywords k ON ck.keyword_id = k.id
+        WHERE k.keyword IN ({placeholders})
+        '''
+        results, total_pages, total_count = get_paginated_results(query, keywords, page, page_size)
+        logger.info(
+            f"Found {total_count} conversations matching keywords: {', '.join(keywords)} (page {page} of {total_pages})")
+        return results, total_pages, total_count
+    except Exception as e:
+        logger.error(f"Error searching conversations by keywords {keywords}: {e}")
+        raise
+
+def load_chat_history(conversation_id, page=1, page_size=50):
+    try:
+        query = "SELECT role, content FROM rag_qa_chats WHERE conversation_id = ? ORDER BY timestamp"
+        results, total_pages, total_count = get_paginated_results(query, (conversation_id,), page, page_size)
+        history = [(msg[1] if msg[0] == 'human' else None, msg[1] if msg[0] == 'ai' else None) for msg in results]
+        logger.info(
+            f"Loaded {len(history)} messages for conversation '{conversation_id}' (page {page} of {total_pages})")
+        return history, total_pages, total_count
+    except Exception as e:
+        logger.error(f"Error loading chat history for conversation '{conversation_id}': {e}")
+        raise
+
+#
+# End of RAG_QA_Chat_DB.py
+####################################################################################################

App_Function_Libraries/DB/SQLite_DB.py

@@ -1000,7 +1000,6 @@ def add_media_version(conn, media_id: int, prompt: str, summary: str) -> None:
 
 
 # Function to search the database with advanced options, including keyword search and full-text search
-
 def sqlite_search_db(search_query: str, search_fields: List[str], keywords: str, page: int = 1, results_per_page: int = 10, connection=None):
     if page < 1:
         raise ValueError("Page number must be 1 or greater.")
@@ -1053,6 +1052,7 @@ def sqlite_search_db(search_query: str, search_fields: List[str], keywords: str,
         with db.get_connection() as conn:
             return execute_query(conn)
 
+
 # Gradio function to handle user input and display results with pagination, with better feedback
 def search_and_display(search_query, search_fields, keywords, page):
     results = sqlite_search_db(search_query, search_fields, keywords, page)

App_Function_Libraries/Gradio_Related.py

@@ -1,423 +1,423 @@
-# Gradio_Related.py
-#########################################
-# Gradio UI Functions Library
-# I fucking hate Gradio.
-#
-#########################################
-#
-# Built-In Imports
-import logging
-import os
-import webbrowser
-
-#
-# Import 3rd-Party Libraries
-import gradio as gr
-#
-# Local Imports
-from App_Function_Libraries.DB.DB_Manager import get_db_config
-from App_Function_Libraries.Gradio_UI.Arxiv_tab import create_arxiv_tab
-from App_Function_Libraries.Gradio_UI.Audio_ingestion_tab import create_audio_processing_tab
-from App_Function_Libraries.Gradio_UI.Book_Ingestion_tab import create_import_book_tab
-#from App_Function_Libraries.Gradio_UI.Character_Chat_tab import create_character_card_interaction_tab, create_character_chat_mgmt_tab, create_custom_character_card_tab, \
-#    create_character_card_validation_tab, create_export_characters_tab
-#from App_Function_Libraries.Gradio_UI.Character_interaction_tab import create_narrator_controlled_conversation_tab, \
-#    create_multiple_character_chat_tab
-from App_Function_Libraries.Gradio_UI.Chat_ui import create_chat_management_tab, \
-    create_chat_interface_four, create_chat_interface_multi_api, create_chat_interface_stacked, create_chat_interface
-from App_Function_Libraries.Gradio_UI.Config_tab import create_config_editor_tab
-from App_Function_Libraries.Gradio_UI.Explain_summarize_tab import create_summarize_explain_tab
-from App_Function_Libraries.Gradio_UI.Export_Functionality import create_export_tab
-from App_Function_Libraries.Gradio_UI.Backup_Functionality import create_backup_tab, create_view_backups_tab, \
-    create_restore_backup_tab
-from App_Function_Libraries.Gradio_UI.Import_Functionality import create_import_single_prompt_tab, \
-    create_import_obsidian_vault_tab, create_import_item_tab, create_import_multiple_prompts_tab
-from App_Function_Libraries.Gradio_UI.Introduction_tab import create_introduction_tab
-from App_Function_Libraries.Gradio_UI.Keywords import create_view_keywords_tab, create_add_keyword_tab, \
-    create_delete_keyword_tab, create_export_keywords_tab
-#from App_Function_Libraries.Gradio_UI.Live_Recording import create_live_recording_tab
-from App_Function_Libraries.Gradio_UI.Llamafile_tab import create_chat_with_llamafile_tab
-#from App_Function_Libraries.Gradio_UI.MMLU_Pro_tab import create_mmlu_pro_tab
-from App_Function_Libraries.Gradio_UI.Media_edit import create_prompt_clone_tab, create_prompt_edit_tab, \
-    create_media_edit_and_clone_tab, create_media_edit_tab
-from App_Function_Libraries.Gradio_UI.Media_wiki_tab import create_mediawiki_import_tab, create_mediawiki_config_tab
-from App_Function_Libraries.Gradio_UI.PDF_ingestion_tab import create_pdf_ingestion_tab, create_pdf_ingestion_test_tab
-from App_Function_Libraries.Gradio_UI.Plaintext_tab_import import create_plain_text_import_tab
-from App_Function_Libraries.Gradio_UI.Podcast_tab import create_podcast_tab
-from App_Function_Libraries.Gradio_UI.Prompt_Suggestion_tab import create_prompt_suggestion_tab
-from App_Function_Libraries.Gradio_UI.RAG_QA_Chat_tab import create_rag_qa_chat_tab
-from App_Function_Libraries.Gradio_UI.Re_summarize_tab import create_resummary_tab
-from App_Function_Libraries.Gradio_UI.Search_Tab import create_prompt_search_tab, \
-    create_search_summaries_tab, create_search_tab
-from App_Function_Libraries.Gradio_UI.RAG_Chat_tab import create_rag_tab
-from App_Function_Libraries.Gradio_UI.Embeddings_tab import create_embeddings_tab, create_view_embeddings_tab, \
-    create_purge_embeddings_tab
-from App_Function_Libraries.Gradio_UI.Trash import create_view_trash_tab, create_empty_trash_tab, \
-    create_delete_trash_tab, create_search_and_mark_trash_tab
-from App_Function_Libraries.Gradio_UI.Utilities import create_utilities_yt_timestamp_tab, create_utilities_yt_audio_tab, \
-    create_utilities_yt_video_tab
-from App_Function_Libraries.Gradio_UI.Video_transcription_tab import create_video_transcription_tab
-from App_Function_Libraries.Gradio_UI.View_tab import create_manage_items_tab
-from App_Function_Libraries.Gradio_UI.Website_scraping_tab import create_website_scraping_tab
-from App_Function_Libraries.Gradio_UI.Chat_Workflows import chat_workflows_tab
-from App_Function_Libraries.Gradio_UI.View_DB_Items_tab import create_prompt_view_tab, \
-    create_view_all_with_versions_tab, create_viewing_tab
-#
-# Gradio UI Imports
-from App_Function_Libraries.Gradio_UI.Evaluations_Benchmarks_tab import create_geval_tab, create_infinite_bench_tab
-#from App_Function_Libraries.Local_LLM.Local_LLM_huggingface import create_huggingface_tab
-from App_Function_Libraries.Local_LLM.Local_LLM_ollama import create_ollama_tab
-from App_Function_Libraries.Gradio_UI.RAG_QA_Chat_Notes import create_rag_qa_chat_notes_tab
-
-#
-#######################################################################################################################
-# Function Definitions
-#
-
-
-# Disable Gradio Analytics
-os.environ['GRADIO_ANALYTICS_ENABLED'] = 'False'
-
-
-custom_prompt_input = None
-server_mode = False
-share_public = False
-custom_prompt_summarize_bulleted_notes = ("""
-                    <s>You are a bulleted notes specialist. [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
-                        **Bulleted Note Creation Guidelines**
-
-                        **Headings**:
-                        - Based on referenced topics, not categories like quotes or terms
-                        - Surrounded by **bold** formatting 
-                        - Not listed as bullet points
-                        - No space between headings and list items underneath
-
-                        **Emphasis**:
-                        - **Important terms** set in bold font
-                        - **Text ending in a colon**: also bolded
-
-                        **Review**:
-                        - Ensure adherence to specified format
-                        - Do not reference these instructions in your response.</s>[INST] {{ .Prompt }} [/INST]
-                    """)
-#
-# End of globals
-#######################################################################################################################
-#
-# Start of Video/Audio Transcription and Summarization Functions
-#
-# Functions:
-# FIXME
-#
-#
-################################################################################################################
-# Functions for Re-Summarization
-#
-# Functions:
-# FIXME
-# End of Re-Summarization Functions
-#
-############################################################################################################################################################################################################################
-#
-# Explain/Summarize This Tab
-#
-# Functions:
-# FIXME
-#
-#
-############################################################################################################################################################################################################################
-#
-# Transcript Comparison Tab
-#
-# Functions:
-# FIXME
-#
-#
-###########################################################################################################################################################################################################################
-#
-# Search Tab
-#
-# Functions:
-# FIXME
-#
-# End of Search Tab Functions
-#
-##############################################################################################################################################################################################################################
-#
-# Llamafile Tab
-#
-# Functions:
-# FIXME
-#
-# End of Llamafile Tab Functions
-##############################################################################################################################################################################################################################
-#
-# Chat Interface Tab Functions
-#
-# Functions:
-# FIXME
-#
-#
-# End of Chat Interface Tab Functions
-################################################################################################################################################################################################################################
-#
-# Media Edit Tab Functions
-# Functions:
-# Fixme
-# create_media_edit_tab():
-##### Trash Tab
-# FIXME
-# Functions:
-#
-# End of Media Edit Tab Functions
-################################################################################################################
-#
-# Import Items Tab Functions
-#
-# Functions:
-#FIXME
-# End of Import Items Tab Functions
-################################################################################################################
-#
-# Export Items Tab Functions
-#
-# Functions:
-# FIXME
-#
-#
-# End of Export Items Tab Functions
-################################################################################################################
-#
-# Keyword Management Tab Functions
-#
-# Functions:
-#  create_view_keywords_tab():
-# FIXME
-#
-# End of Keyword Management Tab Functions
-################################################################################################################
-#
-# Document Editing Tab Functions
-#
-# Functions:
-#   #FIXME
-#
-#
-################################################################################################################
-#
-# Utilities Tab Functions
-# Functions:
-#   create_utilities_yt_video_tab():
-# #FIXME
-
-#
-# End of Utilities Tab Functions
-################################################################################################################
-
-# FIXME - Prompt sample box
-#
-# # Sample data
-# prompts_category_1 = [
-#     "What are the key points discussed in the video?",
-#     "Summarize the main arguments made by the speaker.",
-#     "Describe the conclusions of the study presented."
-# ]
-#
-# prompts_category_2 = [
-#     "How does the proposed solution address the problem?",
-#     "What are the implications of the findings?",
-#     "Can you explain the theory behind the observed phenomenon?"
-# ]
-#
-# all_prompts2 = prompts_category_1 + prompts_category_2
-
-
-def launch_ui(share_public=None, server_mode=False):
-    webbrowser.open_new_tab('http://127.0.0.1:7860/?__theme=dark')
-    share=share_public
-    css = """
-    .result-box {
-        margin-bottom: 20px;
-        border: 1px solid #ddd;
-        padding: 10px;
-    }
-    .result-box.error {
-        border-color: #ff0000;
-        background-color: #ffeeee;
-    }
-    .transcription, .summary {
-        max-height: 800px;
-        overflow-y: auto;
-        border: 1px solid #eee;
-        padding: 10px;
-        margin-top: 10px;
-    }
-    """
-
-    with gr.Blocks(theme='bethecloud/storj_theme',css=css) as iface:
-        gr.HTML(
-            """
-            <script>
-            document.addEventListener('DOMContentLoaded', (event) => {
-                document.body.classList.add('dark');
-                document.querySelector('gradio-app').style.backgroundColor = 'var(--color-background-primary)';
-            });
-            </script>
-            """
-        )
-        db_config = get_db_config()
-        db_type = db_config['type']
-        gr.Markdown(f"# tl/dw: Your LLM-powered Research Multi-tool")
-        gr.Markdown(f"(Using {db_type.capitalize()} Database)")
-        with gr.Tabs():
-            with gr.TabItem("Transcription / Summarization / Ingestion", id="ingestion-grouping"):
-                with gr.Tabs():
-                    create_video_transcription_tab()
-                    create_audio_processing_tab()
-                    create_podcast_tab()
-                    create_import_book_tab()
-                    create_plain_text_import_tab()
-                    create_website_scraping_tab()
-                    create_pdf_ingestion_tab()
-                    create_pdf_ingestion_test_tab()
-                    create_resummary_tab()
-                    create_summarize_explain_tab()
-                    create_live_recording_tab()
-                    create_arxiv_tab()
-
-            with gr.TabItem("Text Search", id="text search"):
-                create_search_tab()
-                create_search_summaries_tab()
-
-            with gr.TabItem("RAG Chat+Notes", id="RAG Chat Notes group"):
-                create_rag_qa_chat_notes_tab()
-
-            with gr.TabItem("RAG Search", id="RAG Search grou"):
-                create_rag_tab()
-                create_rag_qa_chat_tab()
-
-            with gr.TabItem("Chat with an LLM", id="LLM Chat group"):
-                create_chat_interface()
-                create_chat_interface_stacked()
-                create_chat_interface_multi_api()
-                create_chat_interface_four()
-                create_chat_with_llamafile_tab()
-                create_chat_management_tab()
-                chat_workflows_tab()
-
-
-#            with gr.TabItem("Character Chat", id="character chat group"):
-#                with gr.Tabs():
-#                   create_character_card_interaction_tab()
-#                   create_character_chat_mgmt_tab()
-#                   create_custom_character_card_tab()
-#                   create_character_card_validation_tab()
-#                   create_multiple_character_chat_tab()
-#                   create_narrator_controlled_conversation_tab()
-#                   create_export_characters_tab()
-
-
-            with gr.TabItem("View DB Items", id="view db items group"):
-                # This one works
-                create_view_all_with_versions_tab()
-                # This one is WIP
-                create_viewing_tab()
-                create_prompt_view_tab()
-
-
-            with gr.TabItem("Prompts", id='view prompts group'):
-                create_prompt_view_tab()
-                create_prompt_search_tab()
-                create_prompt_edit_tab()
-                create_prompt_clone_tab()
-                create_prompt_suggestion_tab()
-
-
-            with gr.TabItem("Manage / Edit Existing Items", id="manage group"):
-                create_media_edit_tab()
-                create_manage_items_tab()
-                create_media_edit_and_clone_tab()
-                # FIXME
-                #create_compare_transcripts_tab()
-
-
-            with gr.TabItem("Embeddings Management", id="embeddings group"):
-                create_embeddings_tab()
-                create_view_embeddings_tab()
-                create_purge_embeddings_tab()
-
-            with gr.TabItem("Writing Tools", id="writing_tools group"):
-                from App_Function_Libraries.Gradio_UI.Writing_tab import create_document_feedback_tab
-                create_document_feedback_tab()
-                from App_Function_Libraries.Gradio_UI.Writing_tab import create_grammar_style_check_tab
-                create_grammar_style_check_tab()
-                from App_Function_Libraries.Gradio_UI.Writing_tab import create_tone_adjustment_tab
-                create_tone_adjustment_tab()
-                from App_Function_Libraries.Gradio_UI.Writing_tab import create_creative_writing_tab
-                create_creative_writing_tab()
-                from App_Function_Libraries.Gradio_UI.Writing_tab import create_mikupad_tab
-                create_mikupad_tab()
-
-
-            with gr.TabItem("Keywords", id="keywords group"):
-                create_view_keywords_tab()
-                create_add_keyword_tab()
-                create_delete_keyword_tab()
-                create_export_keywords_tab()
-
-            with gr.TabItem("Import", id="import group"):
-                create_import_item_tab()
-                create_import_obsidian_vault_tab()
-                create_import_single_prompt_tab()
-                create_import_multiple_prompts_tab()
-                create_mediawiki_import_tab()
-                create_mediawiki_config_tab()
-
-            with gr.TabItem("Export", id="export group"):
-                create_export_tab()
-
-            with gr.TabItem("Backup Management", id="backup group"):
-                create_backup_tab()
-                create_view_backups_tab()
-                create_restore_backup_tab()
-
-            with gr.TabItem("Utilities", id="util group"):
-                create_utilities_yt_video_tab()
-                create_utilities_yt_audio_tab()
-                create_utilities_yt_timestamp_tab()
-
-            with gr.TabItem("Local LLM", id="local llm group"):
-                create_chat_with_llamafile_tab()
-                create_ollama_tab()
-                #create_huggingface_tab()
-
-            with gr.TabItem("Trashcan", id="trashcan group"):
-                create_search_and_mark_trash_tab()
-                create_view_trash_tab()
-                create_delete_trash_tab()
-                create_empty_trash_tab()
-
-            with gr.TabItem("Evaluations", id="eval"):
-                create_geval_tab()
-                create_infinite_bench_tab()
-                # FIXME
-                #create_mmlu_pro_tab()
-
-            with gr.TabItem("Introduction/Help", id="introduction group"):
-                create_introduction_tab()
-
-            with gr.TabItem("Config Editor", id="config group"):
-                create_config_editor_tab()
-
-    # Launch the interface
-    server_port_variable = 7860
-    os.environ['GRADIO_ANALYTICS_ENABLED'] = 'False'
-    if share==True:
-        iface.launch(share=True)
-    elif server_mode and not share_public:
-        iface.launch(share=False, server_name="0.0.0.0", server_port=server_port_variable, )
-    else:
-        try:
-            iface.launch(share=False, server_name="0.0.0.0", server_port=server_port_variable, )
-        except Exception as e:
-            logging.error(f"Error launching interface: {str(e)}")
+# Gradio_Related.py
+#########################################
+# Gradio UI Functions Library
+# I fucking hate Gradio.
+#
+#########################################
+#
+# Built-In Imports
+import logging
+import os
+import webbrowser
+
+#
+# Import 3rd-Party Libraries
+import gradio as gr
+#
+# Local Imports
+from App_Function_Libraries.DB.DB_Manager import get_db_config
+from App_Function_Libraries.Gradio_UI.Arxiv_tab import create_arxiv_tab
+from App_Function_Libraries.Gradio_UI.Audio_ingestion_tab import create_audio_processing_tab
+from App_Function_Libraries.Gradio_UI.Book_Ingestion_tab import create_import_book_tab
+from App_Function_Libraries.Gradio_UI.Character_Chat_tab import create_character_card_interaction_tab, create_character_chat_mgmt_tab, create_custom_character_card_tab, \
+    create_character_card_validation_tab, create_export_characters_tab
+from App_Function_Libraries.Gradio_UI.Character_interaction_tab import create_narrator_controlled_conversation_tab, \
+    create_multiple_character_chat_tab
+from App_Function_Libraries.Gradio_UI.Chat_ui import create_chat_management_tab, \
+    create_chat_interface_four, create_chat_interface_multi_api, create_chat_interface_stacked, create_chat_interface
+from App_Function_Libraries.Gradio_UI.Config_tab import create_config_editor_tab
+from App_Function_Libraries.Gradio_UI.Explain_summarize_tab import create_summarize_explain_tab
+from App_Function_Libraries.Gradio_UI.Export_Functionality import create_export_tab
+from App_Function_Libraries.Gradio_UI.Backup_Functionality import create_backup_tab, create_view_backups_tab, \
+    create_restore_backup_tab
+from App_Function_Libraries.Gradio_UI.Import_Functionality import create_import_single_prompt_tab, \
+    create_import_obsidian_vault_tab, create_import_item_tab, create_import_multiple_prompts_tab
+from App_Function_Libraries.Gradio_UI.Introduction_tab import create_introduction_tab
+from App_Function_Libraries.Gradio_UI.Keywords import create_view_keywords_tab, create_add_keyword_tab, \
+    create_delete_keyword_tab, create_export_keywords_tab
+from App_Function_Libraries.Gradio_UI.Live_Recording import create_live_recording_tab
+from App_Function_Libraries.Gradio_UI.Llamafile_tab import create_chat_with_llamafile_tab
+#from App_Function_Libraries.Gradio_UI.MMLU_Pro_tab import create_mmlu_pro_tab
+from App_Function_Libraries.Gradio_UI.Media_edit import create_prompt_clone_tab, create_prompt_edit_tab, \
+    create_media_edit_and_clone_tab, create_media_edit_tab
+from App_Function_Libraries.Gradio_UI.Media_wiki_tab import create_mediawiki_import_tab, create_mediawiki_config_tab
+from App_Function_Libraries.Gradio_UI.PDF_ingestion_tab import create_pdf_ingestion_tab, create_pdf_ingestion_test_tab
+from App_Function_Libraries.Gradio_UI.Plaintext_tab_import import create_plain_text_import_tab
+from App_Function_Libraries.Gradio_UI.Podcast_tab import create_podcast_tab
+from App_Function_Libraries.Gradio_UI.Prompt_Suggestion_tab import create_prompt_suggestion_tab
+from App_Function_Libraries.Gradio_UI.RAG_QA_Chat_tab import create_rag_qa_chat_tab
+from App_Function_Libraries.Gradio_UI.Re_summarize_tab import create_resummary_tab
+from App_Function_Libraries.Gradio_UI.Search_Tab import create_prompt_search_tab, \
+    create_search_summaries_tab, create_search_tab
+from App_Function_Libraries.Gradio_UI.RAG_Chat_tab import create_rag_tab
+from App_Function_Libraries.Gradio_UI.Embeddings_tab import create_embeddings_tab, create_view_embeddings_tab, \
+    create_purge_embeddings_tab
+from App_Function_Libraries.Gradio_UI.Trash import create_view_trash_tab, create_empty_trash_tab, \
+    create_delete_trash_tab, create_search_and_mark_trash_tab
+from App_Function_Libraries.Gradio_UI.Utilities import create_utilities_yt_timestamp_tab, create_utilities_yt_audio_tab, \
+    create_utilities_yt_video_tab
+from App_Function_Libraries.Gradio_UI.Video_transcription_tab import create_video_transcription_tab
+from App_Function_Libraries.Gradio_UI.View_tab import create_manage_items_tab
+from App_Function_Libraries.Gradio_UI.Website_scraping_tab import create_website_scraping_tab
+from App_Function_Libraries.Gradio_UI.Chat_Workflows import chat_workflows_tab
+from App_Function_Libraries.Gradio_UI.View_DB_Items_tab import create_prompt_view_tab, \
+    create_view_all_with_versions_tab, create_viewing_tab
+#
+# Gradio UI Imports
+from App_Function_Libraries.Gradio_UI.Evaluations_Benchmarks_tab import create_geval_tab, create_infinite_bench_tab
+#from App_Function_Libraries.Local_LLM.Local_LLM_huggingface import create_huggingface_tab
+from App_Function_Libraries.Local_LLM.Local_LLM_ollama import create_ollama_tab
+from App_Function_Libraries.Gradio_UI.RAG_QA_Chat_Notes import create_rag_qa_chat_notes_tab
+
+#
+#######################################################################################################################
+# Function Definitions
+#
+
+
+# Disable Gradio Analytics
+os.environ['GRADIO_ANALYTICS_ENABLED'] = 'False'
+
+
+custom_prompt_input = None
+server_mode = False
+share_public = False
+custom_prompt_summarize_bulleted_notes = ("""
+                    <s>You are a bulleted notes specialist. [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
+                        **Bulleted Note Creation Guidelines**
+
+                        **Headings**:
+                        - Based on referenced topics, not categories like quotes or terms
+                        - Surrounded by **bold** formatting 
+                        - Not listed as bullet points
+                        - No space between headings and list items underneath
+
+                        **Emphasis**:
+                        - **Important terms** set in bold font
+                        - **Text ending in a colon**: also bolded
+
+                        **Review**:
+                        - Ensure adherence to specified format
+                        - Do not reference these instructions in your response.</s>[INST] {{ .Prompt }} [/INST]
+                    """)
+#
+# End of globals
+#######################################################################################################################
+#
+# Start of Video/Audio Transcription and Summarization Functions
+#
+# Functions:
+# FIXME
+#
+#
+################################################################################################################
+# Functions for Re-Summarization
+#
+# Functions:
+# FIXME
+# End of Re-Summarization Functions
+#
+############################################################################################################################################################################################################################
+#
+# Explain/Summarize This Tab
+#
+# Functions:
+# FIXME
+#
+#
+############################################################################################################################################################################################################################
+#
+# Transcript Comparison Tab
+#
+# Functions:
+# FIXME
+#
+#
+###########################################################################################################################################################################################################################
+#
+# Search Tab
+#
+# Functions:
+# FIXME
+#
+# End of Search Tab Functions
+#
+##############################################################################################################################################################################################################################
+#
+# Llamafile Tab
+#
+# Functions:
+# FIXME
+#
+# End of Llamafile Tab Functions
+##############################################################################################################################################################################################################################
+#
+# Chat Interface Tab Functions
+#
+# Functions:
+# FIXME
+#
+#
+# End of Chat Interface Tab Functions
+################################################################################################################################################################################################################################
+#
+# Media Edit Tab Functions
+# Functions:
+# Fixme
+# create_media_edit_tab():
+##### Trash Tab
+# FIXME
+# Functions:
+#
+# End of Media Edit Tab Functions
+################################################################################################################
+#
+# Import Items Tab Functions
+#
+# Functions:
+#FIXME
+# End of Import Items Tab Functions
+################################################################################################################
+#
+# Export Items Tab Functions
+#
+# Functions:
+# FIXME
+#
+#
+# End of Export Items Tab Functions
+################################################################################################################
+#
+# Keyword Management Tab Functions
+#
+# Functions:
+#  create_view_keywords_tab():
+# FIXME
+#
+# End of Keyword Management Tab Functions
+################################################################################################################
+#
+# Document Editing Tab Functions
+#
+# Functions:
+#   #FIXME
+#
+#
+################################################################################################################
+#
+# Utilities Tab Functions
+# Functions:
+#   create_utilities_yt_video_tab():
+# #FIXME
+
+#
+# End of Utilities Tab Functions
+################################################################################################################
+
+# FIXME - Prompt sample box
+#
+# # Sample data
+# prompts_category_1 = [
+#     "What are the key points discussed in the video?",
+#     "Summarize the main arguments made by the speaker.",
+#     "Describe the conclusions of the study presented."
+# ]
+#
+# prompts_category_2 = [
+#     "How does the proposed solution address the problem?",
+#     "What are the implications of the findings?",
+#     "Can you explain the theory behind the observed phenomenon?"
+# ]
+#
+# all_prompts2 = prompts_category_1 + prompts_category_2
+
+
+def launch_ui(share_public=None, server_mode=False):
+    webbrowser.open_new_tab('http://127.0.0.1:7860/?__theme=dark')
+    share=share_public
+    css = """
+    .result-box {
+        margin-bottom: 20px;
+        border: 1px solid #ddd;
+        padding: 10px;
+    }
+    .result-box.error {
+        border-color: #ff0000;
+        background-color: #ffeeee;
+    }
+    .transcription, .summary {
+        max-height: 800px;
+        overflow-y: auto;
+        border: 1px solid #eee;
+        padding: 10px;
+        margin-top: 10px;
+    }
+    """
+
+    with gr.Blocks(theme='bethecloud/storj_theme',css=css) as iface:
+        gr.HTML(
+            """
+            <script>
+            document.addEventListener('DOMContentLoaded', (event) => {
+                document.body.classList.add('dark');
+                document.querySelector('gradio-app').style.backgroundColor = 'var(--color-background-primary)';
+            });
+            </script>
+            """
+        )
+        db_config = get_db_config()
+        db_type = db_config['type']
+        gr.Markdown(f"# tl/dw: Your LLM-powered Research Multi-tool")
+        gr.Markdown(f"(Using {db_type.capitalize()} Database)")
+        with gr.Tabs():
+            with gr.TabItem("Transcription / Summarization / Ingestion", id="ingestion-grouping"):
+                with gr.Tabs():
+                    create_video_transcription_tab()
+                    create_audio_processing_tab()
+                    create_podcast_tab()
+                    create_import_book_tab()
+                    create_plain_text_import_tab()
+                    create_website_scraping_tab()
+                    create_pdf_ingestion_tab()
+                    create_pdf_ingestion_test_tab()
+                    create_resummary_tab()
+                    create_summarize_explain_tab()
+                    create_live_recording_tab()
+                    create_arxiv_tab()
+
+            with gr.TabItem("Text Search", id="text search"):
+                create_search_tab()
+                create_search_summaries_tab()
+
+            with gr.TabItem("RAG Chat+Notes", id="RAG Chat Notes group"):
+                create_rag_qa_chat_notes_tab()
+
+            with gr.TabItem("RAG Search", id="RAG Search grou"):
+                create_rag_tab()
+                create_rag_qa_chat_tab()
+
+            with gr.TabItem("Chat with an LLM", id="LLM Chat group"):
+                create_chat_interface()
+                create_chat_interface_stacked()
+                create_chat_interface_multi_api()
+                create_chat_interface_four()
+                create_chat_with_llamafile_tab()
+                create_chat_management_tab()
+                chat_workflows_tab()
+
+
+            with gr.TabItem("Character Chat", id="character chat group"):
+                with gr.Tabs():
+                    create_character_card_interaction_tab()
+                    create_character_chat_mgmt_tab()
+                    create_custom_character_card_tab()
+                    create_character_card_validation_tab()
+                    create_multiple_character_chat_tab()
+                    create_narrator_controlled_conversation_tab()
+                    create_export_characters_tab()
+
+
+            with gr.TabItem("View DB Items", id="view db items group"):
+                # This one works
+                create_view_all_with_versions_tab()
+                # This one is WIP
+                create_viewing_tab()
+                create_prompt_view_tab()
+
+
+            with gr.TabItem("Prompts", id='view prompts group'):
+                create_prompt_view_tab()
+                create_prompt_search_tab()
+                create_prompt_edit_tab()
+                create_prompt_clone_tab()
+                create_prompt_suggestion_tab()
+
+
+            with gr.TabItem("Manage / Edit Existing Items", id="manage group"):
+                create_media_edit_tab()
+                create_manage_items_tab()
+                create_media_edit_and_clone_tab()
+                # FIXME
+                #create_compare_transcripts_tab()
+
+
+            with gr.TabItem("Embeddings Management", id="embeddings group"):
+                create_embeddings_tab()
+                create_view_embeddings_tab()
+                create_purge_embeddings_tab()
+
+            with gr.TabItem("Writing Tools", id="writing_tools group"):
+                from App_Function_Libraries.Gradio_UI.Writing_tab import create_document_feedback_tab
+                create_document_feedback_tab()
+                from App_Function_Libraries.Gradio_UI.Writing_tab import create_grammar_style_check_tab
+                create_grammar_style_check_tab()
+                from App_Function_Libraries.Gradio_UI.Writing_tab import create_tone_adjustment_tab
+                create_tone_adjustment_tab()
+                from App_Function_Libraries.Gradio_UI.Writing_tab import create_creative_writing_tab
+                create_creative_writing_tab()
+                from App_Function_Libraries.Gradio_UI.Writing_tab import create_mikupad_tab
+                create_mikupad_tab()
+
+
+            with gr.TabItem("Keywords", id="keywords group"):
+                create_view_keywords_tab()
+                create_add_keyword_tab()
+                create_delete_keyword_tab()
+                create_export_keywords_tab()
+
+            with gr.TabItem("Import", id="import group"):
+                create_import_item_tab()
+                create_import_obsidian_vault_tab()
+                create_import_single_prompt_tab()
+                create_import_multiple_prompts_tab()
+                create_mediawiki_import_tab()
+                create_mediawiki_config_tab()
+
+            with gr.TabItem("Export", id="export group"):
+                create_export_tab()
+
+            with gr.TabItem("Backup Management", id="backup group"):
+                create_backup_tab()
+                create_view_backups_tab()
+                create_restore_backup_tab()
+
+            with gr.TabItem("Utilities", id="util group"):
+                create_utilities_yt_video_tab()
+                create_utilities_yt_audio_tab()
+                create_utilities_yt_timestamp_tab()
+
+            with gr.TabItem("Local LLM", id="local llm group"):
+                create_chat_with_llamafile_tab()
+                create_ollama_tab()
+                #create_huggingface_tab()
+
+            with gr.TabItem("Trashcan", id="trashcan group"):
+                create_search_and_mark_trash_tab()
+                create_view_trash_tab()
+                create_delete_trash_tab()
+                create_empty_trash_tab()
+
+            with gr.TabItem("Evaluations", id="eval"):
+                create_geval_tab()
+                create_infinite_bench_tab()
+                # FIXME
+                #create_mmlu_pro_tab()
+
+            with gr.TabItem("Introduction/Help", id="introduction group"):
+                create_introduction_tab()
+
+            with gr.TabItem("Config Editor", id="config group"):
+                create_config_editor_tab()
+
+    # Launch the interface
+    server_port_variable = 7860
+    os.environ['GRADIO_ANALYTICS_ENABLED'] = 'False'
+    if share==True:
+        iface.launch(share=True)
+    elif server_mode and not share_public:
+        iface.launch(share=False, server_name="0.0.0.0", server_port=server_port_variable, )
+    else:
+        try:
+            iface.launch(share=False, server_name="0.0.0.0", server_port=server_port_variable, )
+        except Exception as e:
+            logging.error(f"Error launching interface: {str(e)}")

App_Function_Libraries/Gradio_UI/Character_Chat_tab.py

@@ -12,6 +12,7 @@ import logging
 import io
 import base64
 from typing import Dict, Any, Optional, List, Tuple, Union, cast
+import zipfile
 #
 # External Imports
 from PIL import Image
@@ -186,6 +187,67 @@ def parse_v1_card(card_data: Dict[str, Any]) -> Dict[str, Any]:
 # End of Character card import functions
 ####################################################
 
+####################################################
+#
+# Character card export functions
+
+def export_character_as_json(character_id):
+    character = get_character_card_by_id(character_id)
+    if character:
+        # Remove the 'id' field from the character data
+        character_data = {k: v for k, v in character.items() if k != 'id'}
+
+        # Convert image to base64 if it exists
+        if 'image' in character_data and character_data['image']:
+            image_data = base64.b64decode(character_data['image'])
+            img = Image.open(io.BytesIO(image_data))
+            buffered = io.BytesIO()
+            img.save(buffered, format="PNG")
+            character_data['image'] = base64.b64encode(buffered.getvalue()).decode('utf-8')
+
+        json_data = json.dumps(character_data, indent=2)
+        return json_data
+    return None
+
+def export_all_characters_as_zip():
+    characters = get_character_cards()
+    with tempfile.NamedTemporaryFile(mode='wb', delete=False, suffix='.zip') as temp_zip:
+        with zipfile.ZipFile(temp_zip, 'w') as zf:
+            for character in characters:
+                character_data = {k: v for k, v in character.items() if k != 'id'}
+
+                # Convert image to base64 if it exists
+                if 'image' in character_data and character_data['image']:
+                    image_data = base64.b64decode(character_data['image'])
+                    img = Image.open(io.BytesIO(image_data))
+                    buffered = io.BytesIO()
+                    img.save(buffered, format="PNG")
+                    character_data['image'] = base64.b64encode(buffered.getvalue()).decode('utf-8')
+                json_data = json.dumps(character_data, indent=2)
+                zf.writestr(f"{character['name']}.json", json_data)
+        return temp_zip.name
+
+def export_single_character(character_selection):
+    if not character_selection:
+        return None, "No character selected."
+
+    character_id = int(character_selection.split('(ID: ')[1].rstrip(')'))
+    json_data = export_character_as_json(character_id)
+
+    if json_data:
+        with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.json', encoding='utf-8') as temp_file:
+            temp_file.write(json_data)
+            return temp_file.name, f"Character '{character_selection.split(' (ID:')[0]}' exported successfully."
+    else:
+        return None, f"Failed to export character '{character_selection.split(' (ID:')[0]}'."
+
+def export_all_characters():
+    zip_path = export_all_characters_as_zip()
+    return zip_path, "All characters exported successfully."
+
+#
+# End of Character card export functions
+####################################################
 
 ####################################################
 #
@@ -1721,457 +1783,65 @@ def create_character_card_validation_tab():
             inputs=[file_upload],
             outputs=[validation_output]
         )
-# v2-not-working-on-export-def create_character_card_validation_tab():
-#     with gr.TabItem("Validate and Edit Character Card"):
-#         gr.Markdown("# Validate and Edit Character Card (v2)")
-#         gr.Markdown("Upload a character card (PNG, WEBP, or JSON) to validate and modify it.")
-#
-#         with gr.Row():
-#             with gr.Column():
-#                 # File uploader
-#                 file_upload = gr.File(
-#                     label="Upload Character Card (PNG, WEBP, JSON)",
-#                     file_types=[".png", ".webp", ".json"]
-#                 )
-#                 # Validation button
-#                 validate_button = gr.Button("Validate and Load Character Card")
-#                 # Output area for validation results
-#                 validation_output = gr.Markdown("")
-#
-#         # Input fields for character card data (duplicated from the create tab)
-#         with gr.Row():
-#             with gr.Column():
-#                 name_input = gr.Textbox(label="Name", placeholder="Enter character name")
-#                 description_input = gr.TextArea(label="Description", placeholder="Enter character description")
-#                 personality_input = gr.TextArea(label="Personality", placeholder="Enter character personality")
-#                 scenario_input = gr.TextArea(label="Scenario", placeholder="Enter character scenario")
-#                 first_mes_input = gr.TextArea(label="First Message", placeholder="Enter the first message")
-#                 mes_example_input = gr.TextArea(label="Example Messages", placeholder="Enter example messages")
-#                 creator_notes_input = gr.TextArea(label="Creator Notes", placeholder="Enter notes for the creator")
-#                 system_prompt_input = gr.TextArea(label="System Prompt", placeholder="Enter system prompt")
-#                 post_history_instructions_input = gr.TextArea(label="Post History Instructions", placeholder="Enter post history instructions")
-#                 alternate_greetings_input = gr.TextArea(
-#                     label="Alternate Greetings (one per line)",
-#                     placeholder="Enter alternate greetings, one per line"
-#                 )
-#                 tags_input = gr.Textbox(label="Tags", placeholder="Enter tags, separated by commas")
-#                 creator_input = gr.Textbox(label="Creator", placeholder="Enter creator name")
-#                 character_version_input = gr.Textbox(label="Character Version", placeholder="Enter character version")
-#                 extensions_input = gr.TextArea(
-#                     label="Extensions (JSON)",
-#                     placeholder="Enter extensions as JSON (optional)"
-#                 )
-#                 image_input = gr.Image(label="Character Image", type="pil")
-#
-#                 # Buttons
-#                 save_button = gr.Button("Save Character Card")
-#                 download_button = gr.Button("Download Character Card")
-#                 download_image_button = gr.Button("Download Character Card as Image")
-#
-#                 # Output status and outputs
-#                 save_status = gr.Markdown("")
-#                 download_output = gr.File(label="Download Character Card", interactive=False)
-#                 download_image_output = gr.File(label="Download Character Card as Image", interactive=False)
-#
-#         # Callback Functions
-#         def extract_json_from_image(file):
-#             try:
-#                 image = Image.open(file.name)
-#                 if "chara" in image.info:
-#                     json_data = image.info["chara"]
-#                     # Decode base64 if necessary
-#                     try:
-#                         json_data = base64.b64decode(json_data).decode('utf-8')
-#                     except Exception:
-#                         pass  # Assume it's already in plain text
-#                     return json_data
-#                 else:
-#                     return None
-#             except Exception as e:
-#                 logging.error(f"Error extracting JSON from image: {e}")
-#                 return None
-#
-#         def validate_v2_card(card_data):
-#             """
-#             Validate a character card according to the V2 specification.
-#
-#             Args:
-#                 card_data (dict): The parsed character card data.
-#
-#             Returns:
-#                 Tuple[bool, List[str]]: A tuple containing a boolean indicating validity and a list of validation messages.
-#             """
-#             validation_messages = []
-#
-#             # Check top-level fields
-#             if 'spec' not in card_data:
-#                 validation_messages.append("Missing 'spec' field.")
-#             elif card_data['spec'] != 'chara_card_v2':
-#                 validation_messages.append(f"Invalid 'spec' value: {card_data['spec']}. Expected 'chara_card_v2'.")
-#
-#             if 'spec_version' not in card_data:
-#                 validation_messages.append("Missing 'spec_version' field.")
-#             else:
-#                 # Ensure 'spec_version' is '2.0' or higher
-#                 try:
-#                     spec_version = float(card_data['spec_version'])
-#                     if spec_version < 2.0:
-#                         validation_messages.append(
-#                             f"'spec_version' must be '2.0' or higher. Found '{card_data['spec_version']}'.")
-#                 except ValueError:
-#                     validation_messages.append(
-#                         f"Invalid 'spec_version' format: {card_data['spec_version']}. Must be a number as a string.")
-#
-#             if 'data' not in card_data:
-#                 validation_messages.append("Missing 'data' field.")
-#                 return False, validation_messages  # Cannot proceed without 'data' field
-#
-#             data = card_data['data']
-#
-#             # Required fields in 'data'
-#             required_fields = ['name', 'description', 'personality', 'scenario', 'first_mes', 'mes_example']
-#             for field in required_fields:
-#                 if field not in data:
-#                     validation_messages.append(f"Missing required field in 'data': '{field}'.")
-#                 elif not isinstance(data[field], str):
-#                     validation_messages.append(f"Field '{field}' must be a string.")
-#                 elif not data[field].strip():
-#                     validation_messages.append(f"Field '{field}' cannot be empty.")
-#
-#             # Optional fields with expected types
-#             optional_fields = {
-#                 'creator_notes': str,
-#                 'system_prompt': str,
-#                 'post_history_instructions': str,
-#                 'alternate_greetings': list,
-#                 'tags': list,
-#                 'creator': str,
-#                 'character_version': str,
-#                 'extensions': dict,
-#                 'character_book': dict  # If present, should be a dict
-#             }
-#
-#             for field, expected_type in optional_fields.items():
-#                 if field in data:
-#                     if not isinstance(data[field], expected_type):
-#                         validation_messages.append(f"Field '{field}' must be of type '{expected_type.__name__}'.")
-#                     elif field == 'extensions':
-#                         # Validate that extensions keys are properly namespaced
-#                         for key in data[field].keys():
-#                             if '/' not in key and '_' not in key:
-#                                 validation_messages.append(
-#                                     f"Extension key '{key}' in 'extensions' should be namespaced to prevent conflicts.")
-#
-#             # If 'alternate_greetings' is present, check that it's a list of non-empty strings
-#             if 'alternate_greetings' in data and isinstance(data['alternate_greetings'], list):
-#                 for idx, greeting in enumerate(data['alternate_greetings']):
-#                     if not isinstance(greeting, str) or not greeting.strip():
-#                         validation_messages.append(
-#                             f"Element {idx} in 'alternate_greetings' must be a non-empty string.")
-#
-#             # If 'tags' is present, check that it's a list of non-empty strings
-#             if 'tags' in data and isinstance(data['tags'], list):
-#                 for idx, tag in enumerate(data['tags']):
-#                     if not isinstance(tag, str) or not tag.strip():
-#                         validation_messages.append(f"Element {idx} in 'tags' must be a non-empty string.")
-#
-#             # Validate 'extensions' field
-#             if 'extensions' in data and not isinstance(data['extensions'], dict):
-#                 validation_messages.append("Field 'extensions' must be a dictionary.")
-#
-#             # Validate 'character_book' if present
-#             # (Assuming you have a validate_character_book function)
-#             # if 'character_book' in data:
-#             #     is_valid_book, book_messages = validate_character_book(data['character_book'])
-#             #     if not is_valid_book:
-#             #         validation_messages.extend(book_messages)
-#
-#             is_valid = len(validation_messages) == 0
-#             return is_valid, validation_messages
-#
-#         # Include the save_character_card, download_character_card, and download_character_card_as_image functions
-#         def save_character_card(
-#                 name, description, personality, scenario, first_mes, mes_example,
-#                 creator_notes, system_prompt, post_history_instructions,
-#                 alternate_greetings_str, tags_str, creator, character_version,
-#                 extensions_str, image
-#         ):
-#             # Build the character card
-#             character_card = build_character_card(
-#                 name, description, personality, scenario, first_mes, mes_example,
-#                 creator_notes, system_prompt, post_history_instructions,
-#                 alternate_greetings_str, tags_str, creator, character_version,
-#                 extensions_str
-#             )
-#
-#             # Validate the character card
-#             is_valid, validation_messages = validate_v2_card(character_card)
-#             if not is_valid:
-#                 # Return validation errors
-#                 validation_output = "Character card validation failed:\n"
-#                 validation_output += "\n".join(validation_messages)
-#                 return validation_output
-#
-#             # If image is provided, encode it to base64
-#             if image:
-#                 img_byte_arr = io.BytesIO()
-#                 image.save(img_byte_arr, format='PNG')
-#                 character_card['data']['image'] = base64.b64encode(img_byte_arr.getvalue()).decode('utf-8')
-#
-#             # Save character card to database
-#             character_id = add_character_card(character_card['data'])
-#             if character_id:
-#                 return f"Character card '{name}' saved successfully."
-#             else:
-#                 return f"Failed to save character card '{name}'. It may already exist."
-#
-#         def download_character_card(
-#                 name, description, personality, scenario, first_mes, mes_example,
-#                 creator_notes, system_prompt, post_history_instructions,
-#                 alternate_greetings_str, tags_str, creator, character_version,
-#                 extensions_str, image
-#         ):
-#             # Build the character card
-#             character_card = build_character_card(
-#                 name, description, personality, scenario, first_mes, mes_example,
-#                 creator_notes, system_prompt, post_history_instructions,
-#                 alternate_greetings_str, tags_str, creator, character_version,
-#                 extensions_str
-#             )
-#
-#             # Validate the character card
-#             is_valid, validation_messages = validate_v2_card(character_card)
-#             if not is_valid:
-#                 # Return validation errors
-#                 validation_output = "Character card validation failed:\n"
-#                 validation_output += "\n".join(validation_messages)
-#                 return gr.update(value=None), validation_output  # Return None for the file output
-#
-#             # If image is provided, include it as base64
-#             if image:
-#                 img_byte_arr = io.BytesIO()
-#                 image.save(img_byte_arr, format='PNG')
-#                 character_card['data']['image'] = base64.b64encode(img_byte_arr.getvalue()).decode('utf-8')
-#
-#             # Convert to JSON string
-#             json_str = json.dumps(character_card, indent=2)
-#
-#             # Write the JSON to a temporary file
-#             with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.json', encoding='utf-8') as temp_file:
-#                 temp_file.write(json_str)
-#                 temp_file_path = temp_file.name
-#
-#             # Return the file path and clear validation output
-#             return temp_file_path, ""
-#
-#         def download_character_card_as_image(
-#                 name, description, personality, scenario, first_mes, mes_example,
-#                 creator_notes, system_prompt, post_history_instructions,
-#                 alternate_greetings_str, tags_str, creator, character_version,
-#                 extensions_str, image
-#         ):
-#             # Build the character card
-#             character_card = build_character_card(
-#                 name, description, personality, scenario, first_mes, mes_example,
-#                 creator_notes, system_prompt, post_history_instructions,
-#                 alternate_greetings_str, tags_str, creator, character_version,
-#                 extensions_str
-#             )
-#
-#             # Validate the character card
-#             is_valid, validation_messages = validate_v2_card(character_card)
-#             if not is_valid:
-#                 # Return validation errors
-#                 validation_output = "Character card validation failed:\n"
-#                 validation_output += "\n".join(validation_messages)
-#                 return gr.update(value=None), validation_output  # Return None for the file output
-#
-#             # Convert the character card JSON to a string
-#             json_str = json.dumps(character_card, indent=2)
-#
-#             # Encode the JSON string to base64
-#             chara_content = base64.b64encode(json_str.encode('utf-8')).decode('utf-8')
-#
-#             # Create PNGInfo object to hold metadata
-#             png_info = PngInfo()
-#             png_info.add_text('chara', chara_content)
-#
-#             # If image is provided, use it; otherwise, create a blank image
-#             if image:
-#                 img = image.copy()
-#             else:
-#                 # Create a default blank image
-#                 img = Image.new('RGB', (512, 512), color='white')
-#
-#             # Save the image to a temporary file with metadata
-#             with tempfile.NamedTemporaryFile(mode='wb', delete=False, suffix='.png') as temp_file:
-#                 img.save(temp_file, format='PNG', pnginfo=png_info)
-#                 temp_file_path = temp_file.name
-#
-#             # Return the file path and clear validation output
-#             return temp_file_path, ""
-#
-#         def build_character_card(
-#                 name, description, personality, scenario, first_mes, mes_example,
-#                 creator_notes, system_prompt, post_history_instructions,
-#                 alternate_greetings_str, tags_str, creator, character_version,
-#                 extensions_str
-#         ):
-#             # Parse alternate_greetings from multiline string
-#             alternate_greetings = [line.strip() for line in alternate_greetings_str.strip().split('\n') if line.strip()]
-#
-#             # Parse tags from comma-separated string
-#             tags = [tag.strip() for tag in tags_str.strip().split(',') if tag.strip()]
-#
-#             # Parse extensions from JSON string
-#             try:
-#                 extensions = json.loads(extensions_str) if extensions_str.strip() else {}
-#             except json.JSONDecodeError as e:
-#                 extensions = {}
-#                 logging.error(f"Error parsing extensions JSON: {e}")
-#
-#             # Build the character card dictionary according to V2 spec
-#             character_card = {
-#                 'spec': 'chara_card_v2',
-#                 'spec_version': '2.0',
-#                 'data': {
-#                     'name': name,
-#                     'description': description,
-#                     'personality': personality,
-#                     'scenario': scenario,
-#                     'first_mes': first_mes,
-#                     'mes_example': mes_example,
-#                     'creator_notes': creator_notes,
-#                     'system_prompt': system_prompt,
-#                     'post_history_instructions': post_history_instructions,
-#                     'alternate_greetings': alternate_greetings,
-#                     'tags': tags,
-#                     'creator': creator,
-#                     'character_version': character_version,
-#                     'extensions': extensions,
-#                 }
-#             }
-#             return character_card
-#
-#         def validate_and_load_character_card(file):
-#             if file is None:
-#                 return ["No file provided for validation."] + [gr.update() for _ in range(15)]
-#
-#             try:
-#                 if file.name.lower().endswith(('.png', '.webp')):
-#                     json_data = extract_json_from_image(file)
-#                     if not json_data:
-#                         return ["Failed to extract JSON data from the image."] + [gr.update() for _ in range(15)]
-#                 elif file.name.lower().endswith('.json'):
-#                     with open(file.name, 'r', encoding='utf-8') as f:
-#                         json_data = f.read()
-#                 else:
-#                     return ["Unsupported file type."] + [gr.update() for _ in range(15)]
-#
-#                 # Parse the JSON content
-#                 try:
-#                     card_data = json.loads(json_data)
-#                 except json.JSONDecodeError as e:
-#                     return [f"JSON decoding error: {e}"] + [gr.update() for _ in range(15)]
-#
-#                 # Validate the character card
-#                 is_valid, validation_messages = validate_v2_card(card_data)
-#
-#                 # Prepare the validation output
-#                 if is_valid:
-#                     validation_output_msg = "Character card is valid according to the V2 specification."
-#                 else:
-#                     validation_output_msg = "Character card validation failed:\n" + "\n".join(validation_messages)
-#
-#                 # Extract data to populate input fields
-#                 data = card_data.get('data', {})
-#
-#                 # Handle image data
-#                 if 'image' in data:
-#                     # Decode base64 image
-#                     image_data = base64.b64decode(data['image'])
-#                     image = Image.open(io.BytesIO(image_data))
-#                 else:
-#                     image = None
-#
-#                 # Prepare values for input fields
-#                 alternate_greetings_str = "\n".join(data.get('alternate_greetings', []))
-#                 tags_str = ", ".join(data.get('tags', []))
-#                 extensions_str = json.dumps(data.get('extensions', {}), indent=2) if data.get('extensions', {}) else ""
-#
-#                 outputs = [
-#                     validation_output_msg,
-#                     data.get('name', ''),
-#                     data.get('description', ''),
-#                     data.get('personality', ''),
-#                     data.get('scenario', ''),
-#                     data.get('first_mes', ''),
-#                     data.get('mes_example', ''),
-#                     data.get('creator_notes', ''),
-#                     data.get('system_prompt', ''),
-#                     data.get('post_history_instructions', ''),
-#                     alternate_greetings_str,
-#                     tags_str,
-#                     data.get('creator', ''),
-#                     data.get('character_version', ''),
-#                     extensions_str,
-#                     image
-#                 ]
-#
-#                 return outputs
-#
-#             except Exception as e:
-#                 logging.error(f"Error validating character card: {e}")
-#                 return [f"An unexpected error occurred: {e}"] + [gr.update() for _ in range(15)]
-#
-#         # Button Callback for validation
-#         validate_button.click(
-#             fn=validate_and_load_character_card,
-#             inputs=[file_upload],
-#             outputs=[
-#                 validation_output,
-#                 name_input, description_input, personality_input, scenario_input,
-#                 first_mes_input, mes_example_input, creator_notes_input, system_prompt_input,
-#                 post_history_instructions_input, alternate_greetings_input, tags_input,
-#                 creator_input, character_version_input, extensions_input, image_input
-#             ]
-#         )
-#
-#         # Button Callbacks for save, download, etc.
-#         save_button.click(
-#             fn=save_character_card,
-#             inputs=[
-#                 name_input, description_input, personality_input, scenario_input,
-#                 first_mes_input, mes_example_input, creator_notes_input, system_prompt_input,
-#                 post_history_instructions_input, alternate_greetings_input, tags_input,
-#                 creator_input, character_version_input, extensions_input, image_input
-#             ],
-#             outputs=[save_status]
-#         )
-#
-#         download_button.click(
-#             fn=download_character_card,
-#             inputs=[
-#                 name_input, description_input, personality_input, scenario_input,
-#                 first_mes_input, mes_example_input, creator_notes_input, system_prompt_input,
-#                 post_history_instructions_input, alternate_greetings_input, tags_input,
-#                 creator_input, character_version_input, extensions_input, image_input
-#             ],
-#             outputs=[download_output, save_status]
-#         )
-#
-#         download_image_button.click(
-#             fn=download_character_card_as_image,
-#             inputs=[
-#                 name_input, description_input, personality_input, scenario_input,
-#                 first_mes_input, mes_example_input, creator_notes_input, system_prompt_input,
-#                 post_history_instructions_input, alternate_greetings_input, tags_input,
-#                 creator_input, character_version_input, extensions_input, image_input
-#             ],
-#             outputs=[download_image_output, save_status]
-#         )
 
 
+def create_export_characters_tab():
+    with gr.TabItem("Export Characters"):
+        gr.Markdown("# Export Characters")
+        gr.Markdown("Export character cards individually as JSON files or all together as a ZIP file.")
+
+        with gr.Row():
+            with gr.Column(scale=1):
+                # Dropdown to select a character for individual export
+                characters = get_character_cards()
+                character_choices = [f"{char['name']} (ID: {char['id']})" for char in characters]
+                export_character_dropdown = gr.Dropdown(
+                    label="Select Character to Export",
+                    choices=character_choices
+                )
+                load_characters_button = gr.Button("Load Existing Characters")
+                export_single_button = gr.Button("Export Selected Character")
+                export_all_button = gr.Button("Export All Characters")
+
+            with gr.Column(scale=1):
+                # Output components
+                export_output = gr.File(label="Exported Character(s)", interactive=False)
+                export_status = gr.Markdown("")
+
+        def export_single_character_wrapper(character_selection):
+            file_path, status_message = export_single_character(character_selection)
+            if file_path:
+                return gr.File.update(value=file_path), status_message
+            else:
+                return gr.File.update(value=None), status_message
+
+        def export_all_characters_wrapper():
+            zip_path = export_all_characters_as_zip()
+            characters = get_character_cards()
+            exported_characters = [char['name'] for char in characters]
+            status_message = f"Exported {len(exported_characters)} characters successfully:\n" + "\n".join(exported_characters)
+            return gr.File.update(value=zip_path), status_message
+
+        # Event listeners
+        load_characters_button.click(
+            fn=lambda: gr.update(choices=[f"{char['name']} (ID: {char['id']})" for char in get_character_cards()]),
+            outputs=export_character_dropdown
+        )
+
+        export_single_button.click(
+            fn=export_single_character_wrapper,
+            inputs=[export_character_dropdown],
+            outputs=[export_output, export_status]
+        )
+
+        export_all_button.click(
+            fn=export_all_characters_wrapper,
+            inputs=[],
+            outputs=[export_output, export_status]
+        )
+
+    return export_character_dropdown, load_characters_button, export_single_button, export_all_button, export_output, export_status
+
 #
 # End of Character_Chat_tab.py
 #######################################################################################################################

App_Function_Libraries/Gradio_UI/Live_Recording.py

@@ -0,0 +1,142 @@
+# Live_Recording.py
+# Description: Gradio UI for live audio recording and transcription.
+#
+# Import necessary modules and functions
+import logging
+import os
+import time
+
+# External Imports
+import gradio as gr
+# Local Imports
+from App_Function_Libraries.Audio.Audio_Transcription_Lib import (record_audio, speech_to_text, save_audio_temp,
+                                                                  stop_recording)
+from App_Function_Libraries.DB.DB_Manager import add_media_to_database
+from App_Function_Libraries.Metrics.metrics_logger import log_counter, log_histogram
+#
+#######################################################################################################################
+#
+# Functions:
+
+whisper_models = ["small", "medium", "small.en", "medium.en", "medium", "large", "large-v1", "large-v2", "large-v3",
+                  "distil-large-v2", "distil-medium.en", "distil-small.en"]
+
+def create_live_recording_tab():
+    with gr.Tab("Live Recording and Transcription"):
+        gr.Markdown("# Live Audio Recording and Transcription")
+        with gr.Row():
+            with gr.Column():
+                duration = gr.Slider(minimum=1, maximum=8000, value=15, label="Recording Duration (seconds)")
+                whisper_models_input = gr.Dropdown(choices=whisper_models, value="medium", label="Whisper Model")
+                vad_filter = gr.Checkbox(label="Use VAD Filter")
+                save_recording = gr.Checkbox(label="Save Recording")
+                save_to_db = gr.Checkbox(label="Save Transcription to Database(Must be checked to save - can be checked afer transcription)", value=False)
+                custom_title = gr.Textbox(label="Custom Title (for database)", visible=False)
+                record_button = gr.Button("Start Recording")
+                stop_button = gr.Button("Stop Recording")
+            with gr.Column():
+                output = gr.Textbox(label="Transcription", lines=10)
+                audio_output = gr.Audio(label="Recorded Audio", visible=False)
+
+        recording_state = gr.State(value=None)
+
+        def start_recording(duration):
+            log_counter("live_recording_start_attempt", labels={"duration": duration})
+            p, stream, audio_queue, stop_event, audio_thread = record_audio(duration)
+            log_counter("live_recording_start_success", labels={"duration": duration})
+            return (p, stream, audio_queue, stop_event, audio_thread)
+
+        def end_recording_and_transcribe(recording_state, whisper_model, vad_filter, save_recording, save_to_db, custom_title):
+            log_counter("live_recording_end_attempt", labels={"model": whisper_model})
+            start_time = time.time()
+
+            if recording_state is None:
+                log_counter("live_recording_end_error", labels={"error": "Recording hasn't started yet"})
+                return "Recording hasn't started yet.", None
+
+            p, stream, audio_queue, stop_event, audio_thread = recording_state
+            audio_data = stop_recording(p, stream, audio_queue, stop_event, audio_thread)
+
+            temp_file = save_audio_temp(audio_data)
+            segments = speech_to_text(temp_file, whisper_model=whisper_model, vad_filter=vad_filter)
+            transcription = "\n".join([segment["Text"] for segment in segments])
+
+            if save_recording:
+                log_counter("live_recording_saved", labels={"model": whisper_model})
+            else:
+                os.remove(temp_file)
+
+            end_time = time.time() - start_time
+            log_histogram("live_recording_end_duration", end_time, labels={"model": whisper_model})
+            log_counter("live_recording_end_success", labels={"model": whisper_model})
+            return transcription, temp_file if save_recording else None
+
+        def save_transcription_to_db(transcription, custom_title):
+            log_counter("save_transcription_to_db_attempt")
+            start_time = time.time()
+            if custom_title.strip() == "":
+                custom_title = "Self-recorded Audio"
+
+            try:
+                url = "self_recorded"
+                info_dict = {
+                    "title": custom_title,
+                    "uploader": "self-recorded",
+                    "webpage_url": url
+                }
+                segments = [{"Text": transcription}]
+                summary = ""
+                keywords = ["self-recorded", "audio"]
+                custom_prompt_input = ""
+                whisper_model = "self-recorded"
+                media_type = "audio"
+
+                result = add_media_to_database(
+                    url=url,
+                    info_dict=info_dict,
+                    segments=segments,
+                    summary=summary,
+                    keywords=keywords,
+                    custom_prompt_input=custom_prompt_input,
+                    whisper_model=whisper_model,
+                    media_type=media_type
+                )
+                end_time = time.time() - start_time
+                log_histogram("save_transcription_to_db_duration", end_time)
+                log_counter("save_transcription_to_db_success")
+                return f"Transcription saved to database successfully. {result}"
+            except Exception as e:
+                logging.error(f"Error saving transcription to database: {str(e)}")
+                log_counter("save_transcription_to_db_error", labels={"error": str(e)})
+                return f"Error saving transcription to database: {str(e)}"
+
+        def update_custom_title_visibility(save_to_db):
+            return gr.update(visible=save_to_db)
+
+        record_button.click(
+            fn=start_recording,
+            inputs=[duration],
+            outputs=[recording_state]
+        )
+
+        stop_button.click(
+            fn=end_recording_and_transcribe,
+            inputs=[recording_state, whisper_models_input, vad_filter, save_recording, save_to_db, custom_title],
+            outputs=[output, audio_output]
+        )
+
+        save_to_db.change(
+            fn=update_custom_title_visibility,
+            inputs=[save_to_db],
+            outputs=[custom_title]
+        )
+
+        gr.Button("Save to Database").click(
+            fn=save_transcription_to_db,
+            inputs=[output, custom_title],
+            outputs=gr.Textbox(label="Database Save Status")
+        )
+
+#
+# End of Functions
+########################################################################################################################

App_Function_Libraries/Gradio_UI/Llamafile_tab.py

@@ -1,122 +1,312 @@
 # Llamafile_tab.py
-# Description: Functions relating to the Llamafile tab
-#
+# Description: Gradio interface for configuring and launching Llamafile with Local LLMs
+
 # Imports
 import os
-import glob
-#
-# External Imports
+import logging
+from typing import Tuple, Optional
 import gradio as gr
-#
-# Local Imports
-from App_Function_Libraries.Local_LLM.Llamafile import start_llamafile
+
+
+from App_Function_Libraries.Local_LLM.Local_LLM_Inference_Engine_Lib import (
+    download_llm_model,
+    llm_models,
+    start_llamafile,
+    get_gguf_llamafile_files
+)
 #
 #######################################################################################################################
 #
 # Functions:
 
-
 def create_chat_with_llamafile_tab():
-    def get_model_files(directory):
-        pattern = os.path.join(directory, "*.{gguf,llamafile}")
-        return [os.path.basename(f) for f in glob.glob(pattern)]
-
-    def update_dropdowns():
-        current_dir_models = get_model_files(".")
-        parent_dir_models = get_model_files("..")
-        return (
-            {"choices": current_dir_models, "value": None},
-            {"choices": parent_dir_models, "value": None}
-        )
+    # Function to update model path based on selection
+    def on_local_model_change(selected_model: str, search_directory: str) -> str:
+        if selected_model and isinstance(search_directory, str):
+            model_path = os.path.abspath(os.path.join(search_directory, selected_model))
+            logging.debug(f"Selected model path: {model_path}")  # Debug print for selected model path
+            return model_path
+        return "Invalid selection or directory."
+
+    # Function to update the dropdown with available models
+    def update_dropdowns(search_directory: str) -> Tuple[dict, str]:
+        logging.debug(f"User-entered directory: {search_directory}")  # Debug print for directory
+        if not os.path.isdir(search_directory):
+            logging.debug(f"Directory does not exist: {search_directory}")  # Debug print for non-existing directory
+            return gr.update(choices=[], value=None), "Directory does not exist."
+
+        logging.debug(f"Directory exists: {search_directory}, scanning for files...")  # Confirm directory exists
+        model_files = get_gguf_llamafile_files(search_directory)
+
+        if not model_files:
+            logging.debug(f"No model files found in {search_directory}")  # Debug print for no files found
+            return gr.update(choices=[], value=None), "No model files found in the specified directory."
+
+        # Update the dropdown choices with the model files found
+        logging.debug(f"Models loaded from {search_directory}: {model_files}")  # Debug: Print model files loaded
+        return gr.update(choices=model_files, value=None), f"Models loaded from {search_directory}."
+
+
+
+    def download_preset_model(selected_model: str) -> Tuple[str, str]:
+        """
+        Downloads the selected preset model.
+
+        Args:
+            selected_model (str): The key of the selected preset model.
+
+        Returns:
+            Tuple[str, str]: Status message and the path to the downloaded model.
+        """
+        model_info = llm_models.get(selected_model)
+        if not model_info:
+            return "Invalid model selection.", ""
+
+        try:
+            model_path = download_llm_model(
+                model_name=model_info["name"],
+                model_url=model_info["url"],
+                model_filename=model_info["filename"],
+                model_hash=model_info["hash"]
+            )
+            return f"Model '{model_info['name']}' downloaded successfully.", model_path
+        except Exception as e:
+            logging.error(f"Error downloading model: {e}")
+            return f"Failed to download model: {e}", ""
 
     with gr.TabItem("Local LLM with Llamafile"):
         gr.Markdown("# Settings for Llamafile")
+
         with gr.Row():
             with gr.Column():
-                am_noob = gr.Checkbox(label="Check this to enable sane defaults", value=False, visible=True)
-                # FIXME - these get deleted at some point?
-                advanced_mode_toggle = gr.Checkbox(label="Advanced Mode - Enable to show all settings", value=False)
-
+                am_noob = gr.Checkbox(label="Enable Sane Defaults", value=False, visible=True)
+                advanced_mode_toggle = gr.Checkbox(label="Advanced Mode - Show All Settings", value=False)
+                # Advanced Inputs
+                verbose_checked = gr.Checkbox(label="Enable Verbose Output", value=False, visible=False)
+                threads_checked = gr.Checkbox(label="Set CPU Threads", value=False, visible=False)
+                threads_value = gr.Number(label="Number of CPU Threads", value=None, precision=0, visible=False)
+                threads_batched_checked = gr.Checkbox(label="Enable Batched Inference", value=False, visible=False)
+                threads_batched_value = gr.Number(label="Batch Size for Inference", value=None, precision=0, visible=False)
+                model_alias_checked = gr.Checkbox(label="Set Model Alias", value=False, visible=False)
+                model_alias_value = gr.Textbox(label="Model Alias", value="", visible=False)
+                ctx_size_checked = gr.Checkbox(label="Set Prompt Context Size", value=False, visible=False)
+                ctx_size_value = gr.Number(label="Prompt Context Size", value=8124, precision=0, visible=False)
+                ngl_checked = gr.Checkbox(label="Enable GPU Layers", value=False, visible=True)
+                ngl_value = gr.Number(label="Number of GPU Layers", value=None, precision=0, visible=True)
+                batch_size_checked = gr.Checkbox(label="Set Batch Size", value=False, visible=False)
+                batch_size_value = gr.Number(label="Batch Size", value=512, visible=False)
+                memory_f32_checked = gr.Checkbox(label="Use 32-bit Floating Point", value=False, visible=False)
+                numa_checked = gr.Checkbox(label="Enable NUMA", value=False, visible=False)
+                server_timeout_value = gr.Number(label="Server Timeout", value=600, precision=0, visible=False)
+                host_checked = gr.Checkbox(label="Set IP to Listen On", value=False, visible=False)
+                host_value = gr.Textbox(label="Host IP Address", value="", visible=False)
+                port_checked = gr.Checkbox(label="Set Server Port", value=False, visible=False)
+                port_value = gr.Number(label="Port Number", value=8080, precision=0, visible=False)
+                api_key_checked = gr.Checkbox(label="Set API Key", value=False, visible=False)
+                api_key_value = gr.Textbox(label="API Key", value="", visible=False)
+                http_threads_checked = gr.Checkbox(label="Set HTTP Server Threads", value=False, visible=False)
+                http_threads_value = gr.Number(label="Number of HTTP Server Threads", value=None, precision=0, visible=False)
+                hf_repo_checked = gr.Checkbox(label="Use Huggingface Repo Model", value=False, visible=False)
+                hf_repo_value = gr.Textbox(label="Huggingface Repo Name", value="", visible=False)
+                hf_file_checked = gr.Checkbox(label="Set Huggingface Model File", value=False, visible=False)
+                hf_file_value = gr.Textbox(label="Huggingface Model File", value="", visible=False)
 
             with gr.Column():
-                # FIXME - make this actually work
-                model_checked = gr.Checkbox(label="Enable Setting Local LLM Model Path", value=False, visible=True)
-                current_dir_dropdown = gr.Dropdown(
-                    label="Select Model from Current Directory (.)",
-                    choices=[],  # Start with an empty list
-                    visible=True
-                )
-                parent_dir_dropdown = gr.Dropdown(
-                    label="Select Model from Parent Directory (..)",
-                    choices=[],  # Start with an empty list
-                    visible=True
+                # Model Selection Section
+                gr.Markdown("## Model Selection")
+
+                # Option 1: Select from Local Filesystem
+                with gr.Row():
+                    search_directory = gr.Textbox(label="Model Directory",
+                                                  placeholder="Enter directory path(currently '.\Models')",
+                                                  value=".\Models",
+                                                  interactive=True)
+
+                # Initial population of local models
+                initial_dropdown_update, _ = update_dropdowns(".\Models")
+                refresh_button = gr.Button("Refresh Models")
+                local_model_dropdown = gr.Dropdown(label="Select Model from Directory", choices=[])
+                # Display selected model path
+                model_value = gr.Textbox(label="Selected Model File Path", value="", interactive=False)
+
+                # Option 2: Download Preset Models
+                gr.Markdown("## Download Preset Models")
+
+                preset_model_dropdown = gr.Dropdown(
+                    label="Select a Preset Model",
+                    choices=list(llm_models.keys()),
+                    value=None,
+                    interactive=True,
+                    info="Choose a preset model to download."
                 )
-                refresh_button = gr.Button("Refresh Model Lists")
-                model_value = gr.Textbox(label="Selected Model File", value="", visible=True)
+                download_preset_button = gr.Button("Download Selected Preset")
+
         with gr.Row():
-            with gr.Column():
-                ngl_checked = gr.Checkbox(label="Enable Setting GPU Layers", value=False, visible=True)
-                ngl_value = gr.Number(label="Number of GPU Layers", value=None, precision=0, visible=True)
-                advanced_inputs = create_llamafile_advanced_inputs()
             with gr.Column():
                 start_button = gr.Button("Start Llamafile")
                 stop_button = gr.Button("Stop Llamafile (doesn't work)")
                 output_display = gr.Markdown()
 
 
-        def update_model_value(current_dir_model, parent_dir_model):
-            if current_dir_model:
-                return current_dir_model
-            elif parent_dir_model:
-                return os.path.join("..", parent_dir_model)
-            else:
-                return ""
+        # Show/hide advanced inputs based on toggle
+        def update_visibility(show_advanced: bool):
+            components = [
+                verbose_checked, threads_checked, threads_value,
+                http_threads_checked, http_threads_value,
+                hf_repo_checked, hf_repo_value,
+                hf_file_checked, hf_file_value,
+                ctx_size_checked, ctx_size_value,
+                ngl_checked, ngl_value,
+                host_checked, host_value,
+                port_checked, port_value
+            ]
+            return [gr.update(visible=show_advanced) for _ in components]
 
-        current_dir_dropdown.change(
-            fn=update_model_value,
-            inputs=[current_dir_dropdown, parent_dir_dropdown],
-            outputs=model_value
-        )
-        parent_dir_dropdown.change(
-            fn=update_model_value,
-            inputs=[current_dir_dropdown, parent_dir_dropdown],
-            outputs=model_value
-        )
+        def on_start_button_click(
+                am_noob: bool,
+                verbose_checked: bool,
+                threads_checked: bool,
+                threads_value: Optional[int],
+                threads_batched_checked: bool,
+                threads_batched_value: Optional[int],
+                model_alias_checked: bool,
+                model_alias_value: str,
+                http_threads_checked: bool,
+                http_threads_value: Optional[int],
+                model_value: str,
+                hf_repo_checked: bool,
+                hf_repo_value: str,
+                hf_file_checked: bool,
+                hf_file_value: str,
+                ctx_size_checked: bool,
+                ctx_size_value: Optional[int],
+                ngl_checked: bool,
+                ngl_value: Optional[int],
+                batch_size_checked: bool,
+                batch_size_value: Optional[int],
+                memory_f32_checked: bool,
+                numa_checked: bool,
+                server_timeout_value: Optional[int],
+                host_checked: bool,
+                host_value: str,
+                port_checked: bool,
+                port_value: Optional[int],
+                api_key_checked: bool,
+                api_key_value: str
+        ) -> str:
+            """
+            Event handler for the Start Llamafile button.
+            """
+            try:
+                result = start_llamafile(
+                    am_noob,
+                    verbose_checked,
+                    threads_checked,
+                    threads_value,
+                    threads_batched_checked,
+                    threads_batched_value,
+                    model_alias_checked,
+                    model_alias_value,
+                    http_threads_checked,
+                    http_threads_value,
+                    model_value,
+                    hf_repo_checked,
+                    hf_repo_value,
+                    hf_file_checked,
+                    hf_file_value,
+                    ctx_size_checked,
+                    ctx_size_value,
+                    ngl_checked,
+                    ngl_value,
+                    batch_size_checked,
+                    batch_size_value,
+                    memory_f32_checked,
+                    numa_checked,
+                    server_timeout_value,
+                    host_checked,
+                    host_value,
+                    port_checked,
+                    port_value,
+                    api_key_checked,
+                    api_key_value
+                )
+                return result
+            except Exception as e:
+                logging.error(f"Error starting Llamafile: {e}")
+                return f"Failed to start Llamafile: {e}"
 
-        refresh_button.click(
-            fn=update_dropdowns,
-            inputs=[],
-            outputs=[current_dir_dropdown, parent_dir_dropdown]
+        advanced_mode_toggle.change(
+            fn=update_visibility,
+            inputs=[advanced_mode_toggle],
+            outputs=[
+                verbose_checked, threads_checked, threads_value,
+                http_threads_checked, http_threads_value,
+                hf_repo_checked, hf_repo_value,
+                hf_file_checked, hf_file_value,
+                ctx_size_checked, ctx_size_value,
+                ngl_checked, ngl_value,
+                host_checked, host_value,
+                port_checked, port_value
+            ]
         )
 
         start_button.click(
-            fn=start_llamafile,
-            inputs=[am_noob, model_checked, model_value, ngl_checked, ngl_value] + advanced_inputs,
+            fn=on_start_button_click,
+            inputs=[
+                am_noob,
+                verbose_checked,
+                threads_checked,
+                threads_value,
+                threads_batched_checked,
+                threads_batched_value,
+                model_alias_checked,
+                model_alias_value,
+                http_threads_checked,
+                http_threads_value,
+                model_value,
+                hf_repo_checked,
+                hf_repo_value,
+                hf_file_checked,
+                hf_file_value,
+                ctx_size_checked,
+                ctx_size_value,
+                ngl_checked,
+                ngl_value,
+                batch_size_checked,
+                batch_size_value,
+                memory_f32_checked,
+                numa_checked,
+                server_timeout_value,
+                host_checked,
+                host_value,
+                port_checked,
+                port_value,
+                api_key_checked,
+                api_key_value
+            ],
             outputs=output_display
         )
 
+        download_preset_button.click(
+            fn=download_preset_model,
+            inputs=[preset_model_dropdown],
+            outputs=[output_display, model_value]
+        )
+
+        # Click event for refreshing models
+        refresh_button.click(
+            fn=update_dropdowns,
+            inputs=[search_directory],  # Ensure that the directory path (string) is passed
+            outputs=[local_model_dropdown, output_display]  # Update dropdown and status
+        )
 
-def create_llamafile_advanced_inputs():
-    verbose_checked = gr.Checkbox(label="Enable Verbose Output", value=False, visible=False)
-    threads_checked = gr.Checkbox(label="Set CPU Threads", value=False, visible=False)
-    threads_value = gr.Number(label="Number of CPU Threads", value=None, precision=0, visible=False)
-    http_threads_checked = gr.Checkbox(label="Set HTTP Server Threads", value=False, visible=False)
-    http_threads_value = gr.Number(label="Number of HTTP Server Threads", value=None, precision=0, visible=False)
-    hf_repo_checked = gr.Checkbox(label="Use Huggingface Repo Model", value=False, visible=False)
-    hf_repo_value = gr.Textbox(label="Huggingface Repo Name", value="", visible=False)
-    hf_file_checked = gr.Checkbox(label="Set Huggingface Model File", value=False, visible=False)
-    hf_file_value = gr.Textbox(label="Huggingface Model File", value="", visible=False)
-    ctx_size_checked = gr.Checkbox(label="Set Prompt Context Size", value=False, visible=False)
-    ctx_size_value = gr.Number(label="Prompt Context Size", value=8124, precision=0, visible=False)
-    host_checked = gr.Checkbox(label="Set IP to Listen On", value=False, visible=False)
-    host_value = gr.Textbox(label="Host IP Address", value="", visible=False)
-    port_checked = gr.Checkbox(label="Set Server Port", value=False, visible=False)
-    port_value = gr.Number(label="Port Number", value=None, precision=0, visible=False)
-
-    return [verbose_checked, threads_checked, threads_value, http_threads_checked, http_threads_value,
-            hf_repo_checked, hf_repo_value, hf_file_checked, hf_file_value, ctx_size_checked, ctx_size_value,
-            host_checked, host_value, port_checked, port_value]
+        # Event to update model_value when a model is selected from the dropdown
+        local_model_dropdown.change(
+            fn=on_local_model_change,  # Function that calculates the model path
+            inputs=[local_model_dropdown, search_directory],  # Inputs: selected model and directory
+            outputs=[model_value]  # Output: Update the model_value textbox with the selected model path
+        )
 
 #
-# End of Llamafile_tab.py
-#########################################################################################################################
+#
+#######################################################################################################################

App_Function_Libraries/Gradio_UI/MMLU_Pro_tab.py

@@ -0,0 +1,115 @@
+# MMLU_Pro_tab.py
+# is a library that contains the Gradio UI code for the MMLU-Pro benchmarking tool.
+#
+##############################################################################################################
+# Imports
+import os
+
+import gradio as gr
+import logging
+#
+# External Imports
+from tqdm import tqdm
+# Local Imports
+from App_Function_Libraries.Benchmarks_Evaluations.MMLU_Pro.MMLU_Pro_rewritten import (
+    load_mmlu_pro, run_mmlu_pro_benchmark, mmlu_pro_main, load_mmlu_pro_config
+)
+#
+##############################################################################################################
+#
+# Functions:
+
+# Set up logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logger = logging.getLogger(__name__)
+
+
+def get_categories():
+    """Fetch categories using the dataset loader from MMLU_Pro_rewritten.py"""
+    try:
+        test_data, _ = load_mmlu_pro()  # Use the function from MMLU_Pro_rewritten.py
+        return list(test_data.keys())  # Return the categories from the test dataset
+    except Exception as e:
+        logger.error(f"Failed to load categories: {e}")
+        return ["Error loading categories"]
+
+
+def load_categories():
+    """Helper function to return the categories for the Gradio dropdown."""
+    categories = get_categories()  # Fetch categories from the dataset
+    if categories:
+        return gr.update(choices=categories, value=categories[0])  # Update dropdown with categories
+    else:
+        return gr.update(choices=["Error loading categories"], value="Error loading categories")
+
+
+def run_benchmark_from_ui(url, api_key, model, timeout, category, parallel, verbosity, log_prompt):
+    """Function to run the benchmark with parameters from the UI."""
+
+    # Override config with UI parameters
+    config = load_mmlu_pro_config(
+        url=url,
+        api_key=api_key,
+        model=model,
+        timeout=timeout,
+        categories=[category] if category else None,
+        parallel=parallel,
+        verbosity=verbosity,
+        log_prompt=log_prompt
+    )
+
+    # Run the benchmarking process
+    try:
+        # Call the main benchmarking function
+        mmlu_pro_main()
+
+        # Assume the final report is generated in "eval_results" folder
+        report_path = os.path.join("eval_results", config["server"]["model"].replace("/", "-"), "final_report.txt")
+
+        # Read the final report
+        with open(report_path, "r") as f:
+            report = f.read()
+
+        return report
+    except Exception as e:
+        logger.error(f"An error occurred during benchmark execution: {e}")
+        return f"An error occurred during benchmark execution. Please check the logs for more information. Error: {str(e)}"
+
+
+def create_mmlu_pro_tab():
+    """Create the Gradio UI tab for MMLU-Pro Benchmark."""
+    with gr.Tab("MMLU-Pro Benchmark"):
+        gr.Markdown("## Run MMLU-Pro Benchmark")
+
+        with gr.Row():
+            with gr.Column():
+                # Inputs for the benchmark
+                url = gr.Textbox(label="Server URL")
+                api_key = gr.Textbox(label="API Key", type="password")
+                model = gr.Textbox(label="Model Name")
+                timeout = gr.Number(label="Timeout (seconds)", value=30)
+                category = gr.Dropdown(label="Category", choices=["Load categories..."])
+                load_categories_btn = gr.Button("Load Categories")
+                parallel = gr.Slider(label="Parallel Requests", minimum=1, maximum=10, step=1, value=1)
+                verbosity = gr.Slider(label="Verbosity Level", minimum=0, maximum=2, step=1, value=1)
+                log_prompt = gr.Checkbox(label="Log Prompt")
+
+            with gr.Column():
+                # Run button and output display
+                run_button = gr.Button("Run Benchmark")
+                output = gr.Textbox(label="Benchmark Results", lines=20)
+
+        # When "Load Categories" is clicked, load the categories into the dropdown
+        load_categories_btn.click(
+            load_categories,
+            outputs=category
+        )
+
+        # When "Run Benchmark" is clicked, trigger the run_benchmark_from_ui function
+        run_button.click(
+            run_benchmark_from_ui,  # Use the function defined to run the benchmark
+            inputs=[url, api_key, model, timeout, category, parallel, verbosity, log_prompt],
+            outputs=output
+        )
+
+    return [url, api_key, model, timeout, category, parallel, verbosity, log_prompt, run_button, output]

App_Function_Libraries/Gradio_UI/RAG_QA_Chat_Notes.py

@@ -0,0 +1,243 @@
+# RAG_QA_Chat_Notes.py
+# Description: This file contains the code for the RAG QA Chat Notes tab in the RAG QA Chat application.
+#
+# Imports
+import logging
+# External Imports
+import gradio as gr
+#
+# Local Imports
+from App_Function_Libraries.DB.RAG_QA_Chat_DB import save_message, add_keywords_to_conversation, \
+    search_conversations_by_keywords, load_chat_history, save_notes, get_notes, clear_notes, \
+    add_keywords_to_note, execute_query, start_new_conversation
+from App_Function_Libraries.RAG.RAG_QA_Chat import rag_qa_chat
+#
+####################################################################################################
+#
+# Functions
+def create_rag_qa_chat_notes_tab():
+    with gr.TabItem("RAG QA Chat"):
+        gr.Markdown("# RAG QA Chat")
+
+        state = gr.State({
+            "conversation_id": None,
+            "page": 1,
+            "context_source": "Entire Media Database",
+        })
+
+        with gr.Row():
+            with gr.Column(scale=1):
+                context_source = gr.Radio(
+                    ["Entire Media Database", "Search Database", "Upload File"],
+                    label="Context Source",
+                    value="Entire Media Database"
+                )
+                existing_file = gr.Dropdown(label="Select Existing File", choices=[], interactive=True)
+                file_page = gr.State(value=1)
+                with gr.Row():
+                    page_number = gr.Number(value=1, label="Page", precision=0)
+                    page_size = gr.Number(value=20, label="Items per page", precision=0)
+                    total_pages = gr.Number(label="Total Pages", interactive=False)
+                with gr.Row():
+                    prev_page_btn = gr.Button("Previous Page")
+                    next_page_btn = gr.Button("Next Page")
+                    page_info = gr.HTML("Page 1")
+
+                search_query = gr.Textbox(label="Search Query", visible=False)
+                search_button = gr.Button("Search", visible=False)
+                search_results = gr.Dropdown(label="Search Results", choices=[], visible=False)
+                file_upload = gr.File(
+                    label="Upload File",
+                    visible=False,
+                    file_types=["txt", "pdf", "epub", "md", "rtf", "json", "csv"]
+                )
+                convert_to_text = gr.Checkbox(label="Convert to plain text", visible=False)
+                keywords = gr.Textbox(label="Keywords (comma-separated)", visible=False)
+            with gr.Column(scale=1):
+                api_choice = gr.Dropdown(
+                    choices=["Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral", "OpenRouter",
+                             "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace"],
+                    label="Select API for RAG",
+                    value="OpenAI"
+                )
+                use_query_rewriting = gr.Checkbox(label="Use Query Rewriting", value=True)
+
+                # FIXME - add load conversations button
+                load_conversation = gr.Dropdown(label="Load Conversation", choices=[])
+                new_conversation = gr.Button("New Conversation")
+                conversation_title = gr.Textbox(label="Conversation Title",
+                                                placeholder="Enter a title for the new conversation")
+
+        with gr.Row():
+            with gr.Column(scale=2):
+                chatbot = gr.Chatbot(height=500)
+                msg = gr.Textbox(label="Enter your message")
+                submit = gr.Button("Submit")
+                clear_chat = gr.Button("Clear Chat History")
+
+            with gr.Column(scale=1):
+                notes = gr.TextArea(label="Notes", placeholder="Enter your notes here...", lines=20)
+                keywords_for_notes = gr.Textbox(label="Keywords for Notes (comma-separated)",
+                                                placeholder="Enter keywords for the note", visible=True)
+                save_notes_btn = gr.Button("Save Notes")  # Renamed to avoid conflict
+                clear_notes_btn = gr.Button("Clear Notes")  # Renamed to avoid conflict
+
+        loading_indicator = gr.HTML(visible=False)
+
+        def rag_qa_chat_wrapper(message, history, state, context_source, existing_file, search_results, file_upload,
+                                convert_to_text, keywords, api_choice, use_query_rewriting):
+            try:
+                conversation_id = state.value["conversation_id"]
+                if not conversation_id:
+                    conversation_id = start_new_conversation("Untitled Conversation")  # Provide a title or handle accordingly
+                    state = update_state(state, conversation_id=conversation_id)
+
+                save_message(conversation_id, 'human', message)
+
+                if keywords:
+                    add_keywords_to_conversation(conversation_id, [kw.strip() for kw in keywords.split(',')])
+
+                # Implement your actual RAG logic here
+                response = "response"#rag_qa_chat(message, conversation_id, context_source, existing_file, search_results,
+                                       #file_upload, convert_to_text, api_choice, use_query_rewriting)
+
+                save_message(conversation_id, 'ai', response)
+
+                new_history = history + [(message, response)]
+
+                logging.info(f"Successfully processed message for conversation '{conversation_id}'")
+                return new_history, "", gr.update(visible=False), state
+
+            except Exception as e:
+                logging.error(f"Error in rag_qa_chat_wrapper: {e}")
+                gr.Error("An unexpected error occurred. Please try again later.")
+                return history, "", gr.update(visible=False), state
+
+        def load_conversation_history(selected_conversation_id, page, page_size, state):
+            if selected_conversation_id:
+                history, total_pages_val, _ = load_chat_history(selected_conversation_id, page, page_size)
+                notes_content = get_notes(selected_conversation_id)  # Retrieve notes here
+                updated_state = update_state(state, conversation_id=selected_conversation_id, page=page)
+                return history, total_pages_val, updated_state, "\n".join(notes_content)
+            return [], 1, state, ""
+
+        def start_new_conversation_wrapper(title, state):
+            new_conversation_id = start_new_conversation(title if title else "Untitled Conversation")
+            return [], update_state(state, conversation_id=new_conversation_id, page=1)
+
+        def update_state(state, **kwargs):
+            new_state = state.value.copy()
+            new_state.update(kwargs)
+            return new_state
+
+        def update_page(direction, current_page, total_pages_val):
+            new_page = max(1, min(current_page + direction, total_pages_val))
+            return new_page
+
+        def update_context_source(choice):
+            return {
+                existing_file: gr.update(visible=choice == "Select Existing File"),
+                prev_page_btn: gr.update(visible=choice == "Search Database"),
+                next_page_btn: gr.update(visible=choice == "Search Database"),
+                page_info: gr.update(visible=choice == "Search Database"),
+                search_query: gr.update(visible=choice == "Search Database"),
+                search_button: gr.update(visible=choice == "Search Database"),
+                search_results: gr.update(visible=choice == "Search Database"),
+                file_upload: gr.update(visible=choice == "Upload File"),
+                convert_to_text: gr.update(visible=choice == "Upload File"),
+                keywords: gr.update(visible=choice == "Upload File")
+            }
+
+        def perform_search(query):
+            try:
+                results = search_conversations_by_keywords([kw.strip() for kw in query.split()])
+                return gr.update(choices=[f"{title} (ID: {id})" for id, title in results[0]])
+            except Exception as e:
+                logging.error(f"Error performing search: {e}")
+                gr.Error(f"Error performing search: {str(e)}")
+                return gr.update(choices=[])
+
+        def clear_chat_history():
+            return [], ""
+
+        def save_notes_function(notes_content, keywords_content):
+            """Save the notes and associated keywords to the database."""
+            conversation_id = state.value["conversation_id"]
+            if conversation_id and notes_content:
+                # Save the note
+                save_notes(conversation_id, notes_content)
+
+                # Get the last inserted note ID
+                query = "SELECT id FROM rag_qa_notes WHERE conversation_id = ? ORDER BY timestamp DESC LIMIT 1"
+                note_id = execute_query(query, (conversation_id,))[0][0]
+
+                if keywords_content:
+                    add_keywords_to_note(note_id, [kw.strip() for kw in keywords_content.split(',')])
+
+                logging.info("Notes and keywords saved successfully!")
+                return notes_content
+            else:
+                logging.warning("No conversation ID or notes to save.")
+                return ""
+
+        def clear_notes_function():
+            """Clear notes for the current conversation."""
+            conversation_id = state.value["conversation_id"]
+            if conversation_id:
+                clear_notes(conversation_id)
+                logging.info("Notes cleared successfully!")
+            return ""
+
+        # Event handlers
+        submit.click(
+            rag_qa_chat_wrapper,
+            inputs=[msg, chatbot, state, context_source, existing_file, search_results, file_upload,
+                    convert_to_text, keywords, api_choice, use_query_rewriting],
+            outputs=[chatbot, msg, loading_indicator, state]
+        )
+
+        load_conversation.change(
+            load_conversation_history,
+            inputs=[load_conversation, page_number, page_size, state],
+            outputs=[chatbot, total_pages, state, notes]
+        )
+
+        new_conversation.click(
+            start_new_conversation_wrapper,
+            inputs=[conversation_title, state],
+            outputs=[chatbot, state]
+        )
+
+        # Pagination Event handlers
+        prev_page_btn.click(
+            lambda current_page, total_pages_val: update_page(-1, current_page, total_pages_val),
+            inputs=[page_number, total_pages],
+            outputs=[page_number]
+        )
+
+        next_page_btn.click(
+            lambda current_page, total_pages_val: update_page(1, current_page, total_pages_val),
+            inputs=[page_number, total_pages],
+            outputs=[page_number]
+        )
+
+        context_source.change(update_context_source, inputs=[context_source],
+                              outputs=[existing_file, prev_page_btn, next_page_btn, page_info,
+                                       search_query, search_button, search_results,
+                                       file_upload, convert_to_text, keywords])
+
+        search_button.click(perform_search, inputs=[search_query], outputs=[search_results])
+
+        clear_chat.click(clear_chat_history, outputs=[chatbot, msg])
+
+        save_notes_btn.click(save_notes_function, inputs=[notes, keywords_for_notes], outputs=[notes])
+        clear_notes_btn.click(clear_notes_function, outputs=[notes])
+
+    return (context_source, existing_file, search_query, search_button, search_results, file_upload,
+            convert_to_text, keywords, api_choice, use_query_rewriting, chatbot, msg, submit, clear_chat,
+            notes, save_notes_btn, clear_notes_btn, load_conversation, new_conversation, conversation_title,
+            prev_page_btn, next_page_btn, page_number, page_size, total_pages)
+
+#
+# End of RAG_QA_Chat_Notes.py
+####################################################################################################

App_Function_Libraries/Gradio_UI/Utilities.py

@@ -10,7 +10,7 @@ from App_Function_Libraries.Utils.Utils import sanitize_filename, downloaded_fil
 
 
 def create_utilities_yt_video_tab():
-    with gr.Tab("YouTube Video Downloader"):
+    with gr.Tab("YouTube Video Downloader", id='youtube_dl'):
         with gr.Row():
             with gr.Column():
                 gr.Markdown(
@@ -28,7 +28,7 @@ def create_utilities_yt_video_tab():
         )
 
 def create_utilities_yt_audio_tab():
-    with gr.Tab("YouTube Audio Downloader"):
+    with gr.Tab("YouTube Audio Downloader", id="youtube audio downloader"):
         with gr.Row():
             with gr.Column():
                 gr.Markdown(
@@ -48,7 +48,7 @@ def create_utilities_yt_audio_tab():
         )
 
 def create_utilities_yt_timestamp_tab():
-    with gr.Tab("YouTube Timestamp URL Generator"):
+    with gr.Tab("YouTube Timestamp URL Generator", id="timestamp-gen"):
         gr.Markdown("## Generate YouTube URL with Timestamp")
         with gr.Row():
             with gr.Column():

App_Function_Libraries/Local_LLM/Local_LLM_Inference_Engine_Lib.py

@@ -0,0 +1,317 @@
+# Local_LLM_Inference_Engine_Lib.py
+#########################################
+# Local LLM Inference Engine Library
+# This library is used to handle downloading, configuring, and launching the Local LLM Inference Engine
+#   via (llama.cpp via llamafile)
+#
+#
+####
+####################
+# Function List
+#
+# 1.
+#
+####################
+# Import necessary libraries
+#import atexit
+import glob
+import logging
+import os
+import re
+import signal
+import subprocess
+import sys
+import time
+from typing import List, Optional
+#
+# Import 3rd-pary Libraries
+import requests
+#
+# Import Local
+from App_Function_Libraries.Web_Scraping.Article_Summarization_Lib import *
+from App_Function_Libraries.Utils.Utils import download_file
+#
+#######################################################################################################################
+# Function Definitions:
+
+
+###############################################################
+# LLM models information
+
+llm_models = {
+    "Mistral-7B-Instruct-v0.2-Q8.llamafile": {
+        "name": "Mistral-7B-Instruct-v0.2-Q8.llamafile",
+        "url": "https://huggingface.co/Mozilla/Mistral-7B-Instruct-v0.2-llamafile/resolve/main/mistral-7b-instruct-v0.2.Q8_0.llamafile?download=true",
+        "filename": "mistral-7b-instruct-v0.2.Q8_0.llamafile",
+        "hash": "1ee6114517d2f770425c880e5abc443da36b193c82abec8e2885dd7ce3b9bfa6"
+    },
+    "Samantha-Mistral-Instruct-7B-Bulleted-Notes-Q8.gguf": {
+        "name": "Samantha-Mistral-Instruct-7B-Bulleted-Notes-Q8.gguf",
+        "url": "https://huggingface.co/cognitivetech/samantha-mistral-instruct-7b-bulleted-notes-GGUF/resolve/main/samantha-mistral-instruct-7b-bulleted-notes.Q8_0.gguf?download=true",
+        "filename": "samantha-mistral-instruct-7b-bulleted-notes.Q8_0.gguf",
+        "hash": "6334c1ab56c565afd86535271fab52b03e67a5e31376946bce7bf5c144e847e4"
+    },
+    "Phi-3-mini-128k-instruct-Q8_0.gguf": {
+        "name": "Phi-3-mini-128k-instruct-Q8_0.gguf",
+        "url": "https://huggingface.co/gaianet/Phi-3-mini-128k-instruct-GGUF/resolve/main/Phi-3-mini-128k-instruct-Q8_0.gguf?download=true",
+        "filename": "Phi-3-mini-128k-instruct-Q8_0.gguf",
+        "hash": "6817b66d1c3c59ab06822e9732f0e594eea44e64cae2110906eac9d17f75d193"
+    },
+    "Meta-Llama-3-8B-Instruct.Q8_0.llamafile": {
+        "name": "Meta-Llama-3-8B-Instruct.Q8_0.llamafile",
+        "url": "https://huggingface.co/Mozilla/Meta-Llama-3-8B-Instruct-llamafile/resolve/main/Meta-Llama-3-8B-Instruct.Q8_0.llamafile?download=true",
+        "filename": "Meta-Llama-3-8B-Instruct.Q8_0.llamafile",
+        "hash": "406868a97f02f57183716c7e4441d427f223fdbc7fa42964ef10c4d60dd8ed37"
+    }
+}
+#
+###############################################################
+
+# Function to download the latest llamafile from the Mozilla-Ocho/llamafile repo
+def download_latest_llamafile(output_filename: str) -> str:
+    """
+    Downloads the latest llamafile binary from the Mozilla-Ocho/llamafile GitHub repository.
+    """
+    logging.info("Checking for and downloading Llamafile if it doesn't already exist...")
+    if os.path.exists(output_filename):
+        logging.debug(f"{output_filename} already exists. Skipping download.")
+        return os.path.abspath(output_filename)
+
+    repo = "Mozilla-Ocho/llamafile"
+    asset_name_prefix = "llamafile-"
+    latest_release_url = f"https://api.github.com/repos/{repo}/releases/latest"
+    response = requests.get(latest_release_url)
+    if response.status_code != 200:
+        raise Exception(f"Failed to fetch latest release info: {response.status_code}")
+
+    latest_release_data = response.json()
+    tag_name = latest_release_data['tag_name']
+
+    release_details_url = f"https://api.github.com/repos/{repo}/releases/tags/{tag_name}"
+    response = requests.get(release_details_url)
+    if response.status_code != 200:
+        raise Exception(f"Failed to fetch release details for tag {tag_name}: {response.status_code}")
+
+    release_data = response.json()
+    assets = release_data.get('assets', [])
+
+    asset_url = None
+    for asset in assets:
+        if re.match(f"{asset_name_prefix}.*", asset['name']):
+            asset_url = asset['browser_download_url']
+            break
+
+    if not asset_url:
+        raise Exception(f"No asset found with prefix {asset_name_prefix}")
+
+    logging.info("Downloading Llamafile...")
+    download_file(asset_url, output_filename)
+
+    logging.debug(f"Downloaded {output_filename} from {asset_url}")
+    return os.path.abspath(output_filename)
+
+def download_llm_model(model_name: str, model_url: str, model_filename: str, model_hash: str) -> str:
+    """
+    Downloads the specified LLM model if not already present.
+    """
+    logging.info(f"Checking availability of model: {model_name}")
+    if os.path.exists(model_filename):
+        logging.debug(f"Model '{model_name}' already exists. Skipping download.")
+        return os.path.abspath(model_filename)
+
+    logging.info(f"Downloading model: {model_name}")
+    download_file(model_url, model_filename, expected_checksum=model_hash)
+    logging.debug(f"Downloaded model '{model_name}' successfully.")
+    return os.path.abspath(model_filename)
+
+def launch_in_new_terminal(executable: str, args: List[str]) -> subprocess.Popen:
+    """
+    Launches the executable in a new terminal window based on the operating system.
+    Returns the subprocess.Popen object.
+    """
+    useros = os.name
+    if useros == "nt":
+        # For Windows
+        args_str = ' '.join(args)
+        command = f'start cmd /k "{executable} {args_str}"'
+    elif useros == "posix":
+        # For Linux (assuming GNOME Terminal; adjust if necessary)
+        args_str = ' '.join(args)
+        command = f'gnome-terminal -- bash -c "{executable} {args_str}; exec bash"'
+    else:
+        # For macOS
+        args_str = ' '.join(args)
+        command = f'open -a Terminal.app "{executable}" --args {args_str}'
+
+    try:
+        process = subprocess.Popen(command, shell=True)
+        logging.info(f"Launched {executable} with arguments: {args}")
+        return process
+    except Exception as e:
+        logging.error(f"Failed to launch the process: {e}")
+        raise
+
+# Function to scan the directory for .gguf and .llamafile files
+def get_gguf_llamafile_files(directory: str) -> List[str]:
+    """
+    Retrieves model files with extensions .gguf or .llamafile from the specified directory.
+    """
+    logging.debug(f"Scanning directory: {directory}")  # Debug print for directory
+
+    # Print all files in the directory for debugging
+    all_files = os.listdir(directory)
+    logging.debug(f"All files in directory: {all_files}")
+
+    pattern_gguf = os.path.join(directory, "*.gguf")
+    pattern_llamafile = os.path.join(directory, "*.llamafile")
+
+    gguf_files = glob.glob(pattern_gguf)
+    llamafile_files = glob.glob(pattern_llamafile)
+
+    # Debug: Print the files found
+    logging.debug(f"Found .gguf files: {gguf_files}")
+    logging.debug(f"Found .llamafile files: {llamafile_files}")
+
+    return [os.path.basename(f) for f in gguf_files + llamafile_files]
+
+
+# Initialize process with type annotation
+process: Optional[subprocess.Popen] = None
+# Function to close out llamafile process on script exit.
+def cleanup_process() -> None:
+    """
+    Terminates the external llamafile process if it is running.
+    """
+    global process
+    if process is not None:
+        process.kill()
+        logging.debug("Terminated the external process")
+        process = None  # Reset the process variable after killing
+
+def signal_handler(sig, frame):
+    """
+    Handles termination signals to ensure the subprocess is cleaned up.
+    """
+    logging.info('Signal handler called with signal: %s', sig)
+    cleanup_process()
+    sys.exit(0)
+
+# Register signal handlers
+def setup_signal_handlers():
+    signal.signal(signal.SIGINT, signal_handler)
+    signal.signal(signal.SIGTERM, signal_handler)
+
+setup_signal_handlers()
+
+def start_llamafile(
+    am_noob: bool,
+    verbose_checked: bool,
+    threads_checked: bool,
+    threads_value: Optional[int],
+    threads_batched_checked: bool,
+    threads_batched_value: Optional[int],
+    model_alias_checked: bool,
+    model_alias_value: str,
+    http_threads_checked: bool,
+    http_threads_value: Optional[int],
+    model_value: str,
+    hf_repo_checked: bool,
+    hf_repo_value: str,
+    hf_file_checked: bool,
+    hf_file_value: str,
+    ctx_size_checked: bool,
+    ctx_size_value: Optional[int],
+    ngl_checked: bool,
+    ngl_value: Optional[int],
+    batch_size_checked: bool,
+    batch_size_value: Optional[int],
+    memory_f32_checked: bool,
+    numa_checked: bool,
+    server_timeout_value: Optional[int],
+    host_checked: bool,
+    host_value: str,
+    port_checked: bool,
+    port_value: Optional[int],
+    api_key_checked: bool,
+    api_key_value: Optional[str],
+) -> str:
+    """
+    Starts the llamafile process based on provided configuration.
+    """
+    global process
+
+    # Construct command based on checked values
+    command = []
+    if am_noob:
+        # Define what 'am_noob' does, e.g., set default parameters
+        command.append('--sane-defaults')  # Replace with actual flag if needed
+
+    if verbose_checked:
+        command.append('-v')
+
+    if threads_checked and threads_value is not None:
+        command.extend(['-t', str(threads_value)])
+
+    if http_threads_checked and http_threads_value is not None:
+        command.extend(['--threads', str(http_threads_value)])
+
+    if threads_batched_checked and threads_batched_value is not None:
+        command.extend(['-tb', str(threads_batched_value)])
+
+    if model_alias_checked and model_alias_value:
+        command.extend(['-a', model_alias_value])
+
+    # Set model path
+    model_path = os.path.abspath(model_value)
+    command.extend(['-m', model_path])
+
+    if hf_repo_checked and hf_repo_value:
+        command.extend(['-hfr', hf_repo_value])
+
+    if hf_file_checked and hf_file_value:
+        command.extend(['-hff', hf_file_value])
+
+    if ctx_size_checked and ctx_size_value is not None:
+        command.extend(['-c', str(ctx_size_value)])
+
+    if ngl_checked and ngl_value is not None:
+        command.extend(['-ngl', str(ngl_value)])
+
+    if batch_size_checked and batch_size_value is not None:
+        command.extend(['-b', str(batch_size_value)])
+
+    if memory_f32_checked:
+        command.append('--memory-f32')
+
+    if numa_checked:
+        command.append('--numa')
+
+    if host_checked and host_value:
+        command.extend(['--host', host_value])
+
+    if port_checked and port_value is not None:
+        command.extend(['--port', str(port_value)])
+
+    if api_key_checked and api_key_value:
+        command.extend(['--api-key', api_key_value])
+
+    try:
+        useros = os.name
+        output_filename = "llamafile.exe" if useros == "nt" else "llamafile"
+
+        # Ensure llamafile is downloaded
+        llamafile_path = download_latest_llamafile(output_filename)
+
+        # Start llamafile process
+        process = launch_in_new_terminal(llamafile_path, command)
+
+        logging.info(f"Llamafile started with command: {' '.join(command)}")
+        return f"Command built and ran: {' '.join(command)} \n\nLlamafile started successfully."
+
+    except Exception as e:
+        logging.error(f"Failed to start llamafile: {e}")
+        return f"Failed to start llamafile: {e}"
+
+#
+# End of Local_LLM_Inference_Engine_Lib.py
+#######################################################################################################################

App_Function_Libraries/Local_LLM/Local_LLM_huggingface.py

@@ -0,0 +1,79 @@
+# import gradio as gr
+# from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
+# import os
+# import torch
+#
+# # Assuming models are stored in a 'models' directory
+# MODELS_DIR = "models"
+#
+#
+# def get_local_models():
+#     if not os.path.exists(MODELS_DIR):
+#         os.makedirs(MODELS_DIR)
+#     return [d for d in os.listdir(MODELS_DIR) if os.path.isdir(os.path.join(MODELS_DIR, d))]
+#
+#
+# def download_model(model_name):
+#     try:
+#         tokenizer = AutoTokenizer.from_pretrained(model_name)
+#         model = AutoModelForCausalLM.from_pretrained(model_name)
+#
+#         # Save the model and tokenizer
+#         save_path = os.path.join(MODELS_DIR, model_name.split('/')[-1])
+#         tokenizer.save_pretrained(save_path)
+#         model.save_pretrained(save_path)
+#
+#         return f"Successfully downloaded model: {model_name}"
+#     except Exception as e:
+#         return f"Failed to download model: {str(e)}"
+#
+#
+# def run_inference(model_name, prompt):
+#     try:
+#         model_path = os.path.join(MODELS_DIR, model_name)
+#         tokenizer = AutoTokenizer.from_pretrained(model_path)
+#         model = AutoModelForCausalLM.from_pretrained(model_path)
+#
+#         # Use GPU if available
+#         device = "cuda" if torch.cuda.is_available() else "cpu"
+#         model.to(device)
+#
+#         # Create a text-generation pipeline
+#         text_generator = pipeline("text-generation", model=model, tokenizer=tokenizer, device=device)
+#
+#         # Generate text
+#         result = text_generator(prompt, max_length=100, num_return_sequences=1)
+#
+#         return result[0]['generated_text']
+#     except Exception as e:
+#         return f"Error running inference: {str(e)}"
+#
+#
+# def create_huggingface_tab():
+#     with gr.Tab("Hugging Face Transformers"):
+#         gr.Markdown("# Hugging Face Transformers Model Management")
+#
+#         with gr.Row():
+#             model_list = gr.Dropdown(label="Available Models", choices=get_local_models())
+#             refresh_button = gr.Button("Refresh Model List")
+#
+#         with gr.Row():
+#             new_model_name = gr.Textbox(label="Model to Download (e.g., 'gpt2' or 'EleutherAI/gpt-neo-1.3B')")
+#             download_button = gr.Button("Download Model")
+#
+#         download_output = gr.Textbox(label="Download Status")
+#
+#         with gr.Row():
+#             run_model = gr.Dropdown(label="Model to Run", choices=get_local_models())
+#             prompt = gr.Textbox(label="Prompt")
+#             run_button = gr.Button("Run Inference")
+#
+#         run_output = gr.Textbox(label="Model Output")
+#
+#         def update_model_lists():
+#             models = get_local_models()
+#             return gr.update(choices=models), gr.update(choices=models)
+#
+#         refresh_button.click(update_model_lists, outputs=[model_list, run_model])
+#         download_button.click(download_model, inputs=[new_model_name], outputs=[download_output])
+#         run_button.click(run_inference, inputs=[run_model, prompt], outputs=[run_output])

App_Function_Libraries/Local_LLM/Local_LLM_ollama.py

@@ -0,0 +1,96 @@
+import platform
+
+import gradio as gr
+import subprocess
+import psutil
+import os
+import signal
+
+
+def get_ollama_models():
+    try:
+        result = subprocess.run(['ollama', 'list'], capture_output=True, text=True, check=True)
+        models = result.stdout.strip().split('\n')[1:]  # Skip header
+        return [model.split()[0] for model in models]
+    except subprocess.CalledProcessError:
+        return []
+
+
+def pull_ollama_model(model_name):
+    try:
+        subprocess.run(['ollama', 'pull', model_name], check=True)
+        return f"Successfully pulled model: {model_name}"
+    except subprocess.CalledProcessError as e:
+        return f"Failed to pull model: {e}"
+
+
+def serve_ollama_model(model_name, port):
+    try:
+        # Check if a server is already running on the specified port
+        for conn in psutil.net_connections():
+            if conn.laddr.port == int(port):
+                return f"Port {port} is already in use. Please choose a different port."
+
+        # Start the Ollama server
+        port = str(port)
+        os.environ["OLLAMA_HOST"] = port
+        cmd = f"ollama serve"
+        process = subprocess.Popen(cmd, shell=True)
+        return f"Started Ollama server for model {model_name} on port {port}. Process ID: {process.pid}"
+    except Exception as e:
+        return f"Error starting Ollama server: {e}"
+
+
+def stop_ollama_server(pid):
+    try:
+        if platform.system() == "Windows":
+            os.system(f"taskkill /F /PID {pid}")
+            return f"Stopped Ollama server with PID {pid}"
+        elif platform.system() == "Linux":
+            os.system(f"kill {pid}")
+            return f"Stopped Ollama server with PID {pid}"
+        elif platform.system() == "Darwin":
+            os.system("""osascript -e 'tell app "Ollama" to quit'""")
+            return f"(Hopefully) Stopped Ollama server using osascript..."
+    except ProcessLookupError:
+        return f"No process found with PID {pid}"
+    except Exception as e:
+        return f"Error stopping Ollama server: {e}"
+
+
+def create_ollama_tab():
+    with gr.Tab("Ollama Model Serving"):
+        gr.Markdown("# Ollama Model Serving")
+
+        with gr.Row():
+            model_list = gr.Dropdown(label="Available Models", choices=get_ollama_models())
+            refresh_button = gr.Button("Refresh Model List")
+
+        with gr.Row():
+            new_model_name = gr.Textbox(label="Model to Pull")
+            pull_button = gr.Button("Pull Model")
+
+        pull_output = gr.Textbox(label="Pull Status")
+
+        with gr.Row():
+            # FIXME - Update to update config.txt file
+            serve_model = gr.Dropdown(label="Model to Serve", choices=get_ollama_models())
+            port = gr.Number(label="Port", value=11434, precision=0)
+            serve_button = gr.Button("Start Server")
+
+        serve_output = gr.Textbox(label="Server Status")
+
+        with gr.Row():
+            pid = gr.Number(label="Server Process ID", precision=0)
+            stop_button = gr.Button("Stop Server")
+
+        stop_output = gr.Textbox(label="Stop Status")
+
+        def update_model_lists():
+            models = get_ollama_models()
+            return gr.update(choices=models), gr.update(choices=models)
+
+        refresh_button.click(update_model_lists, outputs=[model_list, serve_model])
+        pull_button.click(pull_ollama_model, inputs=[new_model_name], outputs=[pull_output])
+        serve_button.click(serve_ollama_model, inputs=[serve_model, port], outputs=[serve_output])
+        stop_button.click(stop_ollama_server, inputs=[pid], outputs=[stop_output])

App_Function_Libraries/PDF/PDF_Ingestion_Lib.py

@@ -11,171 +11,35 @@
 #
 #
 ####################
-import re
-
 # Import necessary libraries
-
-
+import re
+import os
+import shutil
+import tempfile
+from datetime import datetime
+import pymupdf
+import logging
+#
 # Import Local
-
-#######################################################################################################################
-# Function Definitions
+from App_Function_Libraries.DB.DB_Manager import add_media_with_keywords
+from App_Function_Libraries.Metrics.metrics_logger import log_counter, log_histogram
 #
-
-# Ingest a text file into the database with Title/Author/Keywords
-
-
 # Constants
 MAX_FILE_SIZE_MB = 50
 CONVERSION_TIMEOUT_SECONDS = 300
-
-# Marker PDF solution
-# def convert_pdf_to_markdown(pdf_path):
-#     """
-#     Convert a PDF file to Markdown by calling a script in another virtual environment.
-#     """
-#
-#     logging.debug(f"Marker: Converting PDF file to Markdown: {pdf_path}")
-#     # Check if the file size exceeds the maximum allowed size
-#     file_size_mb = os.path.getsize(pdf_path) / (1024 * 1024)
-#     if file_size_mb > MAX_FILE_SIZE_MB:
-#         raise ValueError(f"File size ({file_size_mb:.2f} MB) exceeds the maximum allowed size of {MAX_FILE_SIZE_MB} MB")
-#
-#     logging.debug("Marker: Converting PDF file to Markdown using Marker virtual environment")
-#     # Path to the Python interpreter in the other virtual environment
-#     other_venv_python = "Helper_Scripts/marker_venv/bin/python"
-#
-#     # Path to the conversion script
-#     converter_script = "Helper_Scripts/PDF_Converter.py"
-#
-#     logging.debug("Marker: Attempting to convert PDF file to Markdown...")
-#     try:
-#         result = subprocess.run(
-#             [other_venv_python, converter_script, pdf_path],
-#             capture_output=True,
-#             text=True,
-#             timeout=CONVERSION_TIMEOUT_SECONDS
-#         )
-#         if result.returncode != 0:
-#             raise Exception(f"Conversion failed: {result.stderr}")
-#         return result.stdout
-#     except subprocess.TimeoutExpired:
-#         raise Exception(f"PDF conversion timed out after {CONVERSION_TIMEOUT_SECONDS} seconds")
-#
-#
-# def process_and_ingest_pdf(file, title, author, keywords):
-#     if file is None:
-#         return "Please select a PDF file to upload."
-#
-#     try:
-#         # Create a temporary directory
-#         with tempfile.TemporaryDirectory() as temp_dir:
-#             # Create a path for the temporary PDF file
-#             temp_path = os.path.join(temp_dir, "temp.pdf")
-#
-#             # Copy the contents of the uploaded file to the temporary file
-#             shutil.copy(file.name, temp_path)
-#
-#             # Call the ingest_pdf_file function with the temporary file path
-#             result = ingest_pdf_file(temp_path, title, author, keywords)
-#
-#         return result
-#     except Exception as e:
-#         return f"Error processing PDF: {str(e)}"
-#
-#
-# def ingest_pdf_file(file_path, title=None, author=None, keywords=None):
-#     try:
-#         # Convert PDF to Markdown
-#         markdown_content = convert_pdf_to_markdown(file_path)
-#
-#         # If title is not provided, use the filename without extension
-#         if not title:
-#             title = os.path.splitext(os.path.basename(file_path))[0]
-#
-#         # If author is not provided, set it to 'Unknown'
-#         if not author:
-#             author = 'Unknown'
-#
-#         # If keywords are not provided, use a default keyword
-#         if not keywords:
-#             keywords = 'pdf_file,markdown_converted'
-#         else:
-#             keywords = f'pdf_file,markdown_converted,{keywords}'
-#
-#         # Add the markdown content to the database
-#         add_media_with_keywords(
-#             url=file_path,
-#             title=title,
-#             media_type='document',
-#             content=markdown_content,
-#             keywords=keywords,
-#             prompt='No prompt for PDF files',
-#             summary='No summary for PDF files',
-#             transcription_model='None',
-#             author=author,
-#             ingestion_date=datetime.now().strftime('%Y-%m-%d')
-#         )
-#
-#         return f"PDF file '{title}' converted to Markdown and ingested successfully.", file_path
-#     except ValueError as e:
-#         logging.error(f"File size error: {str(e)}")
-#         return f"Error: {str(e)}", file_path
-#     except Exception as e:
-#         logging.error(f"Error ingesting PDF file: {str(e)}")
-#         return f"Error ingesting PDF file: {str(e)}", file_path
-#
-#
-# def process_and_cleanup_pdf(file, title, author, keywords):
-#     # FIXME - Update to validate file upload/filetype is pdf....
-#     if file is None:
-#         return "No file uploaded. Please upload a PDF file."
-#
-#     temp_dir = tempfile.mkdtemp()
-#     temp_file_path = os.path.join(temp_dir, "temp.pdf")
-#
-#     try:
-#         # Copy the uploaded file to a temporary location
-#         shutil.copy2(file.name, temp_file_path)
-#
-#         # Process the file
-#         result, _ = ingest_pdf_file(temp_file_path, title, author, keywords)
-#
-#         return result
-#     except Exception as e:
-#         logging.error(f"Error in processing and cleanup: {str(e)}")
-#         return f"Error: {str(e)}"
-#     finally:
-#         # Clean up the temporary directory and its contents
-#         try:
-#             shutil.rmtree(temp_dir)
-#             logging.info(f"Removed temporary directory: {temp_dir}")
-#         except Exception as cleanup_error:
-#             logging.error(f"Error during cleanup: {str(cleanup_error)}")
-#             result += f"\nWarning: Could not remove temporary files: {str(cleanup_error)}"
-
-
-import logging
-#
 #
 #######################################################################################################################
+# Function Definitions
 #
-# Non-Marker implementation
-import os
-import shutil
-import tempfile
-from datetime import datetime
-
-import pymupdf
-
-from App_Function_Libraries.DB.DB_Manager import add_media_with_keywords
-
 
 def extract_text_and_format_from_pdf(pdf_path):
     """
     Extract text from a PDF file and convert it to Markdown, preserving formatting.
     """
     try:
+        log_counter("pdf_text_extraction_attempt", labels={"file_path": pdf_path})
+        start_time = datetime.now()
+
         markdown_text = ""
         with pymupdf.open(pdf_path) as doc:
             for page_num, page in enumerate(doc, 1):
@@ -228,9 +92,15 @@ def extract_text_and_format_from_pdf(pdf_path):
         # Clean up hyphenated words
         markdown_text = re.sub(r'(\w+)-\s*\n(\w+)', r'\1\2', markdown_text)
 
+        end_time = datetime.now()
+        processing_time = (end_time - start_time).total_seconds()
+        log_histogram("pdf_text_extraction_duration", processing_time, labels={"file_path": pdf_path})
+        log_counter("pdf_text_extraction_success", labels={"file_path": pdf_path})
+
         return markdown_text
     except Exception as e:
         logging.error(f"Error extracting text and formatting from PDF: {str(e)}")
+        log_counter("pdf_text_extraction_error", labels={"file_path": pdf_path, "error": str(e)})
         raise
 
 
@@ -239,19 +109,26 @@ def extract_metadata_from_pdf(pdf_path):
     Extract metadata from a PDF file using PyMuPDF.
     """
     try:
+        log_counter("pdf_metadata_extraction_attempt", labels={"file_path": pdf_path})
         with pymupdf.open(pdf_path) as doc:
             metadata = doc.metadata
+        log_counter("pdf_metadata_extraction_success", labels={"file_path": pdf_path})
         return metadata
     except Exception as e:
         logging.error(f"Error extracting metadata from PDF: {str(e)}")
+        log_counter("pdf_metadata_extraction_error", labels={"file_path": pdf_path, "error": str(e)})
         return {}
 
 
 def process_and_ingest_pdf(file, title, author, keywords):
     if file is None:
+        log_counter("pdf_ingestion_error", labels={"error": "No file uploaded"})
         return "Please select a PDF file to upload."
 
     try:
+        log_counter("pdf_ingestion_attempt", labels={"file_name": file.name})
+        start_time = datetime.now()
+
         # Create a temporary directory
         with tempfile.TemporaryDirectory() as temp_dir:
             # Create a path for the temporary PDF file
@@ -296,23 +173,40 @@ def process_and_ingest_pdf(file, title, author, keywords):
                 ingestion_date=datetime.now().strftime('%Y-%m-%d')
             )
 
+        end_time = datetime.now()
+        processing_time = (end_time - start_time).total_seconds()
+        log_histogram("pdf_ingestion_duration", processing_time, labels={"file_name": file.name})
+        log_counter("pdf_ingestion_success", labels={"file_name": file.name})
+
         return f"PDF file '{title}' by {author} ingested successfully and converted to Markdown."
     except Exception as e:
         logging.error(f"Error ingesting PDF file: {str(e)}")
+        log_counter("pdf_ingestion_error", labels={"file_name": file.name, "error": str(e)})
         return f"Error ingesting PDF file: {str(e)}"
 
 
 def process_and_cleanup_pdf(file, title, author, keywords):
     if file is None:
+        log_counter("pdf_processing_error", labels={"error": "No file uploaded"})
         return "No file uploaded. Please upload a PDF file."
 
     try:
+        log_counter("pdf_processing_attempt", labels={"file_name": file.name})
+        start_time = datetime.now()
+
         result = process_and_ingest_pdf(file, title, author, keywords)
+
+        end_time = datetime.now()
+        processing_time = (end_time - start_time).total_seconds()
+        log_histogram("pdf_processing_duration", processing_time, labels={"file_name": file.name})
+        log_counter("pdf_processing_success", labels={"file_name": file.name})
+
         return result
     except Exception as e:
         logging.error(f"Error in processing and cleanup: {str(e)}")
+        log_counter("pdf_processing_error", labels={"file_name": file.name, "error": str(e)})
         return f"Error: {str(e)}"
 
 #
 # End of PDF_Ingestion_Lib.py
-#######################################################################################################################
+#######################################################################################################################

App_Function_Libraries/RAG/ChromaDB_Library.py

@@ -8,14 +8,13 @@ from typing import List, Dict, Any
 import chromadb
 from chromadb import Settings
 from itertools import islice
+import numpy as np
 #
 # Local Imports:
 from App_Function_Libraries.Chunk_Lib import chunk_for_embedding, chunk_options
 from App_Function_Libraries.DB.DB_Manager import get_unprocessed_media, mark_media_as_processed
 from App_Function_Libraries.DB.SQLite_DB import process_chunks
-from App_Function_Libraries.RAG.Embeddings_Create import create_embeddings_batch
-# FIXME - related to Chunking
-from App_Function_Libraries.RAG.Embeddings_Create import create_embedding
+from App_Function_Libraries.RAG.Embeddings_Create import create_embedding, create_embeddings_batch
 from App_Function_Libraries.Summarization.Summarization_General_Lib import summarize
 from App_Function_Libraries.Utils.Utils import get_database_path, ensure_directory_exists, \
     load_comprehensive_config
@@ -216,55 +215,119 @@ def reset_chroma_collection(collection_name: str):
         logging.error(f"Error resetting ChromaDB collection: {str(e)}")
 
 
-def store_in_chroma(collection_name: str, texts: List[str], embeddings: List[List[float]], ids: List[str], metadatas: List[Dict[str, Any]]):
-    try:
-        collection = chroma_client.get_or_create_collection(name=collection_name)
+#v2
+def store_in_chroma(collection_name: str, texts: List[str], embeddings: Any, ids: List[str],
+                    metadatas: List[Dict[str, Any]]):
+    # Convert embeddings to list if it's a numpy array
+    if isinstance(embeddings, np.ndarray):
+        embeddings = embeddings.tolist()
+    elif not isinstance(embeddings, list):
+        raise TypeError("Embeddings must be either a list or a numpy array")
+
+    if not embeddings:
+        raise ValueError("No embeddings provided")
+
+    embedding_dim = len(embeddings[0])
 
-        # Log the inputs for debugging
-        logging.debug(f"Storing in ChromaDB - Collection: {collection_name}")
-        logging.debug(f"Texts (first 100 chars): {texts[0][:100]}...")
-        logging.debug(f"Embeddings (first 5 values): {embeddings[0][:5]}")
-        logging.debug(f"IDs: {ids}")
-        logging.debug(f"Metadatas: {metadatas}")
+    logging.info(f"Storing embeddings in ChromaDB - Collection: {collection_name}")
+    logging.info(f"Number of embeddings: {len(embeddings)}, Dimension: {embedding_dim}")
+
+    try:
+        # Attempt to get or create the collection
+        try:
+            collection = chroma_client.get_collection(name=collection_name)
+            logging.info(f"Existing collection '{collection_name}' found")
+
+            # Check dimension of existing embeddings
+            existing_embeddings = collection.get(limit=1, include=['embeddings'])['embeddings']
+            if existing_embeddings:
+                existing_dim = len(existing_embeddings[0])
+                if existing_dim != embedding_dim:
+                    logging.warning(f"Embedding dimension mismatch. Existing: {existing_dim}, New: {embedding_dim}")
+                    logging.warning("Deleting existing collection and creating a new one")
+                    chroma_client.delete_collection(name=collection_name)
+                    collection = chroma_client.create_collection(name=collection_name)
+            else:
+                logging.info("No existing embeddings in the collection")
+        except Exception as e:
+            logging.info(f"Collection '{collection_name}' not found. Creating new collection")
+            collection = chroma_client.create_collection(name=collection_name)
 
-        # Use upsert instead of add/update
+        # Perform the upsert operation
         collection.upsert(
             documents=texts,
             embeddings=embeddings,
             ids=ids,
             metadatas=metadatas
         )
+        logging.info(f"Successfully upserted {len(embeddings)} embeddings")
 
-        # Verify storage
-        for doc_id in ids:
-            result = collection.get(ids=[doc_id], include=["documents", "embeddings", "metadatas"])
-            if not result['embeddings'] or result['embeddings'][0] is None:
-                logging.error(f"Failed to store embedding for {doc_id}")
+        # Verify all stored embeddings
+        results = collection.get(ids=ids, include=["documents", "embeddings", "metadatas"])
+
+        for i, doc_id in enumerate(ids):
+            if results['embeddings'][i] is None:
+                raise ValueError(f"Failed to store embedding for {doc_id}")
             else:
-                logging.info(f"Embedding stored successfully for {doc_id}")
-                logging.debug(f"Stored document: {result['documents'][0][:100]}...")
-                logging.debug(f"Stored metadata: {result['metadatas'][0]}")
+                logging.debug(f"Embedding stored successfully for {doc_id}")
+                logging.debug(f"Stored document preview: {results['documents'][i][:100]}...")
+                logging.debug(f"Stored metadata: {results['metadatas'][i]}")
+
+        logging.info("Successfully stored and verified all embeddings in ChromaDB")
 
     except Exception as e:
-        logging.error(f"Error storing embeddings in ChromaDB: {str(e)}")
+        logging.error(f"Error in store_in_chroma: {str(e)}")
         raise
 
+    return collection
+
 
 # Function to perform vector search using ChromaDB + Keywords from the media_db
+#v2
 def vector_search(collection_name: str, query: str, k: int = 10) -> List[Dict[str, Any]]:
     try:
-        query_embedding = create_embedding(query, embedding_provider, embedding_model, embedding_api_url)
         collection = chroma_client.get_collection(name=collection_name)
+
+        # Fetch a sample of embeddings to check metadata
+        sample_results = collection.get(limit=10, include=["metadatas"])
+        if not sample_results['metadatas']:
+            raise ValueError("No metadata found in the collection")
+
+        # Check if all embeddings use the same model and provider
+        embedding_models = [metadata.get('embedding_model') for metadata in sample_results['metadatas'] if metadata.get('embedding_model')]
+        embedding_providers = [metadata.get('embedding_provider') for metadata in sample_results['metadatas'] if metadata.get('embedding_provider')]
+
+        if not embedding_models or not embedding_providers:
+            raise ValueError("Embedding model or provider information not found in metadata")
+
+        embedding_model = max(set(embedding_models), key=embedding_models.count)
+        embedding_provider = max(set(embedding_providers), key=embedding_providers.count)
+
+        logging.info(f"Using embedding model: {embedding_model} from provider: {embedding_provider}")
+
+        # Generate query embedding using the existing create_embedding function
+        query_embedding = create_embedding(query, embedding_provider, embedding_model, embedding_api_url)
+
+        # Ensure query_embedding is a list
+        if isinstance(query_embedding, np.ndarray):
+            query_embedding = query_embedding.tolist()
+
         results = collection.query(
             query_embeddings=[query_embedding],
             n_results=k,
             include=["documents", "metadatas"]
         )
+
+        if not results['documents'][0]:
+            logging.warning("No results found for the query")
+            return []
+
         return [{"content": doc, "metadata": meta} for doc, meta in zip(results['documents'][0], results['metadatas'][0])]
     except Exception as e:
-        logging.error(f"Error in vector_search: {str(e)}")
+        logging.error(f"Error in vector_search: {str(e)}", exc_info=True)
         raise
 
+
 def schedule_embedding(media_id: int, content: str, media_name: str):
     try:
         chunks = chunk_for_embedding(content, media_name, chunk_options)
@@ -312,4 +375,142 @@ def schedule_embedding(media_id: int, content: str, media_name: str):
 
 #
 # End of Functions for ChromaDB
-#######################################################################################################################
+#######################################################################################################################
+
+
+# FIXME - Suggestions from ChatGPT:
+# 2. Detailed Mapping and Assessment
+# a. preprocess_all_content
+#
+# Test: test_preprocess_all_content
+#
+# Coverage:
+#
+#     Mocks the get_unprocessed_media function to return a predefined unprocessed media list.
+#     Mocks process_and_store_content and mark_media_as_processed to verify their invocation with correct arguments.
+#     Asserts that process_and_store_content and mark_media_as_processed are called exactly once with expected parameters.
+#
+# Assessment:
+#
+#     Strengths: Ensures that preprocess_all_content correctly retrieves unprocessed media, processes each item, and marks it as processed.
+#     Suggestions:
+#         Multiple Media Items: Test with multiple media items to verify loop handling.
+#         Exception Handling: Simulate exceptions within process_and_store_content to ensure proper logging and continuation or halting as intended.
+#
+# b. process_and_store_content
+#
+# Test: test_process_and_store_content
+#
+# Coverage:
+#
+#     Mocks dependencies: chunk_for_embedding, process_chunks, situate_context, create_embeddings_batch, and chroma_client.
+#     Simulates the scenario where the specified ChromaDB collection does not exist initially and needs to be created.
+#     Verifies that chunks are processed, embeddings are created, stored in ChromaDB, and database queries are executed correctly.
+#
+# Assessment:
+#
+#     Strengths: Thoroughly checks the workflow of processing content, including chunking, embedding creation, and storage.
+#     Suggestions:
+#         Existing Collection: Add a test case where the collection already exists to ensure that get_collection is used without attempting to create a new one.
+#         Embedding Creation Disabled: Test with create_embeddings=False to verify alternative code paths.
+#         Error Scenarios: Simulate failures in embedding creation or storage to ensure exceptions are handled gracefully.
+#
+# c. check_embedding_status
+#
+# Test: test_check_embedding_status
+#
+# Coverage:
+#
+#     Mocks the ChromaDB client to return predefined embeddings and metadata.
+#     Verifies that the function correctly identifies the existence of embeddings and retrieves relevant metadata.
+#
+# Assessment:
+#
+#     Strengths: Confirms that the function accurately detects existing embeddings and handles metadata appropriately.
+#     Suggestions:
+#         No Embeddings Found: Test the scenario where no embeddings exist for the selected item.
+#         Missing Metadata: Simulate missing or incomplete metadata to ensure robust error handling.
+#
+# d. reset_chroma_collection
+#
+# Test: test_reset_chroma_collection
+#
+# Coverage:
+#
+#     Mocks the ChromaDB client’s delete_collection and create_collection methods.
+#     Verifies that the specified collection is deleted and recreated.
+#
+# Assessment:
+#
+#     Strengths: Ensures that the reset operation performs both deletion and creation as intended.
+#     Suggestions:
+#         Non-Existent Collection: Test resetting a collection that does not exist to verify behavior.
+#         Exception Handling: Simulate failures during deletion or creation to check error logging and propagation.
+#
+# e. store_in_chroma
+#
+# Test: test_store_in_chroma
+#
+# Coverage:
+#
+#     Mocks the ChromaDB client to return a mock collection.
+#     Verifies that documents, embeddings, IDs, and metadata are upserted correctly into the collection.
+#
+# Assessment:
+#
+#     Strengths: Confirms that embeddings and associated data are stored accurately in ChromaDB.
+#     Suggestions:
+#         Empty Embeddings: Test storing with empty embeddings to ensure proper error handling.
+#         Embedding Dimension Mismatch: Simulate a dimension mismatch to verify that the function handles it as expected.
+#
+# f. vector_search
+#
+# Test: test_vector_search
+#
+# Coverage:
+#
+#     Mocks the ChromaDB client’s get_collection, get, and query methods.
+#     Mocks the create_embedding function to return a predefined embedding.
+#     Verifies that the search retrieves the correct documents and metadata based on the query.
+#
+# Assessment:
+#
+#     Strengths: Ensures that the vector search mechanism correctly interacts with ChromaDB and returns expected results.
+#     Suggestions:
+#         No Results Found: Test queries that return no results to verify handling.
+#         Multiple Results: Ensure that multiple documents are retrieved and correctly formatted.
+#         Metadata Variations: Test with diverse metadata to confirm accurate retrieval.
+#
+# g. batched
+#
+# Test: test_batched
+#
+# Coverage:
+#
+#     Uses pytest.mark.parametrize to test multiple scenarios:
+#         Regular batching.
+#         Batch size larger than the iterable.
+#         Empty iterable.
+#
+# Assessment:
+#
+#     Strengths: Comprehensive coverage of typical and edge batching scenarios.
+#     Suggestions:
+#         Non-Integer Batch Sizes: Test with invalid batch sizes (e.g., zero, negative numbers) to ensure proper handling or error raising.
+#
+# h. situate_context and schedule_embedding
+#
+# Tests: Not directly tested
+#
+# Coverage:
+#
+#     These functions are currently not directly tested in the test_chromadb.py suite.
+#
+# Assessment:
+#
+#     Suggestions:
+#         situate_context:
+#             Unit Test: Since it's a pure function that interacts with the summarize function, create a separate test to mock summarize and verify the context generation.
+#             Edge Cases: Test with empty strings, very long texts, or special characters to ensure robustness.
+#         schedule_embedding:
+#             Integration Test: Since it orchestrates multiple operations (chunking, embedding creation, storage), consider writing an integration test that mocks all dependent functions and verifies the complete workflow.

App_Function_Libraries/RAG/Embeddings_Create.py

@@ -3,12 +3,15 @@
 #
 # Imports:
 import logging
+import os
 import time
 from functools import wraps
 from threading import Lock, Timer
 from typing import List
 #
 # 3rd-Party Imports:
+import numpy as np
+import onnxruntime as ort
 import requests
 from transformers import AutoTokenizer, AutoModel
 import torch
@@ -16,44 +19,75 @@ import torch
 # Local Imports:
 from App_Function_Libraries.LLM_API_Calls import get_openai_embeddings
 from App_Function_Libraries.Utils.Utils import load_comprehensive_config
+from App_Function_Libraries.Metrics.metrics_logger import log_counter, log_histogram
 #
 #######################################################################################################################
 #
 # Functions:
 
-# FIXME - Add all globals to summarize.py
+# FIXME - Version 2
+
+# Load configuration
 loaded_config = load_comprehensive_config()
 embedding_provider = loaded_config['Embeddings']['embedding_provider']
 embedding_model = loaded_config['Embeddings']['embedding_model']
 embedding_api_url = loaded_config['Embeddings']['embedding_api_url']
 embedding_api_key = loaded_config['Embeddings']['embedding_api_key']
+model_dir = loaded_config['Embeddings'].get('model_dir', './App_Function_Libraries/models/embedding_models/')
 
 # Embedding Chunking Settings
 chunk_size = loaded_config['Embeddings']['chunk_size']
 overlap = loaded_config['Embeddings']['overlap']
 
+# Global cache for embedding models
+embedding_models = {}
 
-# FIXME - Add logging
+# Commit hashes
+commit_hashes = {
+    "jinaai/jina-embeddings-v3": "4be32c2f5d65b95e4bcce473545b7883ec8d2edd",
+    "Alibaba-NLP/gte-large-en-v1.5": "104333d6af6f97649377c2afbde10a7704870c7b",
+    "dunzhang/setll_en_400M_v5": "2aa5579fcae1c579de199a3866b6e514bbbf5d10"
+}
 
 class HuggingFaceEmbedder:
-    def __init__(self, model_name, timeout_seconds=120):  # Default timeout of 2 minutes
+    def __init__(self, model_name, cache_dir, timeout_seconds=30):
         self.model_name = model_name
+        self.cache_dir = cache_dir  # Store cache_dir
         self.tokenizer = None
         self.model = None
         self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
         self.timeout_seconds = timeout_seconds
         self.last_used_time = 0
         self.unload_timer = None
+        log_counter("huggingface_embedder_init", labels={"model_name": model_name})
 
     def load_model(self):
+        log_counter("huggingface_model_load_attempt", labels={"model_name": self.model_name})
+        start_time = time.time()
+        # https://huggingface.co/docs/transformers/custom_models
         if self.model is None:
-            self.tokenizer = AutoTokenizer.from_pretrained(self.model_name)
-            self.model = AutoModel.from_pretrained(self.model_name)
+            # Pass cache_dir to from_pretrained to specify download directory
+            self.tokenizer = AutoTokenizer.from_pretrained(
+                self.model_name,
+                trust_remote_code=True,
+                cache_dir=self.cache_dir,  # Specify cache directory
+                revision=commit_hashes.get(self.model_name, None)  # Pass commit hash
+            )
+            self.model = AutoModel.from_pretrained(
+                self.model_name,
+                trust_remote_code=True,
+                cache_dir=self.cache_dir,  # Specify cache directory
+                revision=commit_hashes.get(self.model_name, None)  # Pass commit hash
+            )
             self.model.to(self.device)
         self.last_used_time = time.time()
         self.reset_timer()
+        load_time = time.time() - start_time
+        log_histogram("huggingface_model_load_duration", load_time, labels={"model_name": self.model_name})
+        log_counter("huggingface_model_load_success", labels={"model_name": self.model_name})
 
     def unload_model(self):
+        log_counter("huggingface_model_unload", labels={"model_name": self.model_name})
         if self.model is not None:
             del self.model
             del self.tokenizer
@@ -71,17 +105,119 @@ class HuggingFaceEmbedder:
         self.unload_timer.start()
 
     def create_embeddings(self, texts):
+        log_counter("huggingface_create_embeddings_attempt", labels={"model_name": self.model_name})
+        start_time = time.time()
         self.load_model()
-        inputs = self.tokenizer(texts, return_tensors="pt", padding=True, truncation=True, max_length=512)
+        # https://huggingface.co/docs/transformers/custom_models
+        inputs = self.tokenizer(
+            texts,
+            return_tensors="pt",
+            padding=True,
+            truncation=True,
+            max_length=512
+        )
         inputs = {k: v.to(self.device) for k, v in inputs.items()}
-        with torch.no_grad():
-            outputs = self.model(**inputs)
-        embeddings = outputs.last_hidden_state.mean(dim=1)
-        return embeddings.cpu().numpy()
+        try:
+            with torch.no_grad():
+                outputs = self.model(**inputs)
+            embeddings = outputs.last_hidden_state.mean(dim=1)
+            return embeddings.cpu().float().numpy()  # Convert to float32 before returning
+        except RuntimeError as e:
+            if "Got unsupported ScalarType BFloat16" in str(e):
+                logging.warning("BFloat16 not supported. Falling back to float32.")
+                # Convert model to float32
+                self.model = self.model.float()
+                with torch.no_grad():
+                    outputs = self.model(**inputs)
+                embeddings = outputs.last_hidden_state.mean(dim=1)
+                embedding_time = time.time() - start_time
+                log_histogram("huggingface_create_embeddings_duration", embedding_time,
+                              labels={"model_name": self.model_name})
+                log_counter("huggingface_create_embeddings_success", labels={"model_name": self.model_name})
+                return embeddings.cpu().float().numpy()
+            else:
+                log_counter("huggingface_create_embeddings_failure", labels={"model_name": self.model_name})
+                raise
+
+class ONNXEmbedder:
+    def __init__(self, model_name, onnx_model_dir, timeout_seconds=30):
+        self.model_name = model_name
+        self.model_path = os.path.join(onnx_model_dir, f"{model_name}.onnx")
+        # https://huggingface.co/docs/transformers/custom_models
+        self.tokenizer = AutoTokenizer.from_pretrained(
+            model_name,
+            trust_remote_code=True,
+            cache_dir=onnx_model_dir,  # Ensure tokenizer uses the same directory
+            revision=commit_hashes.get(model_name, None)  # Pass commit hash
+        )
+        self.session = None
+        self.timeout_seconds = timeout_seconds
+        self.last_used_time = 0
+        self.unload_timer = None
+        self.device = "cpu"  # ONNX Runtime will default to CPU unless GPU is configured
+        log_counter("onnx_embedder_init", labels={"model_name": model_name})
+
+    def load_model(self):
+        log_counter("onnx_model_load_attempt", labels={"model_name": self.model_name})
+        start_time = time.time()
+        if self.session is None:
+            if not os.path.exists(self.model_path):
+                raise FileNotFoundError(f"ONNX model not found at {self.model_path}")
+            logging.info(f"Loading ONNX model from {self.model_path}")
+            self.session = ort.InferenceSession(self.model_path)
+        self.last_used_time = time.time()
+        self.reset_timer()
+        load_time = time.time() - start_time
+        log_histogram("onnx_model_load_duration", load_time, labels={"model_name": self.model_name})
+        log_counter("onnx_model_load_success", labels={"model_name": self.model_name})
 
-# Global variable to hold the embedder
-huggingface_embedder = None
+    def unload_model(self):
+        log_counter("onnx_model_unload", labels={"model_name": self.model_name})
+        if self.session is not None:
+            logging.info("Unloading ONNX model to free resources.")
+            self.session = None
+        if self.unload_timer:
+            self.unload_timer.cancel()
+
+    def reset_timer(self):
+        if self.unload_timer:
+            self.unload_timer.cancel()
+        self.unload_timer = Timer(self.timeout_seconds, self.unload_model)
+        self.unload_timer.start()
+
+    def create_embeddings(self, texts: List[str]) -> List[List[float]]:
+        log_counter("onnx_create_embeddings_attempt", labels={"model_name": self.model_name})
+        start_time = time.time()
+        self.load_model()
+        try:
+            inputs = self.tokenizer(
+                texts,
+                return_tensors="np",
+                padding=True,
+                truncation=True,
+                max_length=512
+            )
+            input_ids = inputs["input_ids"].astype(np.int64)
+            attention_mask = inputs["attention_mask"].astype(np.int64)
 
+            ort_inputs = {
+                "input_ids": input_ids,
+                "attention_mask": attention_mask
+            }
+
+            ort_outputs = self.session.run(None, ort_inputs)
+
+            last_hidden_state = ort_outputs[0]
+            embeddings = np.mean(last_hidden_state, axis=1)
+
+            embedding_time = time.time() - start_time
+            log_histogram("onnx_create_embeddings_duration", embedding_time, labels={"model_name": self.model_name})
+            log_counter("onnx_create_embeddings_success", labels={"model_name": self.model_name})
+            return embeddings.tolist()
+        except Exception as e:
+            log_counter("onnx_create_embeddings_failure", labels={"model_name": self.model_name})
+            logging.error(f"Error creating embeddings with ONNX model: {str(e)}")
+            raise
 
 class RateLimiter:
     def __init__(self, max_calls, period):
@@ -102,7 +238,6 @@ class RateLimiter:
             return func(*args, **kwargs)
         return wrapper
 
-
 def exponential_backoff(max_retries=5, base_delay=1):
     def decorator(func):
         @wraps(func)
@@ -119,72 +254,353 @@ def exponential_backoff(max_retries=5, base_delay=1):
         return wrapper
     return decorator
 
-
-# FIXME - refactor/setup to use config file & perform chunking
 @exponential_backoff()
-@RateLimiter(max_calls=50, period=60)  # Adjust these values based on API limits
-def create_embeddings_batch(texts: List[str], provider: str, model: str, api_url: str, timeout_seconds: int = 300) -> \
-List[List[float]]:
-    global huggingface_embedder
-
-    if provider.lower() == 'huggingface':
-        if huggingface_embedder is None or huggingface_embedder.model_name != model:
-            if huggingface_embedder is not None:
-                huggingface_embedder.unload_model()
-            huggingface_embedder = HuggingFaceEmbedder(model, timeout_seconds)
-
-        embeddings = huggingface_embedder.create_embeddings(texts).tolist()
-        return embeddings
-
-    elif provider.lower() == 'openai':
-        logging.debug(f"Creating embeddings for {len(texts)} texts using OpenAI API")
-        return [create_openai_embedding(text, model) for text in texts]
-
-    elif provider.lower() == 'local':
-        response = requests.post(
-            api_url,
-            json={"texts": texts, "model": model},
-            headers={"Authorization": f"Bearer {embedding_api_key}"}
-        )
-        if response.status_code == 200:
-            return response.json()['embeddings']
-        else:
-            raise Exception(f"Error from local API: {response.text}")
-    else:
-        raise ValueError(f"Unsupported embedding provider: {provider}")
-
-
-def create_embedding(text: str, provider: str, model: str, api_url: str) -> List[float]:
-    return create_embeddings_batch([text], provider, model, api_url)[0]
-
-# FIXME
-def create_stella_embeddings(text: str) -> List[float]:
-    if embedding_provider == 'local':
-        # Load the model and tokenizer
-        tokenizer = AutoTokenizer.from_pretrained("dunzhang/stella_en_400M_v5")
-        model = AutoModel.from_pretrained("dunzhang/stella_en_400M_v5")
+@RateLimiter(max_calls=50, period=60)
+def create_embeddings_batch(texts: List[str],
+                            provider: str,
+                            model: str,
+                            api_url: str,
+                            timeout_seconds: int = 300
+                            ) -> List[List[float]]:
+    global embedding_models
+    log_counter("create_embeddings_batch_attempt", labels={"provider": provider, "model": model})
+    start_time = time.time()
 
-        # Tokenize and encode the text
-        inputs = tokenizer(text, return_tensors="pt", padding=True, truncation=True, max_length=512)
+    try:
+        if provider.lower() == 'huggingface':
+            if model not in embedding_models:
+                if model == "dunzhang/stella_en_400M_v5":
+                    embedding_models[model] = ONNXEmbedder(model, model_dir, timeout_seconds)
+                else:
+                    # Pass model_dir to HuggingFaceEmbedder
+                    embedding_models[model] = HuggingFaceEmbedder(model, model_dir, timeout_seconds)
+            embedder = embedding_models[model]
+            embedding_time = time.time() - start_time
+            log_histogram("create_embeddings_batch_duration", embedding_time,
+                          labels={"provider": provider, "model": model})
+            log_counter("create_embeddings_batch_success", labels={"provider": provider, "model": model})
+            return embedder.create_embeddings(texts)
 
-        # Generate embeddings
-        with torch.no_grad():
-            outputs = model(**inputs)
+        elif provider.lower() == 'openai':
+            logging.debug(f"Creating embeddings for {len(texts)} texts using OpenAI API")
+            embedding_time = time.time() - start_time
+            log_histogram("create_embeddings_batch_duration", embedding_time,
+                          labels={"provider": provider, "model": model})
+            log_counter("create_embeddings_batch_success", labels={"provider": provider, "model": model})
+            return [create_openai_embedding(text, model) for text in texts]
 
-        # Use the mean of the last hidden state as the sentence embedding
-        embeddings = outputs.last_hidden_state.mean(dim=1)
-
-        return embeddings[0].tolist()  # Convert to list for consistency
-    elif embedding_provider == 'openai':
-        return get_openai_embeddings(text, embedding_model)
-    else:
-        raise ValueError(f"Unsupported embedding provider: {embedding_provider}")
+        elif provider.lower() == 'local':
+            response = requests.post(
+                api_url,
+                json={"texts": texts, "model": model},
+                headers={"Authorization": f"Bearer {embedding_api_key}"}
+            )
+            if response.status_code == 200:
+                embedding_time = time.time() - start_time
+                log_histogram("create_embeddings_batch_duration", embedding_time,
+                              labels={"provider": provider, "model": model})
+                log_counter("create_embeddings_batch_success", labels={"provider": provider, "model": model})
+                return response.json()['embeddings']
+            else:
+                raise Exception(f"Error from local API: {response.text}")
+        else:
+            raise ValueError(f"Unsupported embedding provider: {provider}")
+    except Exception as e:
+        log_counter("create_embeddings_batch_error", labels={"provider": provider, "model": model, "error": str(e)})
+        logging.error(f"Error in create_embeddings_batch: {str(e)}")
+        raise
 
+def create_embedding(text: str, provider: str, model: str, api_url: str) -> List[float]:
+    log_counter("create_embedding_attempt", labels={"provider": provider, "model": model})
+    start_time = time.time()
+    embedding = create_embeddings_batch([text], provider, model, api_url)[0]
+    if isinstance(embedding, np.ndarray):
+        embedding = embedding.tolist()
+    embedding_time = time.time() - start_time
+    log_histogram("create_embedding_duration", embedding_time, labels={"provider": provider, "model": model})
+    log_counter("create_embedding_success", labels={"provider": provider, "model": model})
+    return embedding
 
 def create_openai_embedding(text: str, model: str) -> List[float]:
+    log_counter("create_openai_embedding_attempt", labels={"model": model})
+    start_time = time.time()
     embedding = get_openai_embeddings(text, model)
+    embedding_time = time.time() - start_time
+    log_histogram("create_openai_embedding_duration", embedding_time, labels={"model": model})
+    log_counter("create_openai_embedding_success", labels={"model": model})
     return embedding
 
+
+# FIXME - Version 1
+# # FIXME - Add all globals to summarize.py
+# loaded_config = load_comprehensive_config()
+# embedding_provider = loaded_config['Embeddings']['embedding_provider']
+# embedding_model = loaded_config['Embeddings']['embedding_model']
+# embedding_api_url = loaded_config['Embeddings']['embedding_api_url']
+# embedding_api_key = loaded_config['Embeddings']['embedding_api_key']
+#
+# # Embedding Chunking Settings
+# chunk_size = loaded_config['Embeddings']['chunk_size']
+# overlap = loaded_config['Embeddings']['overlap']
+#
+#
+# # FIXME - Add logging
+#
+# class HuggingFaceEmbedder:
+#     def __init__(self, model_name, timeout_seconds=120):  # Default timeout of 2 minutes
+#         self.model_name = model_name
+#         self.tokenizer = None
+#         self.model = None
+#         self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+#         self.timeout_seconds = timeout_seconds
+#         self.last_used_time = 0
+#         self.unload_timer = None
+#
+#     def load_model(self):
+#         if self.model is None:
+#             self.tokenizer = AutoTokenizer.from_pretrained(self.model_name)
+#             self.model = AutoModel.from_pretrained(self.model_name)
+#             self.model.to(self.device)
+#         self.last_used_time = time.time()
+#         self.reset_timer()
+#
+#     def unload_model(self):
+#         if self.model is not None:
+#             del self.model
+#             del self.tokenizer
+#             if torch.cuda.is_available():
+#                 torch.cuda.empty_cache()
+#             self.model = None
+#             self.tokenizer = None
+#         if self.unload_timer:
+#             self.unload_timer.cancel()
+#
+#     def reset_timer(self):
+#         if self.unload_timer:
+#             self.unload_timer.cancel()
+#         self.unload_timer = Timer(self.timeout_seconds, self.unload_model)
+#         self.unload_timer.start()
+#
+#     def create_embeddings(self, texts):
+#         self.load_model()
+#         inputs = self.tokenizer(texts, return_tensors="pt", padding=True, truncation=True, max_length=512)
+#         inputs = {k: v.to(self.device) for k, v in inputs.items()}
+#         with torch.no_grad():
+#             outputs = self.model(**inputs)
+#         embeddings = outputs.last_hidden_state.mean(dim=1)
+#         return embeddings.cpu().numpy()
+#
+# # Global variable to hold the embedder
+# huggingface_embedder = None
+#
+#
+# class RateLimiter:
+#     def __init__(self, max_calls, period):
+#         self.max_calls = max_calls
+#         self.period = period
+#         self.calls = []
+#         self.lock = Lock()
+#
+#     def __call__(self, func):
+#         def wrapper(*args, **kwargs):
+#             with self.lock:
+#                 now = time.time()
+#                 self.calls = [call for call in self.calls if call > now - self.period]
+#                 if len(self.calls) >= self.max_calls:
+#                     sleep_time = self.calls[0] - (now - self.period)
+#                     time.sleep(sleep_time)
+#                 self.calls.append(time.time())
+#             return func(*args, **kwargs)
+#         return wrapper
+#
+#
+# def exponential_backoff(max_retries=5, base_delay=1):
+#     def decorator(func):
+#         @wraps(func)
+#         def wrapper(*args, **kwargs):
+#             for attempt in range(max_retries):
+#                 try:
+#                     return func(*args, **kwargs)
+#                 except Exception as e:
+#                     if attempt == max_retries - 1:
+#                         raise
+#                     delay = base_delay * (2 ** attempt)
+#                     logging.warning(f"Attempt {attempt + 1} failed. Retrying in {delay} seconds. Error: {str(e)}")
+#                     time.sleep(delay)
+#         return wrapper
+#     return decorator
+#
+#
+# # FIXME - refactor/setup to use config file & perform chunking
+# @exponential_backoff()
+# @RateLimiter(max_calls=50, period=60)
+# def create_embeddings_batch(texts: List[str], provider: str, model: str, api_url: str, timeout_seconds: int = 300) -> List[List[float]]:
+#     global embedding_models
+#
+#     try:
+#         if provider.lower() == 'huggingface':
+#             if model not in embedding_models:
+#                 if model == "dunzhang/stella_en_400M_v5":
+#                     embedding_models[model] = ONNXEmbedder(model, model_dir, timeout_seconds)
+#                 else:
+#                     embedding_models[model] = HuggingFaceEmbedder(model, timeout_seconds)
+#             embedder = embedding_models[model]
+#             return embedder.create_embeddings(texts)
+#
+#         elif provider.lower() == 'openai':
+#             logging.debug(f"Creating embeddings for {len(texts)} texts using OpenAI API")
+#             return [create_openai_embedding(text, model) for text in texts]
+#
+#         elif provider.lower() == 'local':
+#             response = requests.post(
+#                 api_url,
+#                 json={"texts": texts, "model": model},
+#                 headers={"Authorization": f"Bearer {embedding_api_key}"}
+#             )
+#             if response.status_code == 200:
+#                 return response.json()['embeddings']
+#             else:
+#                 raise Exception(f"Error from local API: {response.text}")
+#         else:
+#             raise ValueError(f"Unsupported embedding provider: {provider}")
+#     except Exception as e:
+#         logging.error(f"Error in create_embeddings_batch: {str(e)}")
+#         raise
+#
+# def create_embedding(text: str, provider: str, model: str, api_url: str) -> List[float]:
+#     return create_embeddings_batch([text], provider, model, api_url)[0]
+#
+#
+# def create_openai_embedding(text: str, model: str) -> List[float]:
+#     embedding = get_openai_embeddings(text, model)
+#     return embedding
+#
+#
+# # FIXME - refactor to use onnx embeddings callout
+# def create_stella_embeddings(text: str) -> List[float]:
+#     if embedding_provider == 'local':
+#         # Load the model and tokenizer
+#         tokenizer = AutoTokenizer.from_pretrained("dunzhang/stella_en_400M_v5")
+#         model = AutoModel.from_pretrained("dunzhang/stella_en_400M_v5")
+#
+#         # Tokenize and encode the text
+#         inputs = tokenizer(text, return_tensors="pt", padding=True, truncation=True, max_length=512)
+#
+#         # Generate embeddings
+#         with torch.no_grad():
+#             outputs = model(**inputs)
+#
+#         # Use the mean of the last hidden state as the sentence embedding
+#         embeddings = outputs.last_hidden_state.mean(dim=1)
+#
+#         return embeddings[0].tolist()  # Convert to list for consistency
+#     elif embedding_provider == 'openai':
+#         return get_openai_embeddings(text, embedding_model)
+#     else:
+#         raise ValueError(f"Unsupported embedding provider: {embedding_provider}")
+# #
+# # End of F
+# ##############################################################
+#
+#
+# ##############################################################
+# #
+# # ONNX Embeddings Functions
+#
+# # FIXME - UPDATE
+# # Define the model path
+# model_dir = "/tldw/App_Function_Libraries/models/embedding_models/"
+# model_name = "your-huggingface-model-name"
+# onnx_model_path = os.path.join(model_dir, model_name, "model.onnx")
+#
+# # Tokenizer download (if applicable)
+# #tokenizer = AutoTokenizer.from_pretrained(model_name)
+#
+# # Ensure the model directory exists
+# #if not os.path.exists(onnx_model_path):
+#     # You can add logic to download the ONNX model from a remote source
+#     # if it's not already available in the folder.
+#     # Example: huggingface_hub.download (if model is hosted on Hugging Face Hub)
+# #    raise Exception(f"ONNX model not found at {onnx_model_path}")
+#
+# class ONNXEmbedder:
+#     def __init__(self, model_name, model_dir, timeout_seconds=120):
+#         self.model_name = model_name
+#         self.model_path = os.path.join(model_dir, f"{model_name}.onnx")
+#         self.tokenizer = AutoTokenizer.from_pretrained(model_name)
+#         self.session = None
+#         self.timeout_seconds = timeout_seconds
+#         self.last_used_time = 0
+#         self.unload_timer = None
+#         self.device = "cpu"  # ONNX Runtime will default to CPU unless GPU is configured
+#
+#     def load_model(self):
+#         if self.session is None:
+#             if not os.path.exists(self.model_path):
+#                 raise FileNotFoundError(f"ONNX model not found at {self.model_path}")
+#             logging.info(f"Loading ONNX model from {self.model_path}")
+#             self.session = ort.InferenceSession(self.model_path)
+#         self.last_used_time = time.time()
+#         self.reset_timer()
+#
+#     def unload_model(self):
+#         if self.session is not None:
+#             logging.info("Unloading ONNX model to free resources.")
+#             self.session = None
+#         if self.unload_timer:
+#             self.unload_timer.cancel()
+#
+#     def reset_timer(self):
+#         if self.unload_timer:
+#             self.unload_timer.cancel()
+#         self.unload_timer = Timer(self.timeout_seconds, self.unload_model)
+#         self.unload_timer.start()
+#
+#     def create_embeddings(self, texts: List[str]) -> List[List[float]]:
+#         self.load_model()
+#
+#         try:
+#             inputs = self.tokenizer(texts, return_tensors="np", padding=True, truncation=True, max_length=512)
+#             input_ids = inputs["input_ids"].astype(np.int64)
+#             attention_mask = inputs["attention_mask"].astype(np.int64)
+#
+#             ort_inputs = {
+#                 "input_ids": input_ids,
+#                 "attention_mask": attention_mask
+#             }
+#
+#             ort_outputs = self.session.run(None, ort_inputs)
+#
+#             last_hidden_state = ort_outputs[0]
+#             embeddings = np.mean(last_hidden_state, axis=1)
+#
+#             return embeddings.tolist()
+#         except Exception as e:
+#             logging.error(f"Error creating embeddings with ONNX model: {str(e)}")
+#             raise
+#
+# # Global cache for the ONNX embedder instance
+# onnx_embedder = None
+#
+# # Global cache for embedding models
+# embedding_models = {}
+#
+# def create_onnx_embeddings(texts: List[str]) -> List[List[float]]:
+#     global onnx_embedder
+#     model_dir = "/tldw/App_Function_Libraries/models/embedding_models/"
+#     model_name = "your-huggingface-model-name"  # This can be pulled from config
+#
+#     if onnx_embedder is None:
+#         onnx_embedder = ONNXEmbedder(model_name=model_name, model_dir=model_dir)
+#
+#     # Generate embeddings
+#     embeddings = onnx_embedder.create_embeddings(texts)
+#     return embeddings
+#
+# #
+# # End of ONNX Embeddings Functions
+# ##############################################################
+
 #
 # End of File.
 #######################################################################################################################

App_Function_Libraries/RAG/RAG_Library_2.py

@@ -5,15 +5,24 @@
 import configparser
 import logging
 import os
+import time
 from typing import Dict, Any, List, Optional
+
+from App_Function_Libraries.DB.Character_Chat_DB import get_character_chats, perform_full_text_search_chat, \
+    fetch_keywords_for_chats
+#
 # Local Imports
 from App_Function_Libraries.RAG.ChromaDB_Library import process_and_store_content, vector_search, chroma_client
+from App_Function_Libraries.RAG.RAG_Persona_Chat import perform_vector_search_chat
+from App_Function_Libraries.Summarization.Local_Summarization_Lib import summarize_with_custom_openai
 from App_Function_Libraries.Web_Scraping.Article_Extractor_Lib import scrape_article
 from App_Function_Libraries.DB.DB_Manager import search_db, fetch_keywords_for_media
 from App_Function_Libraries.Utils.Utils import load_comprehensive_config
+from App_Function_Libraries.Metrics.metrics_logger import log_counter, log_histogram
 #
 # 3rd-Party Imports
 import openai
+from flashrank import Ranker, RerankRequest
 #
 ########################################################################################################################
 #
@@ -109,6 +118,8 @@ config.read('config.txt')
 
 # RAG Search with keyword filtering
 def enhanced_rag_pipeline(query: str, api_choice: str, keywords: str = None) -> Dict[str, Any]:
+    log_counter("enhanced_rag_pipeline_attempt", labels={"api_choice": api_choice})
+    start_time = time.time()
     try:
         # Load embedding provider from config, or fallback to 'openai'
         embedding_provider = config.get('Embeddings', 'provider', fallback='openai')
@@ -118,32 +129,57 @@ def enhanced_rag_pipeline(query: str, api_choice: str, keywords: str = None) ->
 
         # Process keywords if provided
         keyword_list = [k.strip().lower() for k in keywords.split(',')] if keywords else []
-        logging.debug(f"enhanced_rag_pipeline - Keywords: {keyword_list}")
+        logging.debug(f"\n\nenhanced_rag_pipeline - Keywords: {keyword_list}")
 
         # Fetch relevant media IDs based on keywords if keywords are provided
         relevant_media_ids = fetch_relevant_media_ids(keyword_list) if keyword_list else None
-        logging.debug(f"enhanced_rag_pipeline - relevant media IDs: {relevant_media_ids}")
+        logging.debug(f"\n\nenhanced_rag_pipeline - relevant media IDs: {relevant_media_ids}")
 
         # Perform vector search
         vector_results = perform_vector_search(query, relevant_media_ids)
-        logging.debug(f"enhanced_rag_pipeline - Vector search results: {vector_results}")
+        logging.debug(f"\n\nenhanced_rag_pipeline - Vector search results: {vector_results}")
 
         # Perform full-text search
         fts_results = perform_full_text_search(query, relevant_media_ids)
-        logging.debug(f"enhanced_rag_pipeline - Full-text search results: {fts_results}")
+        logging.debug("\n\nenhanced_rag_pipeline - Full-text search results:")
+        logging.debug(
+            "\n\nenhanced_rag_pipeline - Full-text search results:\n" + "\n".join(
+                [str(item) for item in fts_results]) + "\n"
+        )
 
         # Combine results
         all_results = vector_results + fts_results
 
-        # FIXME - Apply Re-Ranking of results here
-        apply_re_ranking = False
+        apply_re_ranking = True
         if apply_re_ranking:
-            # Implement re-ranking logic here
-            pass
-        # Extract content from results
+            logging.debug(f"\nenhanced_rag_pipeline - Applying Re-Ranking")
+            # FIXME - add option to use re-ranking at call time
+            # FIXME - specify model + add param to modify at call time
+            # FIXME - add option to set a custom top X results
+            # You can specify a model if necessary, e.g., model_name="ms-marco-MiniLM-L-12-v2"
+            ranker = Ranker()
+
+            # Prepare passages for re-ranking
+            passages = [{"id": i, "text": result['content']} for i, result in enumerate(all_results)]
+            rerank_request = RerankRequest(query=query, passages=passages)
+
+            # Rerank the results
+            reranked_results = ranker.rerank(rerank_request)
+
+            # Sort results based on the re-ranking score
+            reranked_results = sorted(reranked_results, key=lambda x: x['score'], reverse=True)
+
+            # Log reranked results
+            logging.debug(f"\n\nenhanced_rag_pipeline - Reranked results: {reranked_results}")
+
+            # Update all_results based on reranking
+            all_results = [all_results[result['id']] for result in reranked_results]
+
+        # Extract content from results (top 10)
         context = "\n".join([result['content'] for result in all_results[:10]])  # Limit to top 10 results
         logging.debug(f"Context length: {len(context)}")
         logging.debug(f"Context: {context[:200]}")
+
         # Generate answer using the selected API
         answer = generate_answer(api_choice, context, query)
 
@@ -153,111 +189,220 @@ def enhanced_rag_pipeline(query: str, api_choice: str, keywords: str = None) ->
                 "answer": "No relevant information based on your query and keywords were found in the database. Your query has been directly passed to the LLM, and here is its answer: \n\n" + answer,
                 "context": "No relevant information based on your query and keywords were found in the database. The only context used was your query: \n\n" + query
             }
-
+        # Metrics
+        pipeline_duration = time.time() - start_time
+        log_histogram("enhanced_rag_pipeline_duration", pipeline_duration, labels={"api_choice": api_choice})
+        log_counter("enhanced_rag_pipeline_success", labels={"api_choice": api_choice})
         return {
             "answer": answer,
             "context": context
         }
 
     except Exception as e:
+        # Metrics
+        log_counter("enhanced_rag_pipeline_error", labels={"api_choice": api_choice, "error": str(e)})
+        logging.error(f"Error in enhanced_rag_pipeline: {str(e)}")
         logging.error(f"Error in enhanced_rag_pipeline: {str(e)}")
         return {
             "answer": "An error occurred while processing your request.",
             "context": ""
         }
 
-
+# Need to write a test for this function FIXME
 def generate_answer(api_choice: str, context: str, query: str) -> str:
+    # Metrics
+    log_counter("generate_answer_attempt", labels={"api_choice": api_choice})
+    start_time = time.time()
     logging.debug("Entering generate_answer function")
     config = load_comprehensive_config()
     logging.debug(f"Config sections: {config.sections()}")
     prompt = f"Context: {context}\n\nQuestion: {query}"
-    if api_choice == "OpenAI":
-        from App_Function_Libraries.Summarization.Summarization_General_Lib import summarize_with_openai
-        return summarize_with_openai(config['API']['openai_api_key'], prompt, "")
-    elif api_choice == "Anthropic":
-        from App_Function_Libraries.Summarization.Summarization_General_Lib import summarize_with_anthropic
-        return summarize_with_anthropic(config['API']['anthropic_api_key'], prompt, "")
-    elif api_choice == "Cohere":
-        from App_Function_Libraries.Summarization.Summarization_General_Lib import summarize_with_cohere
-        return summarize_with_cohere(config['API']['cohere_api_key'], prompt, "")
-    elif api_choice == "Groq":
-        from App_Function_Libraries.Summarization.Summarization_General_Lib import summarize_with_groq
-        return summarize_with_groq(config['API']['groq_api_key'], prompt, "")
-    elif api_choice == "OpenRouter":
-        from App_Function_Libraries.Summarization.Summarization_General_Lib import summarize_with_openrouter
-        return summarize_with_openrouter(config['API']['openrouter_api_key'], prompt, "")
-    elif api_choice == "HuggingFace":
-        from App_Function_Libraries.Summarization.Summarization_General_Lib import summarize_with_huggingface
-        return summarize_with_huggingface(config['API']['huggingface_api_key'], prompt, "")
-    elif api_choice == "DeepSeek":
-        from App_Function_Libraries.Summarization.Summarization_General_Lib import summarize_with_deepseek
-        return summarize_with_deepseek(config['API']['deepseek_api_key'], prompt, "")
-    elif api_choice == "Mistral":
-        from App_Function_Libraries.Summarization.Summarization_General_Lib import summarize_with_mistral
-        return summarize_with_mistral(config['API']['mistral_api_key'], prompt, "")
-    elif api_choice == "Local-LLM":
-        from App_Function_Libraries.Summarization.Local_Summarization_Lib import summarize_with_local_llm
-        return summarize_with_local_llm(config['API']['local_llm_path'], prompt, "")
-    elif api_choice == "Llama.cpp":
-        from App_Function_Libraries.Summarization.Local_Summarization_Lib import summarize_with_llama
-        return summarize_with_llama(config['API']['llama_api_key'], prompt, "")
-    elif api_choice == "Kobold":
-        from App_Function_Libraries.Summarization.Local_Summarization_Lib import summarize_with_kobold
-        return summarize_with_kobold(config['API']['kobold_api_key'], prompt, "")
-    elif api_choice == "Ooba":
-        from App_Function_Libraries.Summarization.Local_Summarization_Lib import summarize_with_oobabooga
-        return summarize_with_oobabooga(config['API']['ooba_api_key'], prompt, "")
-    elif api_choice == "TabbyAPI":
-        from App_Function_Libraries.Summarization.Local_Summarization_Lib import summarize_with_tabbyapi
-        return summarize_with_tabbyapi(config['API']['tabby_api_key'], prompt, "")
-    elif api_choice == "vLLM":
-        from App_Function_Libraries.Summarization.Local_Summarization_Lib import summarize_with_vllm
-        return summarize_with_vllm(config['API']['vllm_api_key'], prompt, "")
-    elif api_choice == "ollama":
-        from App_Function_Libraries.Summarization.Local_Summarization_Lib import summarize_with_ollama
-        return summarize_with_ollama(config['API']['ollama_api_key'], prompt, "")
-    else:
-        raise ValueError(f"Unsupported API choice: {api_choice}")
+    try:
+        if api_choice == "OpenAI":
+            from App_Function_Libraries.Summarization.Summarization_General_Lib import summarize_with_openai
+            answer_generation_duration = time.time() - start_time
+            log_histogram("generate_answer_duration", answer_generation_duration, labels={"api_choice": api_choice})
+            log_counter("generate_answer_success", labels={"api_choice": api_choice})
+            return summarize_with_openai(config['API']['openai_api_key'], prompt, "")
+
+        elif api_choice == "Anthropic":
+            from App_Function_Libraries.Summarization.Summarization_General_Lib import summarize_with_anthropic
+            answer_generation_duration = time.time() - start_time
+            log_histogram("generate_answer_duration", answer_generation_duration, labels={"api_choice": api_choice})
+            log_counter("generate_answer_success", labels={"api_choice": api_choice})
+            return summarize_with_anthropic(config['API']['anthropic_api_key'], prompt, "")
+
+        elif api_choice == "Cohere":
+            from App_Function_Libraries.Summarization.Summarization_General_Lib import summarize_with_cohere
+            answer_generation_duration = time.time() - start_time
+            log_histogram("generate_answer_duration", answer_generation_duration, labels={"api_choice": api_choice})
+            log_counter("generate_answer_success", labels={"api_choice": api_choice})
+            return summarize_with_cohere(config['API']['cohere_api_key'], prompt, "")
+
+        elif api_choice == "Groq":
+            from App_Function_Libraries.Summarization.Summarization_General_Lib import summarize_with_groq
+            answer_generation_duration = time.time() - start_time
+            log_histogram("generate_answer_duration", answer_generation_duration, labels={"api_choice": api_choice})
+            log_counter("generate_answer_success", labels={"api_choice": api_choice})
+            return summarize_with_groq(config['API']['groq_api_key'], prompt, "")
+
+        elif api_choice == "OpenRouter":
+            from App_Function_Libraries.Summarization.Summarization_General_Lib import summarize_with_openrouter
+            answer_generation_duration = time.time() - start_time
+            log_histogram("generate_answer_duration", answer_generation_duration, labels={"api_choice": api_choice})
+            log_counter("generate_answer_success", labels={"api_choice": api_choice})
+            return summarize_with_openrouter(config['API']['openrouter_api_key'], prompt, "")
+
+        elif api_choice == "HuggingFace":
+            from App_Function_Libraries.Summarization.Summarization_General_Lib import summarize_with_huggingface
+            answer_generation_duration = time.time() - start_time
+            log_histogram("generate_answer_duration", answer_generation_duration, labels={"api_choice": api_choice})
+            log_counter("generate_answer_success", labels={"api_choice": api_choice})
+            return summarize_with_huggingface(config['API']['huggingface_api_key'], prompt, "")
+
+        elif api_choice == "DeepSeek":
+            from App_Function_Libraries.Summarization.Summarization_General_Lib import summarize_with_deepseek
+            answer_generation_duration = time.time() - start_time
+            log_histogram("generate_answer_duration", answer_generation_duration, labels={"api_choice": api_choice})
+            log_counter("generate_answer_success", labels={"api_choice": api_choice})
+            return summarize_with_deepseek(config['API']['deepseek_api_key'], prompt, "")
+
+        elif api_choice == "Mistral":
+            from App_Function_Libraries.Summarization.Summarization_General_Lib import summarize_with_mistral
+            answer_generation_duration = time.time() - start_time
+            log_histogram("generate_answer_duration", answer_generation_duration, labels={"api_choice": api_choice})
+            log_counter("generate_answer_success", labels={"api_choice": api_choice})
+            return summarize_with_mistral(config['API']['mistral_api_key'], prompt, "")
+
+        # Local LLM APIs
+        elif api_choice == "Local-LLM":
+            from App_Function_Libraries.Summarization.Local_Summarization_Lib import summarize_with_local_llm
+            answer_generation_duration = time.time() - start_time
+            log_histogram("generate_answer_duration", answer_generation_duration, labels={"api_choice": api_choice})
+            log_counter("generate_answer_success", labels={"api_choice": api_choice})
+            # FIXME
+            return summarize_with_local_llm(config['Local-API']['local_llm_path'], prompt, "")
+
+        elif api_choice == "Llama.cpp":
+            from App_Function_Libraries.Summarization.Local_Summarization_Lib import summarize_with_llama
+            answer_generation_duration = time.time() - start_time
+            log_histogram("generate_answer_duration", answer_generation_duration, labels={"api_choice": api_choice})
+            log_counter("generate_answer_success", labels={"api_choice": api_choice})
+            return summarize_with_llama(prompt, "", config['Local-API']['llama_api_key'], None, None)
+        elif api_choice == "Kobold":
+            from App_Function_Libraries.Summarization.Local_Summarization_Lib import summarize_with_kobold
+            answer_generation_duration = time.time() - start_time
+            log_histogram("generate_answer_duration", answer_generation_duration, labels={"api_choice": api_choice})
+            log_counter("generate_answer_success", labels={"api_choice": api_choice})
+            return summarize_with_kobold(prompt, config['Local-API']['kobold_api_key'], "", system_message=None, temp=None)
+
+        elif api_choice == "Ooba":
+            from App_Function_Libraries.Summarization.Local_Summarization_Lib import summarize_with_oobabooga
+            answer_generation_duration = time.time() - start_time
+            log_histogram("generate_answer_duration", answer_generation_duration, labels={"api_choice": api_choice})
+            log_counter("generate_answer_success", labels={"api_choice": api_choice})
+            return summarize_with_oobabooga(prompt, config['Local-API']['ooba_api_key'], custom_prompt="", system_message=None, temp=None)
+
+        elif api_choice == "TabbyAPI":
+            from App_Function_Libraries.Summarization.Local_Summarization_Lib import summarize_with_tabbyapi
+            answer_generation_duration = time.time() - start_time
+            log_histogram("generate_answer_duration", answer_generation_duration, labels={"api_choice": api_choice})
+            log_counter("generate_answer_success", labels={"api_choice": api_choice})
+            return summarize_with_tabbyapi(prompt, None, None, None, None, )
+
+        elif api_choice == "vLLM":
+            from App_Function_Libraries.Summarization.Local_Summarization_Lib import summarize_with_vllm
+            answer_generation_duration = time.time() - start_time
+            log_histogram("generate_answer_duration", answer_generation_duration, labels={"api_choice": api_choice})
+            log_counter("generate_answer_success", labels={"api_choice": api_choice})
+            return summarize_with_vllm(prompt, "", config['Local-API']['vllm_api_key'], None, None)
+
+        elif api_choice.lower() == "ollama":
+            from App_Function_Libraries.Summarization.Local_Summarization_Lib import summarize_with_ollama
+            answer_generation_duration = time.time() - start_time
+            log_histogram("generate_answer_duration", answer_generation_duration, labels={"api_choice": api_choice})
+            log_counter("generate_answer_success", labels={"api_choice": api_choice})
+            return summarize_with_ollama(prompt, "", config['Local-API']['ollama_api_IP'], config['Local-API']['ollama_api_key'], None, None, None)
+
+        elif api_choice.lower() == "custom_openai_api":
+            logging.debug(f"RAG Answer Gen: Trying with Custom_OpenAI API")
+            summary = summarize_with_custom_openai(prompt, "", config['API']['custom_openai_api_key'], None,
+                                                   None)
+        else:
+            log_counter("generate_answer_error", labels={"api_choice": api_choice, "error": str()})
+            raise ValueError(f"Unsupported API choice: {api_choice}")
+    except Exception as e:
+        log_counter("generate_answer_error", labels={"api_choice": api_choice, "error": str(e)})
+        logging.error(f"Error in generate_answer: {str(e)}")
+        return "An error occurred while generating the answer."
 
 def perform_vector_search(query: str, relevant_media_ids: List[str] = None) -> List[Dict[str, Any]]:
+    log_counter("perform_vector_search_attempt")
+    start_time = time.time()
     all_collections = chroma_client.list_collections()
     vector_results = []
-    for collection in all_collections:
-        collection_results = vector_search(collection.name, query, k=5)
-        filtered_results = [
-            result for result in collection_results
-            if relevant_media_ids is None or result['metadata'].get('media_id') in relevant_media_ids
-        ]
-        vector_results.extend(filtered_results)
-    return vector_results
-
+    try:
+        for collection in all_collections:
+            collection_results = vector_search(collection.name, query, k=5)
+            filtered_results = [
+                result for result in collection_results
+                if relevant_media_ids is None or result['metadata'].get('media_id') in relevant_media_ids
+            ]
+            vector_results.extend(filtered_results)
+        search_duration = time.time() - start_time
+        log_histogram("perform_vector_search_duration", search_duration)
+        log_counter("perform_vector_search_success", labels={"result_count": len(vector_results)})
+        return vector_results
+    except Exception as e:
+        log_counter("perform_vector_search_error", labels={"error": str(e)})
+        logging.error(f"Error in perform_vector_search: {str(e)}")
+        raise
 
 def perform_full_text_search(query: str, relevant_media_ids: List[str] = None) -> List[Dict[str, Any]]:
-    fts_results = search_db(query, ["content"], "", page=1, results_per_page=5)
-    filtered_fts_results = [
-        {
-            "content": result['content'],
-            "metadata": {"media_id": result['id']}
-        }
-        for result in fts_results
-        if relevant_media_ids is None or result['id'] in relevant_media_ids
-    ]
-    return filtered_fts_results
+    log_counter("perform_full_text_search_attempt")
+    start_time = time.time()
+    try:
+        fts_results = search_db(query, ["content"], "", page=1, results_per_page=5)
+        filtered_fts_results = [
+            {
+                "content": result['content'],
+                "metadata": {"media_id": result['id']}
+            }
+            for result in fts_results
+            if relevant_media_ids is None or result['id'] in relevant_media_ids
+        ]
+        search_duration = time.time() - start_time
+        log_histogram("perform_full_text_search_duration", search_duration)
+        log_counter("perform_full_text_search_success", labels={"result_count": len(filtered_fts_results)})
+        return filtered_fts_results
+    except Exception as e:
+        log_counter("perform_full_text_search_error", labels={"error": str(e)})
+        logging.error(f"Error in perform_full_text_search: {str(e)}")
+        raise
 
 
 def fetch_relevant_media_ids(keywords: List[str]) -> List[int]:
+    log_counter("fetch_relevant_media_ids_attempt", labels={"keyword_count": len(keywords)})
+    start_time = time.time()
     relevant_ids = set()
-    try:
-        for keyword in keywords:
+    for keyword in keywords:
+        try:
             media_ids = fetch_keywords_for_media(keyword)
             relevant_ids.update(media_ids)
-    except Exception as e:
-        logging.error(f"Error fetching relevant media IDs: {str(e)}")
+        except Exception as e:
+            log_counter("fetch_relevant_media_ids_error", labels={"error": str(e)})
+            logging.error(f"Error fetching relevant media IDs for keyword '{keyword}': {str(e)}")
+            # Continue processing other keywords
+
+    fetch_duration = time.time() - start_time
+    log_histogram("fetch_relevant_media_ids_duration", fetch_duration)
+    log_counter("fetch_relevant_media_ids_success", labels={"result_count": len(relevant_ids)})
     return list(relevant_ids)
 
 
 def filter_results_by_keywords(results: List[Dict[str, Any]], keywords: List[str]) -> List[Dict[str, Any]]:
+    log_counter("filter_results_by_keywords_attempt", labels={"result_count": len(results), "keyword_count": len(keywords)})
+    start_time = time.time()
     if not keywords:
         return results
 
@@ -283,6 +428,9 @@ def filter_results_by_keywords(results: List[Dict[str, Any]], keywords: List[str
         except Exception as e:
             logging.error(f"Error processing result: {result}. Error: {str(e)}")
 
+    filter_duration = time.time() - start_time
+    log_histogram("filter_results_by_keywords_duration", filter_duration)
+    log_counter("filter_results_by_keywords_success", labels={"filtered_count": len(filtered_results)})
     return filtered_results
 
 # FIXME: to be implememted
@@ -300,6 +448,173 @@ def extract_media_id_from_result(result: str) -> Optional[int]:
 ########################################################################################################################
 
 
+############################################################################################################
+#
+# Chat RAG
+
+def enhanced_rag_pipeline_chat(query: str, api_choice: str, character_id: int, keywords: Optional[str] = None) -> Dict[str, Any]:
+    """
+    Enhanced RAG pipeline tailored for the Character Chat tab.
+
+    Args:
+        query (str): The user's input query.
+        api_choice (str): The API to use for generating the response.
+        character_id (int): The ID of the character being interacted with.
+        keywords (Optional[str]): Comma-separated keywords to filter search results.
+
+    Returns:
+        Dict[str, Any]: Contains the generated answer and the context used.
+    """
+    log_counter("enhanced_rag_pipeline_chat_attempt", labels={"api_choice": api_choice, "character_id": character_id})
+    start_time = time.time()
+    try:
+        # Load embedding provider from config, or fallback to 'openai'
+        embedding_provider = config.get('Embeddings', 'provider', fallback='openai')
+        logging.debug(f"Using embedding provider: {embedding_provider}")
+
+        # Process keywords if provided
+        keyword_list = [k.strip().lower() for k in keywords.split(',')] if keywords else []
+        logging.debug(f"enhanced_rag_pipeline_chat - Keywords: {keyword_list}")
+
+        # Fetch relevant chat IDs based on character_id and keywords
+        if keyword_list:
+            relevant_chat_ids = fetch_keywords_for_chats(keyword_list)
+        else:
+            relevant_chat_ids = fetch_all_chat_ids(character_id)
+        logging.debug(f"enhanced_rag_pipeline_chat - Relevant chat IDs: {relevant_chat_ids}")
+
+        if not relevant_chat_ids:
+            logging.info(f"No chats found for the given keywords and character ID: {character_id}")
+            # Fallback to generating answer without context
+            answer = generate_answer(api_choice, "", query)
+            # Metrics
+            pipeline_duration = time.time() - start_time
+            log_histogram("enhanced_rag_pipeline_chat_duration", pipeline_duration, labels={"api_choice": api_choice})
+            log_counter("enhanced_rag_pipeline_chat_success",
+                        labels={"api_choice": api_choice, "character_id": character_id})
+            return {
+                "answer": answer,
+                "context": ""
+            }
+
+        # Perform vector search within the relevant chats
+        vector_results = perform_vector_search_chat(query, relevant_chat_ids)
+        logging.debug(f"enhanced_rag_pipeline_chat - Vector search results: {vector_results}")
+
+        # Perform full-text search within the relevant chats
+        fts_results = perform_full_text_search_chat(query, relevant_chat_ids)
+        logging.debug("enhanced_rag_pipeline_chat - Full-text search results:")
+        logging.debug("\n".join([str(item) for item in fts_results]))
+
+        # Combine results
+        all_results = vector_results + fts_results
+
+        apply_re_ranking = True
+        if apply_re_ranking:
+            logging.debug("enhanced_rag_pipeline_chat - Applying Re-Ranking")
+            ranker = Ranker()
+
+            # Prepare passages for re-ranking
+            passages = [{"id": i, "text": result['content']} for i, result in enumerate(all_results)]
+            rerank_request = RerankRequest(query=query, passages=passages)
+
+            # Rerank the results
+            reranked_results = ranker.rerank(rerank_request)
+
+            # Sort results based on the re-ranking score
+            reranked_results = sorted(reranked_results, key=lambda x: x['score'], reverse=True)
+
+            # Log reranked results
+            logging.debug(f"enhanced_rag_pipeline_chat - Reranked results: {reranked_results}")
+
+            # Update all_results based on reranking
+            all_results = [all_results[result['id']] for result in reranked_results]
+
+        # Extract context from top results (limit to top 10)
+        context = "\n".join([result['content'] for result in all_results[:10]])
+        logging.debug(f"Context length: {len(context)}")
+        logging.debug(f"Context: {context[:200]}")  # Log only the first 200 characters for brevity
+
+        # Generate answer using the selected API
+        answer = generate_answer(api_choice, context, query)
+
+        if not all_results:
+            logging.info(f"No results found. Query: {query}, Keywords: {keywords}")
+            return {
+                "answer": "No relevant information based on your query and keywords were found in the database. Your query has been directly passed to the LLM, and here is its answer: \n\n" + answer,
+                "context": "No relevant information based on your query and keywords were found in the database. The only context used was your query: \n\n" + query
+            }
+
+        return {
+            "answer": answer,
+            "context": context
+        }
+
+    except Exception as e:
+        log_counter("enhanced_rag_pipeline_chat_error", labels={"api_choice": api_choice, "character_id": character_id, "error": str(e)})
+        logging.error(f"Error in enhanced_rag_pipeline_chat: {str(e)}")
+        return {
+            "answer": "An error occurred while processing your request.",
+            "context": ""
+        }
+
+
+def fetch_relevant_chat_ids(character_id: int, keywords: List[str]) -> List[int]:
+    """
+    Fetch chat IDs associated with a character and filtered by keywords.
+
+    Args:
+        character_id (int): The ID of the character.
+        keywords (List[str]): List of keywords to filter chats.
+
+    Returns:
+        List[int]: List of relevant chat IDs.
+    """
+    log_counter("fetch_relevant_chat_ids_attempt", labels={"character_id": character_id, "keyword_count": len(keywords)})
+    start_time = time.time()
+    relevant_ids = set()
+    try:
+        media_ids = fetch_keywords_for_chats(keywords)
+        fetch_duration = time.time() - start_time
+        log_histogram("fetch_relevant_chat_ids_duration", fetch_duration)
+        log_counter("fetch_relevant_chat_ids_success",
+                    labels={"character_id": character_id, "result_count": len(relevant_ids)})
+        relevant_ids.update(media_ids)
+        return list(relevant_ids)
+    except Exception as e:
+        log_counter("fetch_relevant_chat_ids_error", labels={"character_id": character_id, "error": str(e)})
+        logging.error(f"Error fetching relevant chat IDs: {str(e)}")
+        return []
+
+
+def fetch_all_chat_ids(character_id: int) -> List[int]:
+    """
+    Fetch all chat IDs associated with a specific character.
+
+    Args:
+        character_id (int): The ID of the character.
+
+    Returns:
+        List[int]: List of all chat IDs for the character.
+    """
+    log_counter("fetch_all_chat_ids_attempt", labels={"character_id": character_id})
+    start_time = time.time()
+    try:
+        chats = get_character_chats(character_id=character_id)
+        chat_ids = [chat['id'] for chat in chats]
+        fetch_duration = time.time() - start_time
+        log_histogram("fetch_all_chat_ids_duration", fetch_duration)
+        log_counter("fetch_all_chat_ids_success", labels={"character_id": character_id, "chat_count": len(chat_ids)})
+        return chat_ids
+    except Exception as e:
+        log_counter("fetch_all_chat_ids_error", labels={"character_id": character_id, "error": str(e)})
+        logging.error(f"Error fetching all chat IDs for character {character_id}: {str(e)}")
+        return []
+
+#
+# End of Chat RAG
+############################################################################################################
+
 # Function to preprocess and store all existing content in the database
 # def preprocess_all_content(database, create_contextualized=True, api_name="gpt-3.5-turbo"):
 #     unprocessed_media = get_unprocessed_media()

App_Function_Libraries/RAG/RAG_Persona_Chat.py

@@ -0,0 +1,103 @@
+# RAG_Persona_Chat.py
+# Description: Functions for RAG Persona Chat
+#
+# Imports
+import logging
+from typing import List, Dict, Any, Tuple
+#
+# External Imports
+#
+# Local Imports
+from App_Function_Libraries.RAG.Embeddings_Create import create_embedding, embedding_provider, embedding_model, \
+    embedding_api_url
+from App_Function_Libraries.RAG.ChromaDB_Library import chroma_client, store_in_chroma
+#
+#######################################################################################################################
+#
+# RAG Chat Embeddings
+
+def perform_vector_search_chat(query: str, relevant_chat_ids: List[int], k: int = 10) -> List[Dict[str, Any]]:
+    """
+    Perform a vector search within the specified chat IDs.
+
+    Args:
+        query (str): The user's query.
+        relevant_chat_ids (List[int]): List of chat IDs to search within.
+        k (int): Number of top results to retrieve.
+
+    Returns:
+        List[Dict[str, Any]]: List of search results with content and metadata.
+    """
+    try:
+        # Convert chat IDs to unique identifiers used in ChromaDB
+        chat_ids = [f"chat_{chat_id}" for chat_id in relevant_chat_ids]
+
+        # Define the collection name for chat embeddings
+        collection_name = "all_chat_embeddings"  # Ensure this collection exists and contains chat embeddings
+
+        # Generate the query embedding
+        query_embedding = create_embedding(query, embedding_provider, embedding_model, embedding_api_url)
+
+        # Get the collection
+        collection = chroma_client.get_collection(name=collection_name)
+
+        # Perform the vector search
+        results = collection.query(
+            query_embeddings=[query_embedding],
+            where={"id": {"$in": chat_ids}},  # Assuming 'id' is stored as document IDs
+            n_results=k,
+            include=["documents", "metadatas"]
+        )
+
+        # Process results
+        search_results = []
+        for doc, meta in zip(results['documents'][0], results['metadatas'][0]):
+            search_results.append({
+                "content": doc,
+                "metadata": meta
+            })
+
+        return search_results
+    except Exception as e:
+        logging.error(f"Error in perform_vector_search_chat: {e}")
+        return []
+
+
+def embed_and_store_chat(chat_id: int, chat_history: List[Tuple[str, str]], conversation_name: str):
+    """
+    Embed and store chat messages in ChromaDB.
+
+    Args:
+        chat_id (int): The ID of the chat.
+        chat_history (List[Tuple[str, str]]): List of (user_message, bot_response) tuples.
+        conversation_name (str): The name of the conversation.
+    """
+    try:
+        for idx, (user_msg, bot_msg) in enumerate(chat_history, 1):
+            # Combine user and bot messages for context
+            combined_content = f"User: {user_msg}\nBot: {bot_msg}"
+
+            # Create embedding
+            embedding = create_embedding(combined_content, embedding_provider, embedding_model, embedding_api_url)
+
+            # Unique identifier for ChromaDB
+            document_id = f"chat_{chat_id}_msg_{idx}"
+
+            # Metadata with chat_id
+            metadata = {"chat_id": chat_id, "message_index": idx, "conversation_name": conversation_name}
+
+            # Store in ChromaDB
+            store_in_chroma(
+                collection_name="all_chat_embeddings",
+                texts=[combined_content],
+                embeddings=[embedding],
+                ids=[document_id],
+                metadatas=[metadata]
+            )
+            logging.debug(f"Stored chat message {idx} of chat ID {chat_id} in ChromaDB.")
+    except Exception as e:
+        logging.error(f"Error embedding and storing chat ID {chat_id}: {e}")
+
+#
+# End of RAG_Persona_Chat.py
+#######################################################################################################################

App_Function_Libraries/RAG/RAG_QA_Chat.py

@@ -1,84 +1,137 @@
-# Podcast_tab.py
-# Description: Gradio UI for ingesting podcasts into the database
-#
-# Imports
-#
-#
-# External Imports
-import json
-import logging
-import tempfile
-from typing import List, Tuple, IO, Union
-#
-# Local Imports
-from App_Function_Libraries.DB.DB_Manager import db, search_db, DatabaseError, get_media_content
-from App_Function_Libraries.RAG.RAG_Library_2 import generate_answer
-#
-########################################################################################################################
-#
-# Functions:
-
-def rag_qa_chat(message: str, history: List[Tuple[str, str]], context: Union[str, IO[str]], api_choice: str) -> Tuple[List[Tuple[str, str]], str]:
-    try:
-        # Prepare the context based on the selected source
-        if hasattr(context, 'read'):
-            # Handle uploaded file
-            context_text = context.read()
-            if isinstance(context_text, bytes):
-                context_text = context_text.decode('utf-8')
-        elif isinstance(context, str) and context.startswith("media_id:"):
-            # Handle existing file or search result
-            media_id = int(context.split(":")[1])
-            context_text = get_media_content(media_id)  # Implement this function to fetch content from the database
-        else:
-            context_text = str(context)
-
-        # Prepare the full context including chat history
-        full_context = "\n".join([f"Human: {h[0]}\nAI: {h[1]}" for h in history])
-        full_context += f"\n\nContext: {context_text}\n\nHuman: {message}\nAI:"
-
-        # Generate response using the selected API
-        response = generate_answer(api_choice, full_context, message)
-
-        # Update history
-        history.append((message, response))
-
-        return history, ""
-    except DatabaseError as e:
-        logging.error(f"Database error in rag_qa_chat: {str(e)}")
-        return history, f"An error occurred while accessing the database: {str(e)}"
-    except Exception as e:
-        logging.error(f"Unexpected error in rag_qa_chat: {str(e)}")
-        return history, f"An unexpected error occurred: {str(e)}"
-
-
-
-def save_chat_history(history: List[Tuple[str, str]]) -> str:
-    # Save chat history to a file
-    with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.json') as temp_file:
-        json.dump(history, temp_file)
-        return temp_file.name
-
-
-def load_chat_history(file: IO[str]) -> List[Tuple[str, str]]:
-    # Load chat history from a file
-    return json.load(file)
-
-
-def search_database(query: str) -> List[Tuple[int, str]]:
-    # Implement database search functionality
-    results = search_db(query, ["title", "content"], "", page=1, results_per_page=10)
-    return [(result['id'], result['title']) for result in results]
-
-
-def get_existing_files() -> List[Tuple[int, str]]:
-    # Fetch list of existing files from the database
-    with db.get_connection() as conn:
-        cursor = conn.cursor()
-        cursor.execute("SELECT id, title FROM Media ORDER BY title")
-        return cursor.fetchall()
-
-
-#
-# End of RAG_QA_Chat.py
-########################################################################################################################
+# RAG_QA_Chat.py
+# Description: Functions supporting the RAG QA Chat functionality
+#
+# Imports
+#
+#
+# External Imports
+import json
+import logging
+import tempfile
+import time
+from typing import List, Tuple, IO, Union
+#
+# Local Imports
+from App_Function_Libraries.DB.DB_Manager import db, search_db, DatabaseError, get_media_content
+from App_Function_Libraries.RAG.RAG_Library_2 import generate_answer
+from App_Function_Libraries.Metrics.metrics_logger import log_counter, log_histogram
+#
+########################################################################################################################
+#
+# Functions:
+
+def rag_qa_chat(message: str, history: List[Tuple[str, str]], context: Union[str, IO[str]], api_choice: str) -> Tuple[List[Tuple[str, str]], str]:
+    log_counter("rag_qa_chat_attempt", labels={"api_choice": api_choice})
+    start_time = time.time()
+    try:
+        # Prepare the context based on the selected source
+        if hasattr(context, 'read'):
+            # Handle uploaded file
+            context_text = context.read()
+            if isinstance(context_text, bytes):
+                context_text = context_text.decode('utf-8')
+            log_counter("rag_qa_chat_uploaded_file")
+        elif isinstance(context, str) and context.startswith("media_id:"):
+            # Handle existing file or search result
+            media_id = int(context.split(":")[1])
+            context_text = get_media_content(media_id)
+            log_counter("rag_qa_chat_existing_media", labels={"media_id": media_id})
+        else:
+            context_text = str(context)
+            log_counter("rag_qa_chat_string_context")
+
+        # Prepare the full context including chat history
+        full_context = "\n".join([f"Human: {h[0]}\nAI: {h[1]}" for h in history])
+        full_context += f"\n\nContext: {context_text}\n\nHuman: {message}\nAI:"
+
+        # Generate response using the selected API
+        response = generate_answer(api_choice, full_context, message)
+
+        # Update history
+        history.append((message, response))
+
+        chat_duration = time.time() - start_time
+        log_histogram("rag_qa_chat_duration", chat_duration, labels={"api_choice": api_choice})
+        log_counter("rag_qa_chat_success", labels={"api_choice": api_choice})
+
+        return history, ""
+    except DatabaseError as e:
+        log_counter("rag_qa_chat_database_error", labels={"error": str(e)})
+        logging.error(f"Database error in rag_qa_chat: {str(e)}")
+        return history, f"An error occurred while accessing the database: {str(e)}"
+    except Exception as e:
+        log_counter("rag_qa_chat_unexpected_error", labels={"error": str(e)})
+        logging.error(f"Unexpected error in rag_qa_chat: {str(e)}")
+        return history, f"An unexpected error occurred: {str(e)}"
+
+
+
+def save_chat_history(history: List[Tuple[str, str]]) -> str:
+    # Save chat history to a file
+    log_counter("save_chat_history_attempt")
+    start_time = time.time()
+    try:
+        with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.json') as temp_file:
+            json.dump(history, temp_file)
+            save_duration = time.time() - start_time
+            log_histogram("save_chat_history_duration", save_duration)
+            log_counter("save_chat_history_success")
+            return temp_file.name
+    except Exception as e:
+        log_counter("save_chat_history_error", labels={"error": str(e)})
+        logging.error(f"Error saving chat history: {str(e)}")
+        raise
+
+
+def load_chat_history(file: IO[str]) -> List[Tuple[str, str]]:
+    log_counter("load_chat_history_attempt")
+    start_time = time.time()
+    try:
+        # Load chat history from a file
+        history = json.load(file)
+        load_duration = time.time() - start_time
+        log_histogram("load_chat_history_duration", load_duration)
+        log_counter("load_chat_history_success")
+        return history
+    except Exception as e:
+        log_counter("load_chat_history_error", labels={"error": str(e)})
+        logging.error(f"Error loading chat history: {str(e)}")
+        raise
+
+def search_database(query: str) -> List[Tuple[int, str]]:
+    try:
+        log_counter("search_database_attempt")
+        start_time = time.time()
+        # Implement database search functionality
+        results = search_db(query, ["title", "content"], "", page=1, results_per_page=10)
+        search_duration = time.time() - start_time
+        log_histogram("search_database_duration", search_duration)
+        log_counter("search_database_success", labels={"result_count": len(results)})
+        return [(result['id'], result['title']) for result in results]
+    except Exception as e:
+        log_counter("search_database_error", labels={"error": str(e)})
+        logging.error(f"Error searching database: {str(e)}")
+        raise
+
+
+def get_existing_files() -> List[Tuple[int, str]]:
+    log_counter("get_existing_files_attempt")
+    start_time = time.time()
+    try:
+        # Fetch list of existing files from the database
+        with db.get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute("SELECT id, title FROM Media ORDER BY title")
+            results = cursor.fetchall()
+        fetch_duration = time.time() - start_time
+        log_histogram("get_existing_files_duration", fetch_duration)
+        log_counter("get_existing_files_success", labels={"file_count": len(results)})
+        return results
+    except Exception as e:
+        log_counter("get_existing_files_error", labels={"error": str(e)})
+        logging.error(f"Error fetching existing files: {str(e)}")
+        raise
+
+#
+# End of RAG_QA_Chat.py
+########################################################################################################################

App_Function_Libraries/RAG/eval_Chroma_Embeddings.py

@@ -0,0 +1,133 @@
+# eval_Chroma_Embeddings.py
+# Description: This script is used to evaluate the embeddings and chunking process for the ChromaDB model.
+#
+# Imports
+import io
+from typing import List
+#
+# External Imports
+from chromadb import Documents, EmbeddingFunction, Embeddings
+from chromadb.utils import embedding_functions
+from chunking_evaluation import BaseChunker, rigorous_document_search
+from chunking_evaluation import BaseChunker, GeneralEvaluation
+from chunking_evaluation.evaluation_framework.base_evaluation import BaseEvaluation
+
+#
+# Local Imports
+from App_Function_Libraries.Chunk_Lib import improved_chunking_process
+from App_Function_Libraries.RAG.ChromaDB_Library import embedding_model, embedding_api_url
+from App_Function_Libraries.RAG.Embeddings_Create import create_embeddings_batch, embedding_provider
+from App_Function_Libraries.Utils.Utils import load_comprehensive_config
+#
+########################################################################################################################
+#
+# Functions:
+import chardet
+# FIXME
+
+
+def detect_file_encoding(file_path):
+    with open(file_path, 'rb') as file:
+        raw_data = file.read()
+        print(chardet.detect(raw_data)['encoding'])
+    return chardet.detect(raw_data)['encoding']
+
+
+class CustomEmbeddingFunction(EmbeddingFunction):
+    def __call__(self, input: Documents) -> Embeddings:
+        # Load config here
+        config = load_comprehensive_config()
+        embedding_provider = config.get('Embeddings', 'embedding_provider', fallback='openai')
+        embedding_model = config.get('Embeddings', 'embedding_model', fallback='text-embedding-3-small')
+        embedding_api_url = config.get('Embeddings', 'api_url', fallback='')
+
+        # Use your existing create_embeddings_batch function
+        embeddings = create_embeddings_batch(input, embedding_provider, embedding_model, embedding_api_url)
+        return embeddings
+
+
+class CustomChunker(BaseChunker):
+    def __init__(self, chunk_options):
+        self.chunk_options = chunk_options
+
+    def split_text(self, text: str) -> List[str]:
+        # Use your existing improved_chunking_process function
+        chunks = improved_chunking_process(text, self.chunk_options)
+        return [chunk['text'] for chunk in chunks]
+
+    def read_file(self, file_path: str) -> str:
+        encoding = detect_file_encoding(file_path)
+        with open(file_path, 'r', encoding=encoding) as file:
+            return file.read()
+
+def utf8_file_reader(file_path):
+    with io.open(file_path, 'r', encoding='utf-8') as file:
+        return file.read()
+
+
+class CustomEvaluation(BaseEvaluation):
+    def _get_chunks_and_metadata(self, splitter):
+        documents = []
+        metadatas = []
+        for corpus_id in self.corpus_list:
+            corpus_path = corpus_id
+            if self.corpora_id_paths is not None:
+                corpus_path = self.corpora_id_paths[corpus_id]
+
+            corpus = splitter.read_file(corpus_path)
+
+            current_documents = splitter.split_text(corpus)
+            current_metadatas = []
+            for document in current_documents:
+                try:
+                    _, start_index, end_index = rigorous_document_search(corpus, document)
+                except:
+                    print(f"Error in finding {document} in {corpus_id}")
+                    raise Exception(f"Error in finding {document} in {corpus_id}")
+                current_metadatas.append({"start_index": start_index, "end_index": end_index, "corpus_id": corpus_id})
+            documents.extend(current_documents)
+            metadatas.extend(current_metadatas)
+        return documents, metadatas
+
+
+# Instantiate your custom chunker
+chunk_options = {
+    'method': 'words',
+    'max_size': 400,
+    'overlap': 200,
+    'adaptive': False,
+    'multi_level': False,
+    'language': 'english'
+}
+custom_chunker = CustomChunker(chunk_options)
+
+# Instantiate your custom embedding function
+custom_ef = CustomEmbeddingFunction()
+
+
+# Evaluate the embedding function
+
+# Evaluate the chunker
+evaluation = GeneralEvaluation()
+import chardet
+
+def smart_file_reader(file_path):
+    encoding = detect_file_encoding(file_path)
+    with io.open(file_path, 'r', encoding=encoding) as file:
+        return file.read()
+
+# Set the custom file reader
+#evaluation._file_reader = smart_file_reader
+
+
+# Generate Embedding results
+embedding_results = evaluation.run(custom_chunker, custom_ef)
+print(f"Embedding Results:\n\t{embedding_results}")
+
+# Generate Chunking results
+chunk_results = evaluation.run(custom_chunker, custom_ef)
+print(f"Chunking Results:\n\t{chunk_results}")
+
+#
+# End of File
+########################################################################################################################

App_Function_Libraries/Summarization/Local_Summarization_Lib.py

@@ -21,6 +21,7 @@
 import json
 import logging
 import os
+import time
 from typing import Union
 
 import requests
@@ -640,10 +641,19 @@ def summarize_with_vllm(
         return f"Error: Unexpected error during vLLM summarization - {str(e)}"
 
 
-# FIXME - update to be a summarize request
-def summarize_with_ollama(input_data, custom_prompt, api_key=None, temp=None, system_message=None, model=None, api_url="http://127.0.0.1:11434/api/generate",):
+def summarize_with_ollama(
+    input_data,
+    custom_prompt,
+    api_url="http://127.0.0.1:11434/v1/chat/completions",
+    api_key=None,
+    temp=None,
+    system_message=None,
+    model=None,
+    max_retries=5,
+    retry_delay=20
+):
     try:
-        logging.debug("ollama: Loading and validating configurations")
+        logging.debug("Ollama: Loading and validating configurations")
         loaded_config_data = load_and_log_configs()
         if loaded_config_data is None:
             logging.error("Failed to load configuration data")
@@ -661,7 +671,19 @@ def summarize_with_ollama(input_data, custom_prompt, api_key=None, temp=None, sy
                 else:
                     logging.warning("Ollama: No API key found in config file")
 
-        model = loaded_config_data['models']['ollama']
+            # Set model from parameter or config
+            if model is None:
+                model = loaded_config_data['models'].get('ollama')
+                if model is None:
+                    logging.error("Ollama: Model not found in config file")
+                    return "Ollama: Model not found in config file"
+
+            # Set api_url from parameter or config
+            if api_url is None:
+                api_url = loaded_config_data['local_api_ip'].get('ollama')
+                if api_url is None:
+                    logging.error("Ollama: API URL not found in config file")
+                    return "Ollama: API URL not found in config file"
 
         # Load transcript
         logging.debug("Ollama: Loading JSON data")
@@ -690,57 +712,92 @@ def summarize_with_ollama(input_data, custom_prompt, api_key=None, temp=None, sy
         else:
             raise ValueError("Ollama: Invalid input data format")
 
-        if custom_prompt is None:
-            custom_prompt = f"{summarizer_prompt}\n\n\n\n{text}"
-        else:
-            custom_prompt = f"{custom_prompt}\n\n\n\n{text}"
-
         headers = {
             'accept': 'application/json',
             'content-type': 'application/json',
         }
-        if len(ollama_api_key) > 5:
+        if ollama_api_key and len(ollama_api_key) > 5:
             headers['Authorization'] = f'Bearer {ollama_api_key}'
 
-        ollama_prompt = f"{custom_prompt} \n\n\n\n{text}"
-        if system_message is None:
-            system_message = "You are a helpful AI assistant."
-        logging.debug(f"llama: Prompt being sent is {ollama_prompt}")
+        ollama_prompt = f"{custom_prompt}\n\n{text}"
         if system_message is None:
             system_message = "You are a helpful AI assistant."
+        logging.debug(f"Ollama: Prompt being sent is: {ollama_prompt}")
 
-        data = {
+        data_payload = {
             "model": model,
             "messages": [
-                {"role": "system",
-                 "content": system_message
-                 },
-                {"role": "user",
-                 "content": ollama_prompt
-                 }
+                {
+                    "role": "system",
+                    "content": system_message
+                },
+                {
+                    "role": "user",
+                    "content": ollama_prompt
+                }
             ],
+            'temperature': temp
         }
 
-        logging.debug("Ollama: Submitting request to API endpoint")
-        print("Ollama: Submitting request to API endpoint")
-        response = requests.post(api_url, headers=headers, json=data)
-        response_data = response.json()
-        logging.debug("API Response Data: %s", response_data)
+        for attempt in range(1, max_retries + 1):
+            logging.debug("Ollama: Submitting request to API endpoint")
+            print("Ollama: Submitting request to API endpoint")
+            try:
+                response = requests.post(api_url, headers=headers, json=data_payload, timeout=30)
+                response.raise_for_status()  # Raises HTTPError for bad responses
+                response_data = response.json()
+            except requests.exceptions.Timeout:
+                logging.error("Ollama: Request timed out.")
+                return "Ollama: Request timed out."
+            except requests.exceptions.HTTPError as http_err:
+                logging.error(f"Ollama: HTTP error occurred: {http_err}")
+                return f"Ollama: HTTP error occurred: {http_err}"
+            except requests.exceptions.RequestException as req_err:
+                logging.error(f"Ollama: Request exception: {req_err}")
+                return f"Ollama: Request exception: {req_err}"
+            except json.JSONDecodeError:
+                logging.error("Ollama: Failed to decode JSON response")
+                return "Ollama: Failed to decode JSON response."
+            except Exception as e:
+                logging.error(f"Ollama: An unexpected error occurred: {str(e)}")
+                return f"Ollama: An unexpected error occurred: {str(e)}"
+
+            logging.debug(f"API Response Data: {response_data}")
 
-        if response.status_code == 200:
-            # if 'X' in response_data:
-            logging.debug(response_data)
-            summary = response_data['content'].strip()
-            logging.debug("Ollama: Summarization successful")
-            print("Summarization successful.")
-            return summary
-        else:
-            logging.error(f"Ollama: API request failed with status code {response.status_code}: {response.text}")
-            return f"Ollama: API request failed: {response.text}"
+            if response.status_code == 200:
+                # Inspect available keys
+                available_keys = list(response_data.keys())
+                logging.debug(f"Ollama: Available keys in response: {available_keys}")
+
+                # Attempt to retrieve 'response'
+                summary = None
+                if 'response' in response_data and response_data['response']:
+                    summary = response_data['response'].strip()
+                elif 'choices' in response_data and len(response_data['choices']) > 0:
+                    choice = response_data['choices'][0]
+                    if 'message' in choice and 'content' in choice['message']:
+                        summary = choice['message']['content'].strip()
+
+                if summary:
+                    logging.debug("Ollama: Chat request successful")
+                    print("\n\nChat request successful.")
+                    return summary
+                elif response_data.get('done_reason') == 'load':
+                    logging.warning(f"Ollama: Model is loading. Attempt {attempt} of {max_retries}. Retrying in {retry_delay} seconds...")
+                    time.sleep(retry_delay)
+                else:
+                    logging.error("Ollama: API response does not contain 'response' or 'choices'.")
+                    return "Ollama: API response does not contain 'response' or 'choices'."
+            else:
+                logging.error(f"Ollama: API request failed with status code {response.status_code}: {response.text}")
+                return f"Ollama: API request failed: {response.text}"
+
+        logging.error("Ollama: Maximum retry attempts reached. Model is still loading.")
+        return "Ollama: Maximum retry attempts reached. Model is still loading."
 
     except Exception as e:
-        logging.error("Ollama: Error in processing: %s", str(e))
-        return f"Ollama: Error occurred while processing summary with ollama: {str(e)}"
+        logging.error("\n\nOllama: Error in processing: %s", str(e))
+        return f"Ollama: Error occurred while processing summary with Ollama: {str(e)}"
 
 
 # FIXME - update to be a summarize request

App_Function_Libraries/Summarization/Summarization_General_Lib.py

@@ -73,7 +73,7 @@ def summarize(
         elif api_name.lower() == "mistral":
             return summarize_with_mistral(api_key, input_data, custom_prompt_arg, temp, system_message)
         elif api_name.lower() == "llama.cpp":
-            return summarize_with_llama(input_data, custom_prompt_arg, temp, system_message)
+            return summarize_with_llama(input_data, custom_prompt_arg, api_key, temp, system_message)
         elif api_name.lower() == "kobold":
             return summarize_with_kobold(input_data, api_key, custom_prompt_arg, temp, system_message)
         elif api_name.lower() == "ooba":
@@ -86,6 +86,10 @@ def summarize(
             return summarize_with_local_llm(input_data, custom_prompt_arg, temp, system_message)
         elif api_name.lower() == "huggingface":
             return summarize_with_huggingface(api_key, input_data, custom_prompt_arg, temp, )#system_message)
+        elif api_name.lower() == "custom-openai":
+            return summarize_with_custom_openai(api_key, input_data, custom_prompt_arg, temp, system_message)
+        elif api_name.lower() == "ollama":
+            return summarize_with_ollama(input_data, custom_prompt_arg, None, api_key, temp, system_message)
         else:
             return f"Error: Invalid API Name {api_name}"
 
@@ -1147,7 +1151,7 @@ def perform_transcription(video_path, offset, whisper_model, vad_filter, diarize
 
         return audio_file_path, diarized_segments
 
-    # Non-diarized transcription (existing functionality)
+    # Non-diarized transcription
     if os.path.exists(segments_json_path):
         logging.info(f"Segments file already exists: {segments_json_path}")
         try:

App_Function_Libraries/Web_Scraping/Article_Extractor_Lib.py

@@ -13,13 +13,14 @@
 ####################
 #
 # Import necessary libraries
+import json
 import logging
 # 3rd-Party Imports
 import asyncio
 import os
 import tempfile
 from datetime import datetime
-from typing import List, Dict
+from typing import List, Dict, Union
 from urllib.parse import urljoin, urlparse
 from xml.dom import minidom
 from playwright.async_api import async_playwright
@@ -376,6 +377,152 @@ def scrape_and_convert_with_filter(source: str, output_file: str, filter_functio
 
     logging.info(f"Scraped and filtered content saved to {output_file}")
 
+
+###################################################
+#
+# Bookmark Parsing Functions
+
+def parse_chromium_bookmarks(json_data: dict) -> Dict[str, Union[str, List[str]]]:
+    """
+    Parse Chromium-based browser bookmarks from JSON data.
+
+    :param json_data: The JSON data from the bookmarks file
+    :return: A dictionary with bookmark names as keys and URLs as values or lists of URLs if duplicates exist
+    """
+    bookmarks = {}
+
+    def recurse_bookmarks(nodes):
+        for node in nodes:
+            if node.get('type') == 'url':
+                name = node.get('name')
+                url = node.get('url')
+                if name and url:
+                    if name in bookmarks:
+                        if isinstance(bookmarks[name], list):
+                            bookmarks[name].append(url)
+                        else:
+                            bookmarks[name] = [bookmarks[name], url]
+                    else:
+                        bookmarks[name] = url
+            elif node.get('type') == 'folder' and 'children' in node:
+                recurse_bookmarks(node['children'])
+
+    # Chromium bookmarks have a 'roots' key
+    if 'roots' in json_data:
+        for root in json_data['roots'].values():
+            if 'children' in root:
+                recurse_bookmarks(root['children'])
+    else:
+        recurse_bookmarks(json_data.get('children', []))
+
+    return bookmarks
+
+
+def parse_firefox_bookmarks(html_content: str) -> Dict[str, Union[str, List[str]]]:
+    """
+    Parse Firefox bookmarks from HTML content.
+
+    :param html_content: The HTML content from the bookmarks file
+    :return: A dictionary with bookmark names as keys and URLs as values or lists of URLs if duplicates exist
+    """
+    bookmarks = {}
+    soup = BeautifulSoup(html_content, 'html.parser')
+
+    # Firefox stores bookmarks within <a> tags inside <dt>
+    for a in soup.find_all('a'):
+        name = a.get_text()
+        url = a.get('href')
+        if name and url:
+            if name in bookmarks:
+                if isinstance(bookmarks[name], list):
+                    bookmarks[name].append(url)
+                else:
+                    bookmarks[name] = [bookmarks[name], url]
+            else:
+                bookmarks[name] = url
+
+    return bookmarks
+
+
+def load_bookmarks(file_path: str) -> Dict[str, Union[str, List[str]]]:
+    """
+    Load bookmarks from a file (JSON for Chrome/Edge or HTML for Firefox).
+
+    :param file_path: Path to the bookmarks file
+    :return: A dictionary with bookmark names as keys and URLs as values or lists of URLs if duplicates exist
+    :raises ValueError: If the file format is unsupported or parsing fails
+    """
+    if not os.path.isfile(file_path):
+        logging.error(f"File '{file_path}' does not exist.")
+        raise FileNotFoundError(f"File '{file_path}' does not exist.")
+
+    _, ext = os.path.splitext(file_path)
+    ext = ext.lower()
+
+    if ext == '.json' or ext == '':
+        # Attempt to parse as JSON (Chrome/Edge)
+        try:
+            with open(file_path, 'r', encoding='utf-8') as f:
+                json_data = json.load(f)
+            return parse_chromium_bookmarks(json_data)
+        except json.JSONDecodeError:
+            logging.error("Failed to parse JSON. Ensure the file is a valid Chromium bookmarks JSON file.")
+            raise ValueError("Invalid JSON format for Chromium bookmarks.")
+    elif ext in ['.html', '.htm']:
+        # Parse as HTML (Firefox)
+        try:
+            with open(file_path, 'r', encoding='utf-8') as f:
+                html_content = f.read()
+            return parse_firefox_bookmarks(html_content)
+        except Exception as e:
+            logging.error(f"Failed to parse HTML bookmarks: {e}")
+            raise ValueError(f"Failed to parse HTML bookmarks: {e}")
+    else:
+        logging.error("Unsupported file format. Please provide a JSON (Chrome/Edge) or HTML (Firefox) bookmarks file.")
+        raise ValueError("Unsupported file format for bookmarks.")
+
+
+def collect_bookmarks(file_path: str) -> Dict[str, Union[str, List[str]]]:
+    """
+    Collect bookmarks from the provided bookmarks file and return a dictionary.
+
+    :param file_path: Path to the bookmarks file
+    :return: Dictionary with bookmark names as keys and URLs as values or lists of URLs if duplicates exist
+    """
+    try:
+        bookmarks = load_bookmarks(file_path)
+        logging.info(f"Successfully loaded {len(bookmarks)} bookmarks from '{file_path}'.")
+        return bookmarks
+    except (FileNotFoundError, ValueError) as e:
+        logging.error(f"Error loading bookmarks: {e}")
+        return {}
+
+# Usage:
+# from Article_Extractor_Lib import collect_bookmarks
+#
+# # Path to your bookmarks file
+# # For Chrome or Edge (JSON format)
+# chromium_bookmarks_path = "/path/to/Bookmarks"
+#
+# # For Firefox (HTML format)
+# firefox_bookmarks_path = "/path/to/bookmarks.html"
+#
+# # Collect bookmarks from Chromium-based browser
+# chromium_bookmarks = collect_bookmarks(chromium_bookmarks_path)
+# print("Chromium Bookmarks:")
+# for name, url in chromium_bookmarks.items():
+#     print(f"{name}: {url}")
+#
+# # Collect bookmarks from Firefox
+# firefox_bookmarks = collect_bookmarks(firefox_bookmarks_path)
+# print("\nFirefox Bookmarks:")
+# for name, url in firefox_bookmarks.items():
+#     print(f"{name}: {url}")
+
+#
+# End of Bookmarking Parsing Functions
+#####################################################################
+
 #
 #
 #######################################################################################################################

App_Function_Libraries/Web_Scraping/Article_Summarization_Lib.py

@@ -165,7 +165,7 @@ def scrape_and_summarize(url, custom_prompt_arg, api_name, api_key, keywords, cu
                 elif api_name.lower() == "ollama":
                     logging.debug(f"MAIN: Trying to summarize with OLLAMA")
                     # def summarize_with_ollama(input_data, api_key, custom_prompt, api_url):
-                    summary = summarize_with_ollama(json_file_path, article_custom_prompt, api_key, None, system_message, None)
+                    summary = summarize_with_ollama(json_file_path, article_custom_prompt, None, api_key, None, system_message, None)
 
                 elif api_name == "custom_openai_api":
                     logging.debug(f"MAIN: Trying to summarize with Custom_OpenAI API")

App_Function_Libraries/html_to_markdown/ast_utils.py

@@ -0,0 +1,59 @@
+# html_to_markdown/ast_utils.py
+
+from typing import Callable, Optional, List, Union
+from s_types import SemanticMarkdownAST
+
+def find_in_ast(ast: Union[SemanticMarkdownAST, List[SemanticMarkdownAST]], predicate: Callable[[SemanticMarkdownAST], bool]) -> Optional[SemanticMarkdownAST]:
+    if isinstance(ast, list):
+        for node in ast:
+            result = find_in_ast(node, predicate)
+            if result:
+                return result
+    else:
+        if predicate(ast):
+            return ast
+        # Recursively search based on node type
+        if hasattr(ast, 'content'):
+            content = ast.content
+            if isinstance(content, list):
+                result = find_in_ast(content, predicate)
+                if result:
+                    return result
+            elif isinstance(content, SemanticMarkdownAST):
+                result = find_in_ast(content, predicate)
+                if result:
+                    return result
+        if hasattr(ast, 'items'):
+            for item in ast.items:
+                result = find_in_ast(item, predicate)
+                if result:
+                    return result
+        if hasattr(ast, 'rows'):
+            for row in ast.rows:
+                result = find_in_ast(row, predicate)
+                if result:
+                    return result
+    return None
+
+def find_all_in_ast(ast: Union[SemanticMarkdownAST, List[SemanticMarkdownAST]], predicate: Callable[[SemanticMarkdownAST], bool]) -> List[SemanticMarkdownAST]:
+    results = []
+    if isinstance(ast, list):
+        for node in ast:
+            results.extend(find_all_in_ast(node, predicate))
+    else:
+        if predicate(ast):
+            results.append(ast)
+        # Recursively search based on node type
+        if hasattr(ast, 'content'):
+            content = ast.content
+            if isinstance(content, list):
+                results.extend(find_all_in_ast(content, predicate))
+            elif isinstance(content, SemanticMarkdownAST):
+                results.extend(find_all_in_ast(content, predicate))
+        if hasattr(ast, 'items'):
+            for item in ast.items:
+                results.extend(find_all_in_ast(item, predicate))
+        if hasattr(ast, 'rows'):
+            for row in ast.rows:
+                results.extend(find_all_in_ast(row, predicate))
+    return results

App_Function_Libraries/html_to_markdown/conversion_options.py

@@ -0,0 +1,21 @@
+# html_to_markdown/conversion_options.py
+
+from typing import Callable, Optional, Union, Dict, Any, List
+from dataclasses import dataclass, field
+
+from s_types import SemanticMarkdownAST, CustomNode
+
+@dataclass
+class ConversionOptions:
+    website_domain: Optional[str] = None
+    extract_main_content: bool = False
+    refify_urls: bool = False
+    url_map: Dict[str, str] = field(default_factory=dict)
+    debug: bool = False
+    override_dom_parser: Optional[Callable[[str], Any]] = None  # Placeholder for DOMParser override
+    enable_table_column_tracking: bool = False
+    override_element_processing: Optional[Callable[[Any, 'ConversionOptions', int], Optional[List[SemanticMarkdownAST]]]] = None
+    process_unhandled_element: Optional[Callable[[Any, 'ConversionOptions', int], Optional[List[SemanticMarkdownAST]]]] = None
+    override_node_renderer: Optional[Callable[[SemanticMarkdownAST, 'ConversionOptions', int], Optional[str]]] = None
+    render_custom_node: Optional[Callable[[CustomNode, 'ConversionOptions', int], Optional[str]]] = None
+    include_meta_data: Union[str, bool] = False  # 'basic', 'extended', or False

App_Function_Libraries/html_to_markdown/dom_utils.py

@@ -0,0 +1,140 @@
+# html_to_markdown/dom_utils.py
+
+from bs4 import BeautifulSoup, Tag
+from typing import Optional
+import logging
+
+from conversion_options import ConversionOptions
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+def find_main_content(soup: BeautifulSoup, options: ConversionOptions) -> Tag:
+    logger.debug("Entering find_main_content function")
+
+    main_element = soup.find('main')
+    if main_element:
+        logger.debug("Existing <main> element found")
+        return main_element
+
+    logger.debug("No <main> element found. Detecting main content.")
+    if not soup.body:
+        logger.debug("No body element found, returning the entire document")
+        return soup
+
+    return detect_main_content(soup.body, options)
+
+def wrap_main_content(main_content: Tag, soup: BeautifulSoup):
+    if main_content.name.lower() != 'main':
+        logger.debug("Wrapping main content in <main> element")
+        main_element = soup.new_tag('main')
+        main_content.wrap(main_element)
+        main_element['id'] = 'detected-main-content'
+        logger.debug("Main content wrapped successfully")
+    else:
+        logger.debug("Main content already wrapped")
+
+def detect_main_content(element: Tag, options: ConversionOptions) -> Tag:
+    candidates = []
+    min_score = 20
+    logger.debug(f"Collecting candidates with minimum score: {min_score}")
+    collect_candidates(element, candidates, min_score, options)
+
+    logger.debug(f"Total candidates found: {len(candidates)}")
+
+    if not candidates:
+        logger.debug("No suitable candidates found, returning root element")
+        return element
+
+    # Sort candidates by score descending
+    candidates.sort(key=lambda x: calculate_score(x, options), reverse=True)
+    logger.debug("Candidates sorted by score")
+
+    best_candidate = candidates[0]
+    for candidate in candidates[1:]:
+        if not any(other.contains(candidate) for other in candidates):
+            if calculate_score(candidate, options) > calculate_score(best_candidate, options):
+                best_candidate = candidate
+                logger.debug(f"New best independent candidate found: {element_to_string(best_candidate)}")
+
+    logger.debug(f"Final main content candidate: {element_to_string(best_candidate)}")
+    return best_candidate
+
+def element_to_string(element: Optional[Tag]) -> str:
+    if not element:
+        return 'No element'
+    classes = '.'.join(element.get('class', []))
+    return f"{element.name}#{element.get('id', 'no-id')}.{classes}"
+
+def collect_candidates(element: Tag, candidates: list, min_score: int, options: ConversionOptions):
+    score = calculate_score(element, options)
+    if score >= min_score:
+        candidates.append(element)
+        logger.debug(f"Candidate found: {element_to_string(element)}, score: {score}")
+
+    for child in element.find_all(recursive=False):
+        collect_candidates(child, candidates, min_score, options)
+
+def calculate_score(element: Tag, options: ConversionOptions) -> int:
+    score = 0
+    score_log = []
+
+    # High impact attributes
+    high_impact_attributes = ['article', 'content', 'main-container', 'main', 'main-content']
+    for attr in high_impact_attributes:
+        if 'class' in element.attrs and attr in element['class']:
+            score += 10
+            score_log.append(f"High impact attribute found: {attr}, score increased by 10")
+        if 'id' in element.attrs and attr in element['id']:
+            score += 10
+            score_log.append(f"High impact ID found: {attr}, score increased by 10")
+
+    # High impact tags
+    high_impact_tags = ['article', 'main', 'section']
+    if element.name.lower() in high_impact_tags:
+        score += 5
+        score_log.append(f"High impact tag found: {element.name}, score increased by 5")
+
+    # Paragraph count
+    paragraph_count = len(element.find_all('p'))
+    paragraph_score = min(paragraph_count, 5)
+    if paragraph_score > 0:
+        score += paragraph_score
+        score_log.append(f"Paragraph count: {paragraph_count}, score increased by {paragraph_score}")
+
+    # Text content length
+    text_content_length = len(element.get_text(strip=True))
+    if text_content_length > 200:
+        text_score = min(text_content_length // 200, 5)
+        score += text_score
+        score_log.append(f"Text content length: {text_content_length}, score increased by {text_score}")
+
+    # Link density
+    link_density = calculate_link_density(element)
+    if link_density < 0.3:
+        score += 5
+        score_log.append(f"Link density: {link_density:.2f}, score increased by 5")
+
+    # Data attributes
+    if element.has_attr('data-main') or element.has_attr('data-content'):
+        score += 10
+        score_log.append("Data attribute for main content found, score increased by 10")
+
+    # Role attribute
+    if element.get('role') and 'main' in element.get('role'):
+        score += 10
+        score_log.append("Role attribute indicating main content found, score increased by 10")
+
+    if options.debug and score_log:
+        logger.debug(f"Scoring for {element_to_string(element)}:")
+        for log in score_log:
+            logger.debug(f"  {log}")
+        logger.debug(f"  Final score: {score}")
+
+    return score
+
+def calculate_link_density(element: Tag) -> float:
+    links = element.find_all('a')
+    link_length = sum(len(link.get_text(strip=True)) for link in links)
+    text_length = len(element.get_text(strip=True)) or 1  # Avoid division by zero
+    return link_length / text_length

App_Function_Libraries/html_to_markdown/html_to_markdown.py

@@ -0,0 +1,46 @@
+# html_to_markdown/html_to_markdown.py
+
+from bs4 import BeautifulSoup
+from typing import Optional
+
+from conversion_options import ConversionOptions
+from dom_utils import find_main_content, wrap_main_content
+from html_to_markdown_ast import html_to_markdown_ast
+from markdown_ast_to_string import markdown_ast_to_string
+from url_utils import refify_urls
+
+import logging
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+def convert_html_to_markdown(html: str, options: Optional[ConversionOptions] = None) -> str:
+    if options is None:
+        options = ConversionOptions()
+
+    if options.debug:
+        logger.setLevel(logging.DEBUG)
+
+    soup = BeautifulSoup(html, 'html.parser')
+
+    if options.extract_main_content:
+        main_content = find_main_content(soup, options)
+        if options.include_meta_data and soup.head and not main_content.find('head'):
+            # Reattach head for metadata extraction
+            new_html = f"<html>{soup.head}{main_content}</html>"
+            soup = BeautifulSoup(new_html, 'html.parser')
+            main_content = soup.html
+    else:
+        if options.include_meta_data and soup.head:
+            main_content = soup
+        else:
+            main_content = soup.body if soup.body else soup
+
+    markdown_ast = html_to_markdown_ast(main_content, options)
+
+    if options.refify_urls:
+        options.url_map = refify_urls(markdown_ast, options.url_map)
+
+    markdown_string = markdown_ast_to_string(markdown_ast, options)
+
+    return markdown_string

App_Function_Libraries/html_to_markdown/html_to_markdown_ast.py

@@ -0,0 +1,212 @@
+# html_to_markdown/html_to_markdown_ast.py
+
+from bs4 import BeautifulSoup, Tag, NavigableString
+from typing import List, Optional, Union
+
+from s_types import (
+    SemanticMarkdownAST, TextNode, BoldNode, ItalicNode, StrikethroughNode,
+    HeadingNode, LinkNode, ImageNode, VideoNode, ListNode, ListItemNode,
+    TableNode, TableRowNode, TableCellNode, CodeNode, BlockquoteNode,
+    SemanticHtmlNode, CustomNode, MetaDataNode
+)
+from conversion_options import ConversionOptions
+import logging
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+def escape_markdown_characters(text: str, is_inline_code: bool = False) -> str:
+    if is_inline_code or not text.strip():
+        return text
+    # Replace special characters
+    replacements = {
+        '\\': '\\\\',
+        '`': '\\`',
+        '*': '\\*',
+        '_': '\\_',
+        '{': '\\{',
+        '}': '\\}',
+        '[': '\\[',
+        ']': '\\]',
+        '(': '\\(',
+        ')': '\\)',
+        '#': '\\#',
+        '+': '\\+',
+        '-': '\\-',
+        '.': '\\.',
+        '!': '\\!',
+        '|': '\\|',
+    }
+    for char, escaped in replacements.items():
+        text = text.replace(char, escaped)
+    return text
+
+def html_to_markdown_ast(element: Tag, options: Optional[ConversionOptions] = None, indent_level: int = 0) -> List[SemanticMarkdownAST]:
+    if options is None:
+        options = ConversionOptions()
+
+    result: List[SemanticMarkdownAST] = []
+
+    for child in element.children:
+        if isinstance(child, NavigableString):
+            text_content = escape_markdown_characters(child.strip())
+            if text_content:
+                logger.debug(f"Text Node: '{text_content}'")
+                result.append(TextNode(content=child.strip()))
+        elif isinstance(child, Tag):
+            # Check for overridden element processing
+            if options.override_element_processing:
+                overridden = options.override_element_processing(child, options, indent_level)
+                if overridden:
+                    logger.debug(f"Element Processing Overridden: '{child.name}'")
+                    result.extend(overridden)
+                    continue
+
+            tag_name = child.name.lower()
+
+            if tag_name in ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']:
+                level = int(tag_name[1])
+                content = escape_markdown_characters(child.get_text(strip=True))
+                if content:
+                    logger.debug(f"Heading {level}: '{content}'")
+                    result.append(HeadingNode(level=level, content=content))
+            elif tag_name == 'p':
+                logger.debug("Paragraph")
+                result.extend(html_to_markdown_ast(child, options, indent_level))
+                # Add a new line after the paragraph
+                result.append(TextNode(content='\n\n'))
+            elif tag_name == 'a':
+                href = child.get('href', '#')
+                if href.startswith("data:image"):
+                    # Skip data URLs for images
+                    result.append(LinkNode(href='-', content=html_to_markdown_ast(child, options, indent_level)))
+                else:
+                    href = href
+                    if options.website_domain and href.startswith(options.website_domain):
+                        href = href[len(options.website_domain):]
+                    # Check if all children are text
+                    if all(isinstance(c, NavigableString) for c in child.children):
+                        content = [TextNode(content=child.get_text(strip=True))]
+                        result.append(LinkNode(href=href, content=content))
+                    else:
+                        content = html_to_markdown_ast(child, options, indent_level)
+                        result.append(LinkNode(href=href, content=content))
+            elif tag_name == 'img':
+                src = child.get('src', '')
+                alt = child.get('alt', '')
+                if src.startswith("data:image"):
+                    src = '-'
+                else:
+                    if options.website_domain and src.startswith(options.website_domain):
+                        src = src[len(options.website_domain):]
+                logger.debug(f"Image: src='{src}', alt='{alt}'")
+                result.append(ImageNode(src=src, alt=alt))
+            elif tag_name == 'video':
+                src = child.get('src', '')
+                poster = child.get('poster', '')
+                controls = child.has_attr('controls')
+                logger.debug(f"Video: src='{src}', poster='{poster}', controls='{controls}'")
+                result.append(VideoNode(src=src, poster=poster, controls=controls))
+            elif tag_name in ['ul', 'ol']:
+                logger.debug(f"{'Unordered' if tag_name == 'ul' else 'Ordered'} List")
+                ordered = tag_name == 'ol'
+                items = []
+                for li in child.find_all('li', recursive=False):
+                    item_content = html_to_markdown_ast(li, options, indent_level + 1)
+                    items.append(ListItemNode(content=item_content))
+                result.append(ListNode(ordered=ordered, items=items))
+            elif tag_name == 'br':
+                logger.debug("Line Break")
+                result.append(TextNode(content='\n'))
+            elif tag_name == 'table':
+                logger.debug("Table")
+                table_node = TableNode()
+                rows = child.find_all('tr')
+                for row in rows:
+                    table_row = TableRowNode()
+                    cells = row.find_all(['th', 'td'])
+                    for cell in cells:
+                        colspan = int(cell.get('colspan', 1))
+                        rowspan = int(cell.get('rowspan', 1))
+                        cell_content = cell.get_text(strip=True)
+                        table_row.cells.append(TableCellNode(content=cell_content, colspan=colspan if colspan >1 else None,
+                                                            rowspan=rowspan if rowspan >1 else None))
+                    table_node.rows.append(table_row)
+                result.append(table_node)
+            elif tag_name == 'head' and options.include_meta_data:
+                meta_node = MetaDataNode(content={
+                    'standard': {},
+                    'openGraph': {},
+                    'twitter': {},
+                    'jsonLd': []
+                })
+                title = child.find('title')
+                if title:
+                    meta_node.content['standard']['title'] = title.get_text(strip=True)
+                meta_tags = child.find_all('meta')
+                non_semantic_tags = ["viewport", "referrer", "Content-Security-Policy"]
+                for meta in meta_tags:
+                    name = meta.get('name')
+                    prop = meta.get('property')
+                    content = meta.get('content', '')
+                    if prop and prop.startswith('og:') and content:
+                        if options.include_meta_data == 'extended':
+                            meta_node.content['openGraph'][prop[3:]] = content
+                    elif name and name.startswith('twitter:') and content:
+                        if options.include_meta_data == 'extended':
+                            meta_node.content['twitter'][name[8:]] = content
+                    elif name and name not in non_semantic_tags and content:
+                        meta_node.content['standard'][name] = content
+                # Extract JSON-LD data
+                if options.include_meta_data == 'extended':
+                    json_ld_scripts = child.find_all('script', type='application/ld+json')
+                    for script in json_ld_scripts:
+                        try:
+                            import json
+                            parsed_data = json.loads(script.string)
+                            meta_node.content['jsonLd'].append(parsed_data)
+                        except json.JSONDecodeError as e:
+                            logger.error(f"Failed to parse JSON-LD: {e}")
+                result.append(meta_node)
+            elif tag_name in ['strong', 'b']:
+                content = html_to_markdown_ast(child, options, indent_level + 1)
+                result.append(BoldNode(content=content if content else ""))
+            elif tag_name in ['em', 'i']:
+                content = html_to_markdown_ast(child, options, indent_level + 1)
+                result.append(ItalicNode(content=content if content else ""))
+            elif tag_name in ['s', 'strike']:
+                content = html_to_markdown_ast(child, options, indent_level + 1)
+                result.append(StrikethroughNode(content=content if content else ""))
+            elif tag_name == 'code':
+                is_code_block = child.parent.name == 'pre'
+                content = child.get_text(strip=True)
+                language = ""
+                if not is_code_block:
+                    classes = child.get('class', [])
+                    for cls in classes:
+                        if cls.startswith("language-"):
+                            language = cls.replace("language-", "")
+                            break
+                result.append(CodeNode(content=content, language=language, inline=not is_code_block))
+            elif tag_name == 'blockquote':
+                content = html_to_markdown_ast(child, options, indent_level +1)
+                result.append(BlockquoteNode(content=content))
+            elif tag_name in [
+                'article', 'aside', 'details', 'figcaption', 'figure', 'footer',
+                'header', 'main', 'mark', 'nav', 'section', 'summary', 'time'
+            ]:
+                logger.debug(f"Semantic HTML Element: '{tag_name}'")
+                content = html_to_markdown_ast(child, options, indent_level +1)
+                result.append(SemanticHtmlNode(htmlType=tag_name, content=content))
+            else:
+                # Handle unhandled elements
+                if options.process_unhandled_element:
+                    processed = options.process_unhandled_element(child, options, indent_level)
+                    if processed:
+                        logger.debug(f"Processing Unhandled Element: '{tag_name}'")
+                        result.extend(processed)
+                        continue
+                # Generic HTML elements
+                logger.debug(f"Generic HTMLElement: '{tag_name}'")
+                result.extend(html_to_markdown_ast(child, options, indent_level +1))
+    return result

App_Function_Libraries/html_to_markdown/main.py

@@ -0,0 +1,45 @@
+# html_to_markdown/main.py
+# Usage: python -m html_to_markdown.main input.html output.md --extract-main --refify-urls --include-meta extended --debug
+# Arguments:
+#     input.html: Path to your input HTML file.
+#     output.md: Desired path for the output Markdown file.
+#     --extract-main: (Optional) Extracts the main content from the HTML.
+#     --refify-urls: (Optional) Refactors URLs to reference-style.
+#     --include-meta: (Optional) Includes metadata. Choose between basic or extended.
+#     --debug: (Optional) Enables debug logging for detailed trace.
+
+from html_to_markdown import convert_html_to_markdown
+from conversion_options import ConversionOptions
+
+def main():
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Convert HTML to Markdown.")
+    parser.add_argument('input_file', help="Path to the input HTML file.")
+    parser.add_argument('output_file', help="Path to the output Markdown file.")
+    parser.add_argument('--extract-main', action='store_true', help="Extract main content.")
+    parser.add_argument('--refify-urls', action='store_true', help="Refify URLs.")
+    parser.add_argument('--include-meta', choices=['basic', 'extended'], default=False, help="Include metadata.")
+    parser.add_argument('--debug', action='store_true', help="Enable debug logging.")
+
+    args = parser.parse_args()
+
+    with open(args.input_file, 'r', encoding='utf-8') as f:
+        html_content = f.read()
+
+    options = ConversionOptions(
+        extract_main_content=args.extract_main,
+        refify_urls=args.refify_urls,
+        include_meta_data=args.include_meta if args.include_meta else False,
+        debug=args.debug
+    )
+
+    markdown = convert_html_to_markdown(html_content, options)
+
+    with open(args.output_file, 'w', encoding='utf-8') as f:
+        f.write(markdown)
+
+    print(f"Conversion complete. Markdown saved to {args.output_file}")
+
+if __name__ == "__main__":
+    main()

App_Function_Libraries/html_to_markdown/markdown_ast_to_string.py

@@ -0,0 +1,163 @@
+# html_to_markdown/markdown_ast_to_string.py
+import json
+from ast_utils import find_in_ast
+from typing import List, Optional, Union
+from s_types import (
+    SemanticMarkdownAST, TextNode, BoldNode, ItalicNode, StrikethroughNode,
+    HeadingNode, LinkNode, ImageNode, VideoNode, ListNode, ListItemNode,
+    TableNode, TableRowNode, TableCellNode, CodeNode, BlockquoteNode,
+    SemanticHtmlNode, CustomNode, MetaDataNode
+)
+from conversion_options import ConversionOptions
+import logging
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+def markdown_ast_to_string(nodes: List[SemanticMarkdownAST], options: Optional[ConversionOptions] = None, indent_level: int = 0) -> str:
+    if options is None:
+        options = ConversionOptions()
+
+    markdown_string = ""
+    markdown_string += markdown_meta_ast_to_string(nodes, options, indent_level)
+    markdown_string += markdown_content_ast_to_string(nodes, options, indent_level)
+    return markdown_string
+
+def markdown_meta_ast_to_string(nodes: List[SemanticMarkdownAST], options: ConversionOptions, indent_level: int) -> str:
+    markdown_string = ""
+    if options.include_meta_data:
+        markdown_string += "---\n"
+        node = find_in_ast(nodes, lambda x: isinstance(x, MetaDataNode))
+        if node and isinstance(node, MetaDataNode):
+            standard = node.content.get('standard', {})
+            for key, value in standard.items():
+                markdown_string += f'{key}: "{value}"\n'
+            if options.include_meta_data == 'extended':
+                open_graph = node.content.get('openGraph', {})
+                twitter = node.content.get('twitter', {})
+                json_ld = node.content.get('jsonLd', [])
+
+                if open_graph:
+                    markdown_string += "openGraph:\n"
+                    for key, value in open_graph.items():
+                        markdown_string += f"  {key}: \"{value}\"\n"
+
+                if twitter:
+                    markdown_string += "twitter:\n"
+                    for key, value in twitter.items():
+                        markdown_string += f"  {key}: \"{value}\"\n"
+
+                if json_ld:
+                    markdown_string += "schema:\n"
+                    for item in json_ld:
+                        jld_type = item.get('@type', '(unknown type)')
+                        markdown_string += f"  {jld_type}:\n"
+                        for key, value in item.items():
+                            if key in ['@context', '@type']:
+                                continue
+                            markdown_string += f"    {key}: {json.dumps(value)}\n"
+        markdown_string += "---\n\n"
+    return markdown_string
+
+def markdown_content_ast_to_string(nodes: List[SemanticMarkdownAST], options: ConversionOptions, indent_level: int) -> str:
+    markdown_string = ""
+    for node in nodes:
+        # Skip meta nodes as they are already handled
+        if isinstance(node, MetaDataNode):
+            continue
+
+        # Override node renderer if provided
+        if options.override_node_renderer:
+            override = options.override_node_renderer(node, options, indent_level)
+            if override:
+                markdown_string += override
+                continue
+
+        if isinstance(node, TextNode):
+            markdown_string += f"{node.content}"
+        elif isinstance(node, BoldNode):
+            content = ast_to_markdown(node.content, options, indent_level)
+            markdown_string += f"**{content}**"
+        elif isinstance(node, ItalicNode):
+            content = ast_to_markdown(node.content, options, indent_level)
+            markdown_string += f"*{content}*"
+        elif isinstance(node, StrikethroughNode):
+            content = ast_to_markdown(node.content, options, indent_level)
+            markdown_string += f"~~{content}~~"
+        elif isinstance(node, HeadingNode):
+            markdown_string += f"\n{'#' * node.level} {node.content}\n\n"
+        elif isinstance(node, LinkNode):
+            content = ast_to_markdown(node.content, options, indent_level)
+            if all(isinstance(c, TextNode) for c in node.content):
+                markdown_string += f"[{content}]({node.href})"
+            else:
+                # Use HTML <a> tag for links with rich content
+                markdown_string += f"<a href=\"{node.href}\">{content}</a>"
+        elif isinstance(node, ImageNode):
+            alt = node.alt or ""
+            src = node.src or ""
+            if alt.strip() or src.strip():
+                markdown_string += f"![{alt}]({src})"
+        elif isinstance(node, VideoNode):
+            markdown_string += f"\n![Video]({node.src})\n"
+            if node.poster:
+                markdown_string += f"![Poster]({node.poster})\n"
+            if node.controls:
+                markdown_string += f"Controls: {node.controls}\n"
+            markdown_string += "\n"
+        elif isinstance(node, ListNode):
+            for idx, item in enumerate(node.items):
+                prefix = f"{idx + 1}." if node.ordered else "-"
+                content = ast_to_markdown(item.content, options, indent_level +1).strip()
+                markdown_string += f"{'  ' * indent_level}{prefix} {content}\n"
+            markdown_string += "\n"
+        elif isinstance(node, TableNode):
+            if not node.rows:
+                continue
+            max_columns = max(
+                sum(cell.colspan or 1 for cell in row.cells) for row in node.rows
+            )
+            for row_idx, row in enumerate(node.rows):
+                for cell in row.cells:
+                    content = cell.content if isinstance(cell.content, str) else ast_to_markdown(cell.content, options, indent_level +1).strip()
+                    markdown_string += f"| {content} "
+                # Fill remaining columns
+                remaining = max_columns - sum(cell.colspan or 1 for cell in row.cells)
+                for _ in range(remaining):
+                    markdown_string += "|  "
+                markdown_string += "|\n"
+                if row_idx == 0:
+                    # Add header separator
+                    markdown_string += "|" + "|".join([' --- ' for _ in range(max_columns)]) + "|\n"
+            markdown_string += "\n"
+        elif isinstance(node, CodeNode):
+            if node.inline:
+                markdown_string += f"`{node.content}`"
+            else:
+                language = node.language or ""
+                markdown_string += f"\n```{language}\n{node.content}\n```\n\n"
+        elif isinstance(node, BlockquoteNode):
+            content = ast_to_markdown(node.content, options, indent_level).strip()
+            markdown_string += f"> {content}\n\n"
+        elif isinstance(node, SemanticHtmlNode):
+            if node.htmlType in ["summary", "time", "aside", "nav", "figcaption", "main", "mark", "header", "footer", "details", "figure"]:
+                markdown_string += f"\n<-{node.htmlType}->\n{ast_to_markdown(node.content, options, indent_level)}\n\n</-{node.htmlType}->\n\n"
+            elif node.htmlType == "article":
+                markdown_string += f"\n\n{ast_to_markdown(node.content, options, indent_level)}\n\n"
+            elif node.htmlType == "section":
+                markdown_string += "---\n\n"
+                markdown_string += f"{ast_to_markdown(node.content, options, indent_level)}\n\n---\n\n"
+        elif isinstance(node, CustomNode):
+            if options.render_custom_node:
+                custom_render = options.render_custom_node(node, options, indent_level)
+                if custom_render:
+                    markdown_string += custom_render
+        # Add more node types as needed
+    return markdown_string
+
+def ast_to_markdown(content: Union[str, List[SemanticMarkdownAST]], options: ConversionOptions, indent_level: int) -> str:
+    if isinstance(content, str):
+        return content
+    else:
+        return markdown_content_ast_to_string(content, options, indent_level)
+

App_Function_Libraries/html_to_markdown/s_types.py

@@ -0,0 +1,126 @@
+# html_to_markdown/types.py
+
+from dataclasses import dataclass, field
+from typing import List, Optional, Union, Dict, Any
+
+@dataclass
+class TextNode:
+    type: str = "text"
+    content: str = ""
+
+@dataclass
+class BoldNode:
+    type: str = "bold"
+    content: Union[str, List['SemanticMarkdownAST']] = ""
+
+@dataclass
+class ItalicNode:
+    type: str = "italic"
+    content: Union[str, List['SemanticMarkdownAST']] = ""
+
+@dataclass
+class StrikethroughNode:
+    type: str = "strikethrough"
+    content: Union[str, List['SemanticMarkdownAST']] = ""
+
+@dataclass
+class HeadingNode:
+    type: str = "heading"
+    level: int = 1
+    content: str = ""
+
+@dataclass
+class LinkNode:
+    type: str = "link"
+    href: str = ""
+    content: List['SemanticMarkdownAST'] = field(default_factory=list)
+
+@dataclass
+class ImageNode:
+    type: str = "image"
+    src: str = ""
+    alt: Optional[str] = ""
+
+@dataclass
+class VideoNode:
+    type: str = "video"
+    src: str = ""
+    poster: Optional[str] = ""
+    controls: bool = False
+
+@dataclass
+class ListItemNode:
+    type: str = "listItem"
+    content: List['SemanticMarkdownAST'] = field(default_factory=list)
+
+@dataclass
+class ListNode:
+    type: str = "list"
+    ordered: bool = False
+    items: List[ListItemNode] = field(default_factory=list)
+
+@dataclass
+class TableCellNode:
+    type: str = "tableCell"
+    content: Union[str, List['SemanticMarkdownAST']] = ""
+    colId: Optional[str] = None
+    colspan: Optional[int] = None
+    rowspan: Optional[int] = None
+
+@dataclass
+class TableRowNode:
+    type: str = "tableRow"
+    cells: List[TableCellNode] = field(default_factory=list)
+
+@dataclass
+class TableNode:
+    type: str = "table"
+    rows: List[TableRowNode] = field(default_factory=list)
+    colIds: Optional[List[str]] = None
+
+@dataclass
+class CodeNode:
+    type: str = "code"
+    language: Optional[str] = ""
+    content: str = ""
+    inline: bool = False
+
+@dataclass
+class BlockquoteNode:
+    type: str = "blockquote"
+    content: List['SemanticMarkdownAST'] = field(default_factory=list)
+
+@dataclass
+class CustomNode:
+    type: str = "custom"
+    content: Any = None
+
+@dataclass
+class SemanticHtmlNode:
+    type: str = "semanticHtml"
+    htmlType: str = ""
+    content: List['SemanticMarkdownAST'] = field(default_factory=list)
+
+@dataclass
+class MetaDataNode:
+    type: str = "meta"
+    content: Dict[str, Any] = field(default_factory=dict)
+
+# Union of all node types
+SemanticMarkdownAST = Union[
+    TextNode,
+    BoldNode,
+    ItalicNode,
+    StrikethroughNode,
+    HeadingNode,
+    LinkNode,
+    ImageNode,
+    VideoNode,
+    ListNode,
+    TableNode,
+    CodeNode,
+    BlockquoteNode,
+    SemanticHtmlNode,
+    CustomNode,
+    MetaDataNode
+]

App_Function_Libraries/html_to_markdown/url_utils.py

@@ -0,0 +1,55 @@
+# html_to_markdown/url_utils.py
+
+from typing import Dict
+
+media_suffixes = [
+    "jpeg", "jpg", "png", "gif", "bmp", "tiff", "tif", "svg",
+    "webp", "ico", "avi", "mov", "mp4", "mkv", "flv", "wmv",
+    "webm", "mpeg", "mpg", "mp3", "wav", "aac", "ogg", "flac",
+    "m4a", "pdf", "doc", "docx", "ppt", "pptx", "xls", "xlsx",
+    "txt", "css", "js", "xml", "json", "html", "htm"
+]
+
+def add_ref_prefix(prefix: str, prefixes_to_refs: Dict[str, str]) -> str:
+    if prefix not in prefixes_to_refs:
+        prefixes_to_refs[prefix] = f'ref{len(prefixes_to_refs)}'
+    return prefixes_to_refs[prefix]
+
+def process_url(url: str, prefixes_to_refs: Dict[str, str]) -> str:
+    if not url.startswith('http'):
+        return url
+    else:
+        parts = url.split('/')
+        media_suffix = parts[-1].split('.')[-1].lower()
+        if media_suffix in media_suffixes:
+            prefix = '/'.join(parts[:-1])
+            ref_prefix = add_ref_prefix(prefix, prefixes_to_refs)
+            return f"{ref_prefix}://{parts[-1]}"
+        else:
+            if len(parts) > 4:
+                return add_ref_prefix(url, prefixes_to_refs)
+            else:
+                return url
+
+def refify_urls(markdown_elements: list, prefixes_to_refs: Dict[str, str] = {}) -> Dict[str, str]:
+    for element in markdown_elements:
+        if isinstance(element, dict):
+            node_type = element.get('type')
+            if node_type == 'link':
+                original_href = element.get('href', '')
+                element['href'] = process_url(original_href, prefixes_to_refs)
+                refify_urls(element.get('content', []), prefixes_to_refs)
+            elif node_type in ['image', 'video']:
+                original_src = element.get('src', '')
+                element['src'] = process_url(original_src, prefixes_to_refs)
+            elif node_type == 'list':
+                for item in element.get('items', []):
+                    refify_urls(item.get('content', []), prefixes_to_refs)
+            elif node_type == 'table':
+                for row in element.get('rows', []):
+                    for cell in row.get('cells', []):
+                        if isinstance(cell.get('content'), list):
+                            refify_urls(cell['content'], prefixes_to_refs)
+            elif node_type in ['blockquote', 'semanticHtml']:
+                refify_urls(element.get('content', []), prefixes_to_refs)
+    return prefixes_to_refs

App_Function_Libraries/models/pyannote_diarization_config.yaml

@@ -0,0 +1,13 @@
+pipeline:
+  params:
+    clustering: AgglomerativeClustering
+    embedding: /FULL/PATH/TO/SCRIPT/tldw/App_Function_Libraries/models/pyannote_model_wespeaker-voxceleb-resnet34-LM.bin #models/pyannote_model_wespeaker-voxceleb-resnet34-LM.bin
+    segmentation: /FULL/PATH/TO/SCRIPT/tldw/App_Function_Libraries/models/pyannote_model_segmentation-3.0.bin #models/pyannote_model_segmentation-3.0.bin
+
+params:
+  segmentation:
+    min_duration_off: 0.0
+  clustering:
+    method: centroid
+    min_cluster_size: 12
+    threshold: 0.7045654963945799

App_Function_Libraries/models/pyannote_model_segmentation-3.0.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da85c29829d4002daedd676e012936488234d9255e65e86dfab9bec6b1729298
+size 5905440

App_Function_Libraries/models/pyannote_model_wespeaker-voxceleb-resnet34-LM.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:366edf44f4c80889a3eb7a9d7bdf02c4aede3127f7dd15e274dcdb826b143c56
+size 26645418