text fix

app/main.py

@@ -15,6 +15,8 @@ app = FastAPI(
         {
             "url": "http://127.0.0.1:8000/api/v1",
             "description": "Local Server",
+        },
+        {
             "url": "https://r3vibe-mother-tongue.hf.space/api/v1",
             "description": "Huggingface Server",
         }

app/matcher.py

@@ -18,8 +18,6 @@ def sequence_match(a, b):
     return difflib.SequenceMatcher(None, a, b).ratio()
 
 
-
-
 def match(original, transcription):
     sequence = sequence_match(original, transcription)
     phonetic = phonetic_match(original, transcription)

app/routers/V1/voice/voice_router.py

@@ -6,7 +6,7 @@ import os
 from app.transcriber import get_transcription
 from app.matcher import match
 from app.mfcc import mfcc_similarty_check
-from app.string_processor import process_text
+from app.string_processor import clean_transcription
 
 
 """ initialize the router """
@@ -51,7 +51,8 @@ async def transcribe_audio(
 
         try:
             text = get_transcription(filename_recorded)
-            sequence, phonetic = match(matcher_text, process_text(text))
+            text = clean_transcription(text)
+            sequence, phonetic = match(matcher_text, text)
             Euclidean, Cosine = mfcc_similarty_check(
                 filename_original, filename_recorded
             )

app/string_processor.py

@@ -1,18 +1,22 @@
-import string
+import unicodedata
 import re
 
 
-def process_text(text):
-    # Step 1: Strip whitespace from both ends
-    text = text.strip()
+def clean_transcription(text):
+    # Normalize the text to NFKD form
+    normalized_text = unicodedata.normalize('NFKD', text)
+    
+    # Remove diacritics
+    cleaned_text = ''.join([c for c in normalized_text if not unicodedata.combining(c)])
+    
+    # Explicitly remove the leading ʻ character and any other specific characters
+    cleaned_text = cleaned_text.replace('ʻ', '')
+    
+    # Remove any remaining special characters (if any)
+    cleaned_text = re.sub(r'[^\w\s]', '', cleaned_text)
+    
+    # Ensure the text is stripped of any unwanted leading or trailing whitespace
+    cleaned_text = cleaned_text.strip()
+    
+    return cleaned_text
 
-    # Step 2: Remove all punctuation (including full stops and commas)
-    text = text.translate(str.maketrans("", "", string.punctuation))
-
-    # Step 3: Extract sentences (assuming you want to keep the text as a whole sentence)
-    sentences = re.split(r"(?<=[.!?]) +", text)
-
-    # Combine the sentences back into a single string without punctuation
-    processed_text = " ".join(sentences)
-
-    return processed_text