Initial numbers

crh_preprocessor/preprocessor.py

@@ -13,7 +13,72 @@ mapping = {
     "w": "v",
     "x": "ks"
 }
- 
+
+zero = {
+    0: 'sıfır',
+}
+
+numbers_map = {
+    1: 'bir',
+    2: 'eki',
+    3: 'üç',
+    4: 'dört',
+    5: 'beş',
+    6: 'altı',
+    7: 'yedi',
+    8: 'sekiz',
+    9: 'doquz',
+    10: 'on',
+    20: 'yigirmi',
+    30: 'otuz',
+    40: 'qırq',
+    50: 'elli',
+    60: 'altmış',
+    70: 'yetmiş',
+    80: 'seksen',
+    90: 'doqsan',
+    100: 'yüz',
+    1000: 'biñ',
+    1_000_000: 'million',
+    1_000_000_000: 'milliard'
+}
+
+
+def spell_numbers(numbers: str) -> str:
+    numbers_map_with_zero = {**numbers_map,**zero}
+    for i in range(0, 10):
+        numbers = numbers.replace(str(i), numbers_map_with_zero[i] + ' ')
+    return numbers.strip()  
+
+
+def num2word(n):
+    if n in numbers_map:
+        return numbers_map[n]
+    elif n < 100:
+        tens = (n // 10) * 10
+        units = n % 10
+        if units == 0:
+            return ''
+        return (numbers_map[tens] + ' ' + numbers_map[units]).strip()
+    elif n < 1000:
+        hundreds = n // 100
+        rest = n % 100
+        return (num2word(hundreds) + ' ' + numbers_map[100] + ' ' + num2word(rest)).strip()
+    elif n < 1_000_000:
+        thousands = n // 1_000
+        rest = n % 1_000
+        return (num2word(thousands) + ' ' + numbers_map[1_000] + ' ' + num2word(rest)).strip()
+    elif n < 1_000_000_000:
+        millions = n // 1_000_000
+        rest = n % 1_000_000
+        return (num2word(millions) + ' ' + numbers_map[1_000_000] + ' ' + num2word(rest)).strip()
+    elif n < 1_000_000_000_000:
+        billions = n // 1_000_000_000
+        rest = n % 1_000_000_000
+        return (num2word(billions) + ' ' + numbers_map[1_000_000_000] + ' ' + num2word(rest)).strip()
+    else:
+        return spell_numbers(str(n))
+
 
 def preprocess(text):
     text = text.lower()  # always treat lowercase
@@ -26,21 +91,42 @@ def preprocess(text):
     for symbol in separators:
         text = text.replace(symbol, ".")
 
-    numbers = {
-        "0": "sıfır",
-        "1": "bir",
-        "2": "eki",
-        "3": "üç",
-        "4": "dört",
-        "5": "beş",
-        "6": "altı",
-        "7": "yedi",
-        "8": "sekiz",
-        "9": "doquz",
-    }
-
-    for number in numbers.keys():
-        text = text.replace(number, numbers[number] + " ")
-
-    return text[1:-1]
+    while True:
+        number_match = re.search("-?\d+(\.|,)?(\d+)?", text)
+        
+        if number_match is None:
+            break
+        
+        print(number_match.string, number_match.start(), number_match.end())
+        
+        number = number_match.string.strip()
+
+        prefix = ""
+
+        if number.startswith("-"):
+            prefix = "minus "
+            number = number.replace("-", "", 1)
+        elif number.startswith("+"):
+            prefix = "plüs "
+            number = number.replace("+", "", 1)
+
+        if "." in number:
+            number = number.split(".")
+            number = prefix + " noqta ".join((num2word(int(number[0])) if int(number[0]) != 0 else spell_numbers(number[0]), spell_numbers(number[1])))
+            text = text.replace(number_match.string.strip(), number, 1)
+            continue
+        elif "," in number:
+            number = number.split(",")
+            number = prefix + " virgül ".join((num2word(int(number[0])) if int(number[0]) != 0 else spell_numbers(number[0]), spell_numbers(number[1])))
+            text = text.replace(number_match.string.strip(), number, 1)
+            continue
+
+        if number.startswith("0"):
+            text = text.replace(number_match.string.strip(), prefix + spell_numbers(number), 1)
+            continue
+        
+        text = text.replace(number_match.string.strip(), prefix + num2word(int(number)), 1)
+        
+
+    return text.strip()
     

requirements-dev.txt

@@ -1,4 +0,0 @@
--r requirements.txt
--r requirements-test.txt
-huggingface_hub
-black

requirements-test.txt

@@ -1,3 +1,5 @@
+-r requirements.txt
 pytest==7.1.3
 pytest-cov==4.0.0
-tabulate==0.8.10
+tabulate==0.8.10
+black

requirements.txt

@@ -1,3 +1,4 @@
 gradio==3.6
 torch>=1.13
-TTS==0.9.0
+TTS==0.9.0
+huggingface_hub

tests/test_preprocessor.py

@@ -1,7 +1,70 @@
-from crh_preprocessor.preprocessor import preprocess
+from crh_preprocessor.preprocessor import preprocess, num2word
+
+
+def test_num2word():
+    assert (
+        num2word(16) == "on altı"
+    )
+    assert (
+        num2word(1324759813) == "bir milliard üç yüz yigirmi dört million yedi yüz elli doquz biñ sekiz yüz on üç"
+    )
+    assert (
+        num2word(1_000_000) == "million"
+    )
 
 
 def test_preprocessor():
     assert (
         preprocess("İşanç Alla-Taalâğa.") == "işan\u04ab alla-taalâğa."
     )  # first i is two symbols (i without dot and dot)
+    assert (
+        preprocess("1000000") == "million"
+    )
+    assert (
+        preprocess("1324700000") == "bir milliard üç yüz yigirmi dört million yedi yüz biñ"
+    )
+    assert (
+        preprocess("1000002") == "bir million eki"
+    )
+    assert (
+        preprocess("16") == "on altı"
+    )
+    assert (
+        preprocess("001") == "sıfır sıfır bir"
+    )
+    assert (
+        preprocess("00") == "sıfır sıfır"
+    )
+    assert (
+        preprocess("10.02") == "on noqta sıfır eki"
+    )
+    assert (
+        preprocess("0.01") == "sıfır noqta sıfır bir"
+    )
+    assert (
+        preprocess("0,01") == "sıfır virgül sıfır bir"
+    )
+    assert (
+        preprocess("00,01") == "sıfır sıfır virgül sıfır bir"
+    )
+    assert (
+        preprocess("-10") == "minus on"
+    )
+    assert (
+        preprocess("+10") == "plüs on"
+    )
+    assert (
+        preprocess("+10.1400") == "plüs on noqta bir dört sıfır sıfır"
+    )
+    assert (
+        preprocess("-10.14156") == "minus on noqta bir dört bir beş altı"
+    )
+    assert (
+        preprocess("10,14156") == "on virgül bir dört bir beş altı"
+    )
+    assert (
+        preprocess("1, 2, 3, 4, 5, 6,7") == "bir virgül eki virgül"
+    )
+    assert (
+        preprocess("1,2,3,4,5,6,7") == "on altı"
+    )