Add data logging

README.md

@@ -18,6 +18,7 @@ Text-to-Speech for Crimean Tatar language
 Source code: https://github.com/robinhad/qirimtatar-tts  
 Online demo: https://huggingface.co/spaces/robinhad/qirimtatar-tts  
 You're welcome to join UA Speech Recognition and Synthesis community: Telegram https://t.me/speech_recognition_uk  
+Note: demo input is saved to improve Text-to-Speech engine and demo experience. By using this demo you give your consent to this.  
 
 ## Examples
 Test sentence:

app.py

@@ -1,3 +1,8 @@
+from os import getenv
+from queue import Queue
+from threading import Thread
+from time import sleep
+from data_logger import log_data
 import gradio as gr
 from crh_transliterator.transliterator import transliterate
 from crh_preprocessor.preprocessor import preprocess
@@ -18,6 +23,27 @@ class VoiceOption(Enum):
     # Abibulla = "Абібулла (чоловічий) 👨"
 
 
+def check_thread(logging_queue: Queue):
+    logging_callback = log_data(hf_token=getenv("HF_API_TOKEN"), dataset_name="crh-tts-output", private=False)
+    while True:
+        sleep(60)
+        batch = []
+        while not logging_queue.empty():
+            batch.append(logging_queue.get())
+
+        if len(batch) > 0:
+            try:
+                logging_callback(batch)
+            except:
+                print("Error happened while pushing data to HF. Puttting items back in queue...")
+                for item in batch:
+                    logging_queue.put(item)
+
+if getenv("HF_API_TOKEN") is not None:
+    log_queue = Queue()
+    t = Thread(target=check_thread, args=(log_queue,))
+    t.start()
+
 print(f"CUDA available? {is_available()}")
 
 
@@ -42,6 +68,8 @@ def tts(text: str, voice: str):
     }
 
     speaker_name = voice_mapping[voice]
+    if getenv("HF_API_TOKEN") is not None:
+        log_queue.put([text, speaker_name, str(datetime.utcnow())])
     text_limit = 7200
     text = (
         text if len(text) < text_limit else text[0:text_limit]

data_logger.py

@@ -0,0 +1,41 @@
+from gradio import utils
+import os
+import csv
+import huggingface_hub
+
+def log_data(hf_token: str, dataset_name: str, private=True):
+    path_to_dataset_repo = huggingface_hub.create_repo(
+            name=dataset_name,
+            token=hf_token,
+            private=private,
+            repo_type="dataset",
+            exist_ok=True,
+        )
+    flagging_dir = "flagged"
+    dataset_dir = os.path.join(flagging_dir, dataset_name)
+    repo = huggingface_hub.Repository(
+            local_dir=dataset_dir,
+            clone_from=path_to_dataset_repo,
+            use_auth_token=hf_token,
+        )
+    repo.git_pull(lfs=True)
+    log_file = os.path.join(dataset_dir, "data.csv")
+
+    def log_function(data):
+        repo.git_pull(lfs=True)
+
+        with open(log_file, "a", newline="", encoding="utf-8") as csvfile:
+            writer = csv.writer(csvfile)
+
+            for row in data:
+                writer.writerow(utils.sanitize_list_for_csv(row))
+
+        with open(log_file, "r", encoding="utf-8") as csvfile:
+            line_count = len([None for row in csv.reader(csvfile)]) - 1
+
+        repo.push_to_hub(commit_message="Flagged sample #{}".format(line_count))
+
+        return line_count
+
+    return log_function
+

requirements-dev.txt

@@ -1,3 +1,4 @@
 -r requirements.txt
 -r requirements-test.txt
+huggingface_hub
 black