Delete tools

tools/__init__.py

@@ -1,3 +0,0 @@
-"""
-工具包
-"""

tools/classify_language.py

@@ -1,197 +0,0 @@
-import regex as re
-
-try:
-    from config import config
-
-    LANGUAGE_IDENTIFICATION_LIBRARY = (
-        config.webui_config.language_identification_library
-    )
-except:
-    LANGUAGE_IDENTIFICATION_LIBRARY = "langid"
-
-module = LANGUAGE_IDENTIFICATION_LIBRARY.lower()
-
-langid_languages = [
-    "af",
-    "am",
-    "an",
-    "ar",
-    "as",
-    "az",
-    "be",
-    "bg",
-    "bn",
-    "br",
-    "bs",
-    "ca",
-    "cs",
-    "cy",
-    "da",
-    "de",
-    "dz",
-    "el",
-    "en",
-    "eo",
-    "es",
-    "et",
-    "eu",
-    "fa",
-    "fi",
-    "fo",
-    "fr",
-    "ga",
-    "gl",
-    "gu",
-    "he",
-    "hi",
-    "hr",
-    "ht",
-    "hu",
-    "hy",
-    "id",
-    "is",
-    "it",
-    "ja",
-    "jv",
-    "ka",
-    "kk",
-    "km",
-    "kn",
-    "ko",
-    "ku",
-    "ky",
-    "la",
-    "lb",
-    "lo",
-    "lt",
-    "lv",
-    "mg",
-    "mk",
-    "ml",
-    "mn",
-    "mr",
-    "ms",
-    "mt",
-    "nb",
-    "ne",
-    "nl",
-    "nn",
-    "no",
-    "oc",
-    "or",
-    "pa",
-    "pl",
-    "ps",
-    "pt",
-    "qu",
-    "ro",
-    "ru",
-    "rw",
-    "se",
-    "si",
-    "sk",
-    "sl",
-    "sq",
-    "sr",
-    "sv",
-    "sw",
-    "ta",
-    "te",
-    "th",
-    "tl",
-    "tr",
-    "ug",
-    "uk",
-    "ur",
-    "vi",
-    "vo",
-    "wa",
-    "xh",
-    "zh",
-    "zu",
-]
-
-
-def classify_language(text: str, target_languages: list = None) -> str:
-    if module == "fastlid" or module == "fasttext":
-        from fastlid import fastlid, supported_langs
-
-        classifier = fastlid
-        if target_languages != None:
-            target_languages = [
-                lang for lang in target_languages if lang in supported_langs
-            ]
-            fastlid.set_languages = target_languages
-    elif module == "langid":
-        import langid
-
-        classifier = langid.classify
-        if target_languages != None:
-            target_languages = [
-                lang for lang in target_languages if lang in langid_languages
-            ]
-            langid.set_languages(target_languages)
-    else:
-        raise ValueError(f"Wrong module {module}")
-
-    lang = classifier(text)[0]
-
-    return lang
-
-
-def classify_zh_ja(text: str) -> str:
-    for idx, char in enumerate(text):
-        unicode_val = ord(char)
-
-        # 检测日语字符
-        if 0x3040 <= unicode_val <= 0x309F or 0x30A0 <= unicode_val <= 0x30FF:
-            return "ja"
-
-        # 检测汉字字符
-        if 0x4E00 <= unicode_val <= 0x9FFF:
-            # 检查周围的字符
-            next_char = text[idx + 1] if idx + 1 < len(text) else None
-
-            if next_char and (
-                0x3040 <= ord(next_char) <= 0x309F or 0x30A0 <= ord(next_char) <= 0x30FF
-            ):
-                return "ja"
-
-    return "zh"
-
-
-def split_alpha_nonalpha(text, mode=1):
-    if mode == 1:
-        pattern = r"(?<=[\u4e00-\u9fff\u3040-\u30FF\d\s])(?=[\p{Latin}])|(?<=[\p{Latin}\s])(?=[\u4e00-\u9fff\u3040-\u30FF\d])"
-    elif mode == 2:
-        pattern = r"(?<=[\u4e00-\u9fff\u3040-\u30FF\s])(?=[\p{Latin}\d])|(?<=[\p{Latin}\d\s])(?=[\u4e00-\u9fff\u3040-\u30FF])"
-    else:
-        raise ValueError("Invalid mode. Supported modes are 1 and 2.")
-
-    return re.split(pattern, text)
-
-
-if __name__ == "__main__":
-    text = "这是一个测试文本"
-    print(classify_language(text))
-    print(classify_zh_ja(text))  # "zh"
-
-    text = "これはテストテキストです"
-    print(classify_language(text))
-    print(classify_zh_ja(text))  # "ja"
-
-    text = "vits和Bert-VITS2是tts模型。花费3days.花费3天。Take 3 days"
-
-    print(split_alpha_nonalpha(text, mode=1))
-    # output: ['vits', '和', 'Bert-VITS', '2是', 'tts', '模型。花费3', 'days.花费3天。Take 3 days']
-
-    print(split_alpha_nonalpha(text, mode=2))
-    # output: ['vits', '和', 'Bert-VITS2', '是', 'tts', '模型。花费', '3days.花费', '3', '天。Take 3 days']
-
-    text = "vits 和 Bert-VITS2 是 tts 模型。花费3days.花费3天。Take 3 days"
-    print(split_alpha_nonalpha(text, mode=1))
-    # output: ['vits ', '和 ', 'Bert-VITS', '2 ', '是 ', 'tts ', '模型。花费3', 'days.花费3天。Take ', '3 ', 'days']
-
-    text = "vits 和 Bert-VITS2 是 tts 模型。花费3days.花费3天。Take 3 days"
-    print(split_alpha_nonalpha(text, mode=2))
-    # output: ['vits ', '和 ', 'Bert-VITS2 ', '是 ', 'tts ', '模型。花费', '3days.花费', '3', '天。Take ', '3 ', 'days']

tools/log.py

@@ -1,16 +0,0 @@
-"""
-logger封装
-"""
-from loguru import logger
-import sys
-
-
-# 移除所有默认的处理器
-logger.remove()
-
-# 自定义格式并添加到标准输出
-log_format = (
-    "<g>{time:MM-DD HH:mm:ss}</g> |<lvl>{level:^8}</lvl>| {file}:{line} | {message}"
-)
-
-logger.add(sys.stdout, format=log_format, backtrace=True, diagnose=True)

tools/sentence.py

@@ -1,173 +0,0 @@
-import logging
-
-import regex as re
-
-from tools.classify_language import classify_language, split_alpha_nonalpha
-
-
-def check_is_none(item) -> bool:
-    """none -> True, not none -> False"""
-    return (
-        item is None
-        or (isinstance(item, str) and str(item).isspace())
-        or str(item) == ""
-    )
-
-
-def markup_language(text: str, target_languages: list = None) -> str:
-    pattern = (
-        r"[\!\"\#\$\%\&\'\(\)\*\+\,\-\.\/\:\;\<\>\=\?\@\[\]\{\}\\\\\^\_\`"
-        r"\！？。＂＃＄％＆＇（）＊＋，－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､、〃》「」"
-        r"『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘\'\‛\“\”\„\‟…‧﹏.]+"
-    )
-    sentences = re.split(pattern, text)
-
-    pre_lang = ""
-    p = 0
-
-    if target_languages is not None:
-        sorted_target_languages = sorted(target_languages)
-        if sorted_target_languages in [["en", "zh"], ["en", "ja"], ["en", "ja", "zh"]]:
-            new_sentences = []
-            for sentence in sentences:
-                new_sentences.extend(split_alpha_nonalpha(sentence))
-            sentences = new_sentences
-
-    for sentence in sentences:
-        if check_is_none(sentence):
-            continue
-
-        lang = classify_language(sentence, target_languages)
-
-        if pre_lang == "":
-            text = text[:p] + text[p:].replace(
-                sentence, f"[{lang.upper()}]{sentence}", 1
-            )
-            p += len(f"[{lang.upper()}]")
-        elif pre_lang != lang:
-            text = text[:p] + text[p:].replace(
-                sentence, f"[{pre_lang.upper()}][{lang.upper()}]{sentence}", 1
-            )
-            p += len(f"[{pre_lang.upper()}][{lang.upper()}]")
-        pre_lang = lang
-        p += text[p:].index(sentence) + len(sentence)
-    text += f"[{pre_lang.upper()}]"
-
-    return text
-
-
-def split_by_language(text: str, target_languages: list = None) -> list:
-    pattern = (
-        r"[\!\"\#\$\%\&\'\(\)\*\+\,\-\.\/\:\;\<\>\=\?\@\[\]\{\}\\\\\^\_\`"
-        r"\！？\。＂＃＄％＆＇（）＊＋，－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､、〃》「」"
-        r"『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘\'\‛\“\”\„\‟…‧﹏.]+"
-    )
-    sentences = re.split(pattern, text)
-
-    pre_lang = ""
-    start = 0
-    end = 0
-    sentences_list = []
-
-    if target_languages is not None:
-        sorted_target_languages = sorted(target_languages)
-        if sorted_target_languages in [["en", "zh"], ["en", "ja"], ["en", "ja", "zh"]]:
-            new_sentences = []
-            for sentence in sentences:
-                new_sentences.extend(split_alpha_nonalpha(sentence))
-            sentences = new_sentences
-
-    for sentence in sentences:
-        if check_is_none(sentence):
-            continue
-
-        lang = classify_language(sentence, target_languages)
-
-        end += text[end:].index(sentence)
-        if pre_lang != "" and pre_lang != lang:
-            sentences_list.append((text[start:end], pre_lang))
-            start = end
-        end += len(sentence)
-        pre_lang = lang
-    sentences_list.append((text[start:], pre_lang))
-
-    return sentences_list
-
-
-def sentence_split(text: str, max: int) -> list:
-    pattern = r"[!(),—+\-.:;?？。，、；：]+"
-    sentences = re.split(pattern, text)
-    discarded_chars = re.findall(pattern, text)
-
-    sentences_list, count, p = [], 0, 0
-
-    # 按被分割的符号遍历
-    for i, discarded_chars in enumerate(discarded_chars):
-        count += len(sentences[i]) + len(discarded_chars)
-        if count >= max:
-            sentences_list.append(text[p : p + count].strip())
-            p += count
-            count = 0
-
-    # 加入最后剩余的文本
-    if p < len(text):
-        sentences_list.append(text[p:])
-
-    return sentences_list
-
-
-def sentence_split_and_markup(text, max=50, lang="auto", speaker_lang=None):
-    # 如果该speaker只支持一种语言
-    if speaker_lang is not None and len(speaker_lang) == 1:
-        if lang.upper() not in ["AUTO", "MIX"] and lang.lower() != speaker_lang[0]:
-            logging.debug(
-                f'lang "{lang}" is not in speaker_lang {speaker_lang},automatically set lang={speaker_lang[0]}'
-            )
-        lang = speaker_lang[0]
-
-    sentences_list = []
-    if lang.upper() != "MIX":
-        if max <= 0:
-            sentences_list.append(
-                markup_language(text, speaker_lang)
-                if lang.upper() == "AUTO"
-                else f"[{lang.upper()}]{text}[{lang.upper()}]"
-            )
-        else:
-            for i in sentence_split(text, max):
-                if check_is_none(i):
-                    continue
-                sentences_list.append(
-                    markup_language(i, speaker_lang)
-                    if lang.upper() == "AUTO"
-                    else f"[{lang.upper()}]{i}[{lang.upper()}]"
-                )
-    else:
-        sentences_list.append(text)
-
-    for i in sentences_list:
-        logging.debug(i)
-
-    return sentences_list
-
-
-if __name__ == "__main__":
-    text = "这几天心里颇不宁静。今晚在院子里坐着乘凉，忽然想起日日走过的荷塘，在这满月的光里，总该另有一番样子吧。月亮渐渐地升高了，墙外马路上孩子们的欢笑，已经听不见了；妻在屋里拍着闰儿，迷迷糊糊地哼着眠歌。我悄悄地披了大衫，带上门出去。"
-    print(markup_language(text, target_languages=None))
-    print(sentence_split(text, max=50))
-    print(sentence_split_and_markup(text, max=50, lang="auto", speaker_lang=None))
-
-    text = "你好，这是一段用来测试自动标注的文本。こんにちは,これは自動ラベリングのテスト用テキストです.Hello, this is a piece of text to test autotagging.你好！今天我们要介绍VITS项目，其重点是使用了GAN Duration predictor和transformer flow,并且接入了Bert模型来提升韵律。Bert embedding会在稍后介绍。"
-    print(split_by_language(text, ["zh", "ja", "en"]))
-
-    text = "vits和Bert-VITS2是tts模型。花费3days.花费3天。Take 3 days"
-
-    print(split_by_language(text, ["zh", "ja", "en"]))
-    # output: [('vits', 'en'), ('和', 'ja'), ('Bert-VITS', 'en'), ('2是', 'zh'), ('tts', 'en'), ('模型。花费3', 'zh'), ('days.', 'en'), ('花费3天。', 'zh'), ('Take 3 days', 'en')]
-
-    print(split_by_language(text, ["zh", "en"]))
-    # output: [('vits', 'en'), ('和', 'zh'), ('Bert-VITS', 'en'), ('2是', 'zh'), ('tts', 'en'), ('模型。花费3', 'zh'), ('days.', 'en'), ('花费3天。', 'zh'), ('Take 3 days', 'en')]
-
-    text = "vits 和 Bert-VITS2 是 tts 模型。花费 3 days. 花费 3天。Take 3 days"
-    print(split_by_language(text, ["zh", "en"]))
-    # output: [('vits ', 'en'), ('和 ', 'zh'), ('Bert-VITS2 ', 'en'), ('是 ', 'zh'), ('tts ', 'en'), ('模型。花费 ', 'zh'), ('3 days. ', 'en'), ('花费 3天。', 'zh'), ('Take 3 days', 'en')]

tools/translate.py

@@ -1,61 +0,0 @@
-"""
-翻译api
-"""
-from config import config
-
-import random
-import hashlib
-import requests
-
-
-def translate(Sentence: str, to_Language: str = "jp", from_Language: str = ""):
-    """
-    :param Sentence: 待翻译语句
-    :param from_Language: 待翻译语句语言
-    :param to_Language: 目标语言
-    :return: 翻译后语句 出错时返回None
-
-    常见语言代码：中文 zh 英语 en 日语 jp
-    """
-    appid = config.translate_config.app_key
-    key = config.translate_config.secret_key
-    if appid == "" or key == "":
-        return "请开发者在config.yml中配置app_key与secret_key"
-    url = "https://fanyi-api.baidu.com/api/trans/vip/translate"
-    texts = Sentence.splitlines()
-    outTexts = []
-    for t in texts:
-        if t != "":
-            # 签名计算 参考文档 https://api.fanyi.baidu.com/product/113
-            salt = str(random.randint(1, 100000))
-            signString = appid + t + salt + key
-            hs = hashlib.md5()
-            hs.update(signString.encode("utf-8"))
-            signString = hs.hexdigest()
-            if from_Language == "":
-                from_Language = "auto"
-            headers = {"Content-Type": "application/x-www-form-urlencoded"}
-            payload = {
-                "q": t,
-                "from": from_Language,
-                "to": to_Language,
-                "appid": appid,
-                "salt": salt,
-                "sign": signString,
-            }
-            # 发送请求
-            try:
-                response = requests.post(
-                    url=url, data=payload, headers=headers, timeout=3
-                )
-                response = response.json()
-                if "trans_result" in response.keys():
-                    result = response["trans_result"][0]
-                    if "dst" in result.keys():
-                        dst = result["dst"]
-                        outTexts.append(dst)
-            except Exception:
-                return Sentence
-        else:
-            outTexts.append(t)
-    return "\n".join(outTexts)