<commit_before>class Uni:
    strict = [
        [
            ["ә"],
            "ӭ"
        ],
        [
            ["ә̄"],
            "ӭ̄"
        ],
        [
            ["а́"],
            "ӓ"
        ],
        [
            ["а́̄"],
            "ӓ̄"
        ],
        [
            ["i"],
            "и̇"
        ],
        [
            ["ī"],
            "и̇̄"
        ],
        [
            ["дз"],
            "ҙ"
        ],
        [
            ["қ"],
            "ӄ"
        ],
        [
            ["ң"],
            "ӈ"
        ],
        [
            ["дж"],
            "җ"
        ],
        [
            ["ӌ"],
            "ҷ"
        ],
    ]

    soft = [
        [
            ["ж"],
            "җ"
        ],
        [
            ["х"],
            "ҳ"
        ],
        [
            ["ч"],
            "ҷ"
        ]
    ]

    @staticmethod
    def short(string):
        string = string.replace("̄", "")
        string = string.replace("̈", "")
        return string

    @staticmethod
    def unify(string, strict_only = False):
        strict_result = string
        for group in uni.strict:
            for query in group[0]:
                strict_result = strict_result.replace(query, group[1])
        #
        soft_result = strict_result
        for group in uni.soft:
            for query in group[0]:
                soft_result = soft_result.replace(query, ''.join(["?" for _ in range(len(query))]))
        if strict_only:
            return strict_result
        else:
            return strict_result, soft_result

    @staticmethod
    def soft_compare(string, soft_query):
        if len(string) != len(soft_query):
            return string
        for x in range(len(string)):
            if string[x] == soft_query[x]:
                pass
            elif string[x] != soft_query[x] and soft_query[x] == "?":
                pass
            else:
                return False
        return True


class Compare:
    def __init__(self, first, second=False):
        if second:
            self.first = self.clean(first)
            self.second = self.clean(second)
        else:
            self.first = first

    extension_symbols = [
        [
            ["ё̄", "е̄", "е"],
            ["ы̄", "ы"],
            ["ӧ", "ӧ̄", "о̄", "о"],
            ["ю̈", "ю̈̄", "ю̄"],
            ["я̄", "я"],
            ["ӭ", "ӭ̄", "э̄", "э"],
            ["ӓ", "ӓ̄", "ā", "а"],
            ["ӱ", "ӱ̄", "ӯ", "у"],
            ["и̇", "и̇̄", "ӣ", "и"],
            ["ӷ", "г"],
            ["ӄ", "к"],
            ["ӈ", "н"],
            ["җ", "ж"],
            ["ҳ", "х"],
            ["ҷ", "ч"]
        ]
    ]

    def clean(self, query):
        query = query.replace("̄", "")
        query = query.replace("̈", "")
        for group in self.extension_symbols:
            for subgroup in group:
                for i in range(len(subgroup)):
                    if i == len(subgroup) - 1:
                        break
                    query = query.replace(subgroup[i], subgroup[-1])

        return query

    def is_endsym(self, sym):
        for group in self.extension_symbols:
            for subgroup in group:
                if subgroup[-1] == sym:
                    return True

        return False

    def is_extension(self, sym):
        for group in self.extension_symbols:
            for subgroup in group:
                if sym in subgroup[:-1]:<commit_msg>new ext sym row<commit_after>class Uni:
    strict = [
        [
            ["ә"],
            "ӭ"
        ],
        [
            ["ә̄"],
            "ӭ̄"
        ],
        [
            ["а́"],
            "ӓ"
        ],
        [
            ["а́̄"],
            "ӓ̄"
        ],
        [
            ["i"],
            "и̇"
        ],
        [
            ["ī"],
            "и̇̄"
        ],
        [
            ["дз"],
            "ҙ"
        ],
        [
            ["қ"],
            "ӄ"
        ],
        [
            ["ң"],
            "ӈ"
        ],
        [
            ["дж"],
            "җ"
        ],
        [
            ["ӌ"],
            "ҷ"
        ],
    ]

    soft = [
        [
            ["ж"],
            "җ"
        ],
        [
            ["х"],
            "ҳ"
        ],
        [
            ["ч"],
            "ҷ"
        ]
    ]

    @staticmethod
    def short(string):
        string = string.replace("̄", "")
        string = string.replace("̈", "")
        return string

    @staticmethod
    def unify(string, strict_only = False):
        strict_result = string
        for group in uni.strict:
            for query in group[0]:
                strict_result = strict_result.replace(query, group[1])
        #
        soft_result = strict_result
        for group in uni.soft:
            for query in group[0]:
                soft_result = soft_result.replace(query, ''.join(["?" for _ in range(len(query))]))
        if strict_only:
            return strict_result
        else:
            return strict_result, soft_result

    @staticmethod
    def soft_compare(string, soft_query):
        if len(string) != len(soft_query):
            return string
        for x in range(len(string)):
            if string[x] == soft_query[x]:
                pass
            elif string[x] != soft_query[x] and soft_query[x] == "?":
                pass
            else:
                return False
        return True


class Compare:
    def __init__(self, first, second=False):
        if second:
            self.first = self.clean(first)
            self.second = self.clean(second)
        else:
            self.first = first

    extension_symbols = [
        [
            ["ё̄", "е̄", "е"],
            ["ы̄", "ы"],
            ["ӧ", "ӧ̄", "о̄", "о"],
            ["ю̈", "ю̈̄", "ю̄"],
            ["я̄", "я"],
            ["ӭ", "ӭ̄", "э̄", "э"],
            ["ӓ", "ӓ̄", "ā", "а"],
            ["ӱ", "ӱ̄", "ӯ", "у"],
            ["и̇", "и̇̄", "ӣ", "и"],
            ["ӷ", "г"],
            ["ӄ", "к"],
            ["ӈ", "н"],
            ["җ", "ж"],
            ["ҳ", "х"],
            ["ҷ", "ч"],
            ["ҙ", "дз"]
        ]
    ]

    def clean(self, query):
        query = query.replace("̄", "")
        query = query.replace("̈", "")
        for group in self.extension_symbols:
            for subgroup in group:
                for i in range(len(subgroup)):
                    if i == len(subgroup) - 1:
                        break
                    query = query.replace(subgroup[i], subgroup[-1])

        return query

    def is_endsym(self, sym):
        for group in self.extension_symbols:
            for subgroup in group:
                if subgroup[-1] == sym:
                    return True

        return False

    def is_extension(self, sym):
        for group in self.extension_symbols:
            for subgroup in group:
                if sym in subgroup[:-1]:
{'fraction_non_alphanumeric': 0.15803972741771785, 'fraction_numerical': 0.0020298680585761927, 'mean_word_length': 1.0110787172011662, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1412517', 'n_tokens_mistral': 2116, 'n_tokens_neox': 2117, 'n_words': 466}