<commit_before># coding: utf-8
import re

from dragonmapper.hanzi import MIXED, BOTH, UNKNOWN
from dragonmapper.hanzi import identify as s_identify
from zhon.hanzi import sentence as cjc_sentence_re

from charset_normalizer.unicode import UnicodeRangeIdentify

from functools import lru_cache


@lru_cache(maxsize=8192)
class ProbeChaos:

    def __init__(self, string, giveup_threshold=0.09):
        """
        :param str string:
        """

        if not isinstance(string, str):
            raise TypeError('Cannot probe chaos from type <{}>, expected <str>'.format(type(string)))

        self._string = string
        self._threshold = giveup_threshold

        self.successive_upper_lower = 0
        self.successive_accent = 0
        self.successive_different_unicode_range = 0
        self.encountered_unicode_range = set()
        self.encountered_punc_sign = 0
        self.unprintable = 0
        self.encountered_white_space = 0
        self.not_encountered_white_space = 0

        self.encountered_unicode_range_occurrences = dict()

        self.not_encountered_white_space_reset = 0

        self.previous_printable_letter = None

        self.previous_encountered_unicode_range = None

        self.total_letter_encountered = 0

        self.total_lower_letter_encountered = 0
        self.total_upper_accent_encountered = 0
        self.total_upper_accent_encountered_inner = 0
        self.total_unaccented_letter_encountered = 0

        self.gave_up = False

        if len(self._string) >= 10:
            self._probe()

    def _probe(self):

        c__ = False

        for c, i_ in zip(self._string, range(0, len(self._string))):

            if not c__:
                state_ = (i_ / len(self._string) >= 0.5)

            # If we already have measured 10 % or more of chaos after reading 50 %, give up.
            if not c__ and state_ and self.ratio >= self._threshold:
                self.gave_up = True
                break
            elif c__ is False and state_:
                c__ = True

            self.total_letter_encountered += 1

            if not c.isprintable():
                if c not in ['\n', '\t', '\r']:
                    if not UnicodeRangeIdentify.is_cjk(c) and not UnicodeRangeIdentify.is_punc(c):
                        self.unprintable += 2

                self.encountered_white_space += 1
                self.not_encountered_white_space = 0
                self.not_encountered_white_space_reset += 1
                continue

            if c.isspace():
                self.encountered_white_space += 1
                self.not_encountered_white_space = 0
                self.not_encountered_white_space_reset += 1
                self.previous_printable_letter = c
                continue

            if self.not_encountered_white_space_reset < 2:
                self.not_encountered_white_space += 1

            if self.previous_printable_letter is None:
                self.previous_printable_letter = c
                continue

            is_accent = UnicodeRangeIdentify.is_accentuated(c)
            u_name = UnicodeRangeIdentify.find_letter_type(c)

            is_upper = c.isupper()
            is_lower = c.islower() if not is_upper else False
            is_alpha = c.isalpha()
            is_latin = UnicodeRangeIdentify.is_latin(c)

            if u_name is not None and u_name not in self.encountered_unicode_range:
                self.encountered_unicode_range_occurrences[u_name] = 0
                self.encountered_unicode_range.add(u_name)

            if is_accent and UnicodeRangeIdentify.is_accentuated(self.previous_printable_letter):
                self.successive_accent += 2

            if is_lower:
                self.total_lower_letter_encountered += 1

            if is_upper and is_accent:
                self.total_upper_accent_encountered += 1
                if self.previous_printable_letter.isalpha():
                    self.total_upper_accent_encountered_inner += 1
            elif not is_accent and is_alpha:
                self.total_unaccented_letter_encountered += 1

            if u_name is not None:
                self.encountered_unicode_range_occurrences[u_name] += 1

                is_punc = UnicodeRangeIdentify.is_punc(c)

                if is_punc is True:
                    self.encountered_punc_sign += 1
                    self.encountered_white_space += 1
                    self.not_encountered_white_space = 0
                    self.not_encountered_white_space_reset += 1
                    continue

                if (is_lower and self.previous_printable_letter.isupper()) or (is_upper and self.previous_printable_letter.islower()):
                    self.successive_upper_lower += 1

                if is_latin:
                    self.previous_encountered_unicode_range = u_name
                    self.previous_printable_letter = c

                if self.previous_encountered_unicode_range is not None and UnicodeRangeIdentify.is_suspiciously_successive_range(u_name, self.previous_encountered_unicode_range) is True:

                    if not UnicodeRangeIdentify.is_punc(self.previous_printable_letter):
                        self.successive_different_unicode_range += 1

            self.previous_encountered_unicode_range = u_name
            self.previous_printable_letter = c

        if len(self._string) < 50:
            self.not_encountered_white_space = 0
        if self.successive_upper_lower < 3:
            self.successive_upper_lower = 0

    @staticmethod
    def _unravel_cjk_suspicious_chinese(string, encountered_unicode_range_occurrences):

        encountered_unicode_range = encountered_unicode_range_occurrences.keys()

        if 'CJK Unified Ideographs' in encountered_unicode_range and ('Hiragana' not in encountered_unicode_range and 'Katakana' not in encountered_unicode_range):
            i_ = s_identify(string)
            if i_ in [MIXED, BOTH]:
                return encountered_unicode_range_occurrences['CJK Unified Ideographs']
            elif i_ != UNKNOWN and len(re.findall(cjc_sentence_re, string)) == 0:
                return encountered_unicode_range_occurrences['CJK Unified Ideographs']

        return UNKNOWN

    @property
    def ratio(self):
        """
        Return a value between 0. and 1.
        Closest to 1. means that the initial string is considered as chaotic,
        Closest to 0. means that the initial string SEEMS NOT chaotic.
        :return: Ratio as floating number
        :rtype: float
        """
        r_ = self.total_upper_accent_encountered if self.total_letter_encountered > 0 and self.total_unaccented_letter_encountered / self.total_letter_encountered < 0.5 else 0
        z_ = UnicodeRangeIdentify.unravel_suspicious_ranges(len(self._string), self.encountered_unicode_range_occurrences)
        p_ = self.encountered_punc_sign if self.encountered_punc_sign / len(self._string) > 0.2 else 0
        return (r_ + p_ + self.successive_upper_lower + self.successive_accent + self.successive_different_unicode_range + self.not_encountered_white_space + self.unprintable + z_ + ProbeChaos._unravel_cjk_suspicious_chinese.__func__(self._string, self.encountered_unicode_range_occurrences)) / len(self._string)  # + len(self.encountered_unicode_range)-1<commit_msg>Adjust Chaos Ratio with Probe Words<commit_after># coding: utf-8
import re
from functools import lru_cache

from dragonmapper.hanzi import MIXED, BOTH, UNKNOWN
from dragonmapper.hanzi import identify as s_identify
from zhon.hanzi import sentence as cjc_sentence_re

from charset_normalizer.probe_coherence import HashableCounter
from charset_normalizer.probe_words import ProbeWords
from charset_normalizer.unicode import UnicodeRangeIdentify


@lru_cache(maxsize=8192)
class ProbeChaos:

    def __init__(self, string, giveup_threshold=0.09):
        """
        :param str string:
        """

        if not isinstance(string, str):
            raise TypeError('Cannot probe chaos from type <{}>, expected <str>'.format(type(string)))

        self._string = string
        self._threshold = giveup_threshold

        self.successive_upper_lower = 0
        self.successive_accent = 0
        self.successive_different_unicode_range = 0
        self.encountered_unicode_range = set()
        self.encountered_punc_sign = 0
        self.unprintable = 0
        self.encountered_white_space = 0
        self.not_encountered_white_space = 0

        self.encountered_unicode_range_occurrences = dict()

        self.not_encountered_white_space_reset = 0

        self.previous_printable_letter = None

        self.previous_encountered_unicode_range = None

        self.total_letter_encountered = 0

        self.total_lower_letter_encountered = 0
        self.total_upper_accent_encountered = 0
        self.total_upper_accent_encountered_inner = 0
        self.total_unaccented_letter_encountered = 0

        self._probe_word = ProbeWords(HashableCounter(self._string.split()))

        self.gave_up = False

        if len(self._string) >= 10:
            self._probe()

    def __add__(self, other):
        """
        :param ProbeChaos other:
        :return:
        """
        k_ = ProbeChaos('', self._threshold)

        k_.successive_upper_lower = self.successive_upper_lower + other.successive_upper_lower
        k_.successive_accent = self.successive_accent + other.successive_accent
        k_.successive_different_unicode_range = self.successive_different_unicode_range + other.successive_different_unicode_range

        for el in self.encountered_unicode_range:
            k_.encountered_unicode_range.add(el)

        for el in other.encountered_unicode_range:
            k_.encountered_unicode_range.add(el)

        k_.encountered_punc_sign = self.encountered_punc_sign + other.encountered_punc_sign
        k_.unprintable = self.unprintable + other.unprintable
        k_.encountered_white_space = self.encountered_white_space + other.encountered_white_space
        k_.not_encountered_white_space = self.not_encountered_white_space + other.not_encountered_white_space

        for u_name, u_occ in self.encountered_unicode_range_occurrences.items():
            if u_name not in k_.encountered_unicode_range_occurrences.keys():
                k_.encountered_unicode_range_occurrences[u_name] = 0
            k_.encountered_unicode_range_occurrences[u_name] += u_occ

        for u_name, u_occ in other.encountered_unicode_range_occurrences.items():
            if u_name not in k_.encountered_unicode_range_occurrences.keys():
                k_.encountered_unicode_range_occurrences[u_name] = 0
            k_.encountered_unicode_range_occurrences[u_name] += u_occ

        k_.not_encountered_white_space_reset = self.not_encountered_white_space_reset + other.not_encountered_white_space_reset
        k_.total_letter_encountered = self.total_letter_encountered + other.total_letter_encountered
        k_.total_lower_letter_encountered = self.total_lower_letter_encountered + other.total_lower_letter_encountered
        k_.total_upper_accent_encountered = self.total_upper_accent_encountered + other.total_upper_accent_encountered
        k_.total_upper_accent_encountered_inner = self.total_upper_accent_encountered_inner + other.total_upper_accent_encountered_inner
        k_.total_unaccented_letter_encountered = self.total_unaccented_letter_encountered + other.total_unaccented_letter_encountered

        k_._probe_word = self._probe_word + other._probe_word

        k_._string = self._string + other._string

        return k_

    def _probe(self):

        c__ = False
        upper_lower_m = False

        for c, i_ in zip(self._string, range(0, len(self._string))):

            if not c__:
                state_ = (i_ / len(self._string) >= 0.5)

            # If we already have measured 10 % or more of chaos after reading 50 %, give up.
            if not c__ and state_ and self.ratio >= self._threshold:
                self.gave_up = True
                break
            elif c__ is False and state_:
                c__ = True

            self.total_letter_encountered += 1

            if not c.isprintable():
                if c not in ['\n', '\t', '\r']:
                    if not UnicodeRangeIdentify.is_cjk(c) and not UnicodeRangeIdentify.is_punc(c):
                        self.unprintable += 2

                self.encountered_white_space += 1
                self.not_encountered_white_space = 0
                self.not_encountered_white_space_reset += 1
                continue

            if c.isspace():
                self.encountered_white_space += 1
                self.not_encountered_white_space = 0
                self.not_encountered_white_space_reset += 1
                self.previous_printable_letter = c
                continue

            if self.not_encountered_white_space_reset < 2:
                self.not_encountered_white_space += 1

            if self.previous_printable_letter is None:
                self.previous_printable_letter = c
                continue

            is_accent = UnicodeRangeIdentify.is_accentuated(c)
            u_name = UnicodeRangeIdentify.find_letter_type(c)

            is_upper = c.isupper()
            is_lower = c.islower() if not is_upper else False
            is_alpha = c.isalpha()
            is_latin = UnicodeRangeIdentify.is_latin(c)

            if u_name is not None and u_name not in self.encountered_unicode_range:
                self.encountered_unicode_range_occurrences[u_name] = 0
                self.encountered_unicode_range.add(u_name)

            if is_accent and UnicodeRangeIdentify.is_accentuated(self.previous_printable_letter):
                self.successive_accent += 2

            if is_lower:
                self.total_lower_letter_encountered += 1

            if is_upper and is_accent:
                self.total_upper_accent_encountered += 1
                if self.previous_printable_letter.isalpha():
                    self.total_upper_accent_encountered_inner += 1
            elif not is_accent and is_alpha:
                self.total_unaccented_letter_encountered += 1

            if u_name is not None:
                self.encountered_unicode_range_occurrences[u_name] += 1

                is_punc = UnicodeRangeIdentify.is_punc(c)

                if is_punc is True:
                    self.encountered_punc_sign += 1
                    self.encountered_white_space += 1
                    self.not_encountered_white_space = 0
                    self.not_encountered_white_space_reset += 1
                    continue

                if (is_lower and self.previous_printable_letter.isupper()) or (is_upper and self.previous_printable_letter.islower()):
                    if not upper_lower_m:
                        upper_lower_m = True
                    else:
                        self.successive_upper_lower += 1
                        upper_lower_m = False
                else:
                    upper_lower_m = False

                if is_latin:
                    self.previous_encountered_unicode_range = u_name
                    self.previous_printable_letter = c

                if self.previous_encountered_unicode_range is not None and UnicodeRangeIdentify.is_suspiciously_successive_range(u_name, self.previous_encountered_unicode_range) is True:

                    if not UnicodeRangeIdentify.is_punc(self.previous_printable_letter):
                        self.successive_different_unicode_range += 1

            self.previous_encountered_unicode_range = u_name
            self.previous_printable_letter = c

        if len(self._string) < 50:
            self.not_encountered_white_space = 0
        if self.successive_upper_lower < 3:
            self.successive_upper_lower = 0

    @staticmethod
    def _unravel_cjk_suspicious_chinese(string, encountered_unicode_range_occurrences):
        if len(string) <= 10:
            return UNKNOWN

        encountered_unicode_range = encountered_unicode_range_occurrences.keys()

        if 'CJK Unified Ideographs' in encountered_unicode_range and ('Hiragana' not in encountered_unicode_range and 'Katakana' not in encountered_unicode_range):
            i_ = s_identify(string)
            if i_ in [MIXED, BOTH]:
                return encountered_unicode_range_occurrences['CJK Unified Ideographs']
            elif i_ != UNKNOWN and len(re.findall(cjc_sentence_re, string)) > 0:
                return -encountered_unicode_range_occurrences['CJK Unified Ideographs']
            elif i_ != UNKNOWN:
                return int(encountered_unicode_range_occurrences['CJK Unified Ideographs']*0.3)

        return UNKNOWN

    @property
    def ratio(self):
        """
        Return a value between 0. and 1.
        Closest to 1. means that the initial string is considered as chaotic,
        Closest to 0. means that the initial string SEEMS NOT chaotic.
        :return: Ratio as floating number
        :rtype: float
        """
        r_ = self.total_upper_accent_encountered if self.total_letter_encountered > 0 and self.total_unaccented_letter_encountered / self.total_letter_encountered < 0.5 else 0
        z_ = UnicodeRangeIdentify.unravel_suspicious_ranges(len(self._string), self.encountered_unicode_range_occurrences)
        p_ = self.encountered_punc_sign if self.encountered_punc_sign / len(self._string) > 0.2 else 0
        return ((r_ + p_ + self.successive_upper_lower + self.successive_accent + self.successive_different_unicode_range + self.not_encountered_white_space + self.unprintable + z_ + ProbeChaos._unravel_cjk_suspicious_chinese.__func__(self._string, self.encountered_unicode_range_occurrences)) / len(self._string)) + self._probe_word.ratio  # + len(self.encountered_unicode_range)-1
{'fraction_non_alphanumeric': 0.06306457947595803, 'fraction_numerical': 0.008303877012848567, 'mean_word_length': 2.404125286478228, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4491592', 'n_tokens_mistral': 5688, 'n_tokens_neox': 5132, 'n_words': 1023}