<commit_before>from coalib.misc.StringConverter import StringConverter


class LineParser:
    def __init__(self,
                 key_value_delimiters=['=', ':'],
                 comment_seperators=['#', ';', '//'],
                 key_delimiters=[',', ' '],
                 section_name_surroundings={'[': "]"},
                 section_override_delimiters=["."]):
        """
        Creates a new line parser. Please note that no delimiter or seperator
        may be an "o" or you may encounter undefined behaviour with the
        escapes.

        :param key_value_delimiters:        Delimiters that delimit a key from
                                            a value
        :param comment_seperators:          Used to initiate a comment
        :param key_delimiters:              Delimiters between several keys
        :param section_name_surroundings:   Dictionary, e.g. {"[", "]"} means a
                                            section name is surrounded by [].
        :param section_override_delimiters: Delimiter for a section override.
                                            E.g. "." would mean that
                                            section.key is a possible key that
                                            puts the key into the section
                                            "section" despite of the current
                                            section.
        """
        self.key_value_delimiters = key_value_delimiters
        self.comment_seperators = comment_seperators
        self.key_delimiters = key_delimiters
        self.section_name_surroundings = section_name_surroundings
        self.section_override_delimiters = section_override_delimiters

    def parse(self, line):
        """
        Note that every value in the returned touple _besides the value_ is
        unescaped. This is so since the value is meant to be put into a Setting
        later thus the escapes may be needed there.

        :param line: the line to parse
        :return:     section_name (empty string if it's no section name),
                     [(section_override, key), ...], value, comment
        """
        line, comment = self.__seperate_by_first_occurrence(
            line,
            self.comment_seperators)
        comment = self.remove_backslashes(comment)
        if line == "":
            return '', [], '', comment

        section_name = self.remove_backslashes(self.__get_section_name(line))
        if section_name != '':
            return section_name, [], '', comment

        # Escapes in value might be needed by the bears
        keys, value = self.__extract_keys_and_value(line)

        key_touples = []
        for key in keys:
            section, key = self.__seperate_by_first_occurrence(
                key,
                self.section_override_delimiters,
                True,
                True)
            key_touples.append((self.remove_backslashes(section),
                                self.remove_backslashes(key)))

        return '', key_touples, value, comment

    @staticmethod
    def remove_backslashes(string):
        i = string.find("\\")
        while i != -1:
            string = string[:i] + string[i+1:]
            i = string.find("\\", i+1)  # Dont check the next char

        return string

    @staticmethod
    def __seperate_by_first_occurrence(string,
                                       delimiters,
                                       strip_delim=False,
                                       return_second_part_nonempty=False):
        """
        Seperates a string by the first of all given delimiters. Any whitespace
        characters will be stripped away from the parts.
<commit_msg>LineParser: Use unescape from StringProcessing<commit_after>from coalib.misc.StringConverter import StringConverter
from coalib.parsing.StringProcessing import unescape


class LineParser:
    def __init__(self,
                 key_value_delimiters=['=', ':'],
                 comment_seperators=['#', ';', '//'],
                 key_delimiters=[',', ' '],
                 section_name_surroundings={'[': "]"},
                 section_override_delimiters=["."]):
        """
        Creates a new line parser. Please note that no delimiter or seperator
        may be an "o" or you may encounter undefined behaviour with the
        escapes.

        :param key_value_delimiters:        Delimiters that delimit a key from
                                            a value
        :param comment_seperators:          Used to initiate a comment
        :param key_delimiters:              Delimiters between several keys
        :param section_name_surroundings:   Dictionary, e.g. {"[", "]"} means a
                                            section name is surrounded by [].
        :param section_override_delimiters: Delimiter for a section override.
                                            E.g. "." would mean that
                                            section.key is a possible key that
                                            puts the key into the section
                                            "section" despite of the current
                                            section.
        """
        self.key_value_delimiters = key_value_delimiters
        self.comment_seperators = comment_seperators
        self.key_delimiters = key_delimiters
        self.section_name_surroundings = section_name_surroundings
        self.section_override_delimiters = section_override_delimiters

    def parse(self, line):
        """
        Note that every value in the returned touple _besides the value_ is
        unescaped. This is so since the value is meant to be put into a Setting
        later thus the escapes may be needed there.

        :param line: the line to parse
        :return:     section_name (empty string if it's no section name),
                     [(section_override, key), ...], value, comment
        """
        line, comment = self.__seperate_by_first_occurrence(
            line,
            self.comment_seperators)
        comment = unescape(comment)
        if line == "":
            return '', [], '', comment

        section_name = unescape(self.__get_section_name(line))
        if section_name != '':
            return section_name, [], '', comment

        # Escapes in value might be needed by the bears
        keys, value = self.__extract_keys_and_value(line)

        key_touples = []
        for key in keys:
            section, key = self.__seperate_by_first_occurrence(
                key,
                self.section_override_delimiters,
                True,
                True)
            key_touples.append((unescape(section), unescape(key)))

        return '', key_touples, value, comment

    @staticmethod
    def __seperate_by_first_occurrence(string,
                                       delimiters,
                                       strip_delim=False,
                                       return_second_part_nonempty=False):
        """
        Seperates a string by the first of all given delimiters. Any whitespace
        characters will be stripped away from the parts.

{'fraction_non_alphanumeric': 0.07256833633966976, 'fraction_numerical': 0.00041626196753156655, 'mean_word_length': 1.5853658536585367, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 34, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3281205', 'n_tokens_mistral': 1776, 'n_tokens_neox': 1716, 'n_words': 513}