<commit_before>import logging
from editorconfig import get_properties, EditorConfigError


class EditorConfigPluginMixin(object):

    HANDLER_NAME = 'EditorConfigPluginHandlerId'

    def activate_plugin(self, window):
        handler_id = window.connect('active_tab_state_changed', self.set_config)
        window.set_data(self.HANDLER_NAME, handler_id)

    def deactivate_plugin(self, window):
        handler_id = window.get_data(self.HANDLER_NAME)
        window.disconnect(handler_id)
        window.set_data(self.HANDLER_NAME, None)

    def set_config(self, window):
        """Get EditorConfig properties for file and change settings"""

        tab = window.get_active_tab()
        document = tab.get_document()
        view = tab.get_view()

        props = self.get_document_properties(document)
        self.process_properties(props)
        self.set_indentation(view,
                             props.get('indent_style'),
                             props.get('indent_size'),
                             props.get('tab_width'))
        self.set_end_of_line(document, props.get('end_of_line'))

    def get_properties_from_filename(self, filename):
        """Retrieve dict of EditorConfig properties for the given filename"""
        try:
            return get_properties(filename)
        except EditorConfigError as e:
            logging.error("Error reading EditorConfig file", exc_info=True)
            return {}

    def process_properties(self, properties):
        """Process property values and remove invalid properties"""

        # Convert tab_width to a number
        if 'tab_width' in properties:
            try:
                properties['tab_width'] = int(properties['tab_width'])
            except ValueError:
                del properties['tab_width']

        # Convert indent_size to a number or set equal to tab_width
        if 'indent_size' in properties:
            if properties['indent_size'] == "tab" and 'tab_width' in properties:
                properties['indent_size'] = properties['tab_width']
            else:
                try:
                    properties['indent_size'] = int(properties['indent_size'])
                except ValueError:
                    del properties['indent_size']

    def set_end_of_line(self, document, end_of_line):
        """Set line ending style based on given end_of_line property"""
        if end_of_line == "lf":
            document.set_property('newline-type', 0)
        elif end_of_line == "cr":
            document.set_property('newline-type', 1)
        elif end_of_line == "crlf":
            document.set_property('newline-type', 2)

    def set_indentation(self, view, indent_style, indent_size, tab_width):
        """Set indentation style for given view based on given properties"""

        if indent_style == 'space':
            view.set_insert_spaces_instead_of_tabs(True)
            if indent_size:
                view.set_tab_width(indent_size)
        elif indent_style == 'tab':
            view.set_insert_spaces_instead_of_tabs(False)
            if tab_width:
                view.set_tab_width(tab_width)<commit_msg>Handle trim_trailing_whitespace property on save<commit_after>import logging
from editorconfig import get_properties, EditorConfigError


class EditorConfigPluginMixin(object):

    HANDLER_NAME = 'EditorConfigPluginHandlerId'
    WHITESPACE_HANDLER_NAME = 'EditorConfigPluginTrimWhitespace'

    def activate_plugin(self, window):
        handler_id = window.connect('active_tab_state_changed', self.set_config)
        window.set_data(self.HANDLER_NAME, handler_id)

    def deactivate_plugin(self, window):
        handler_id = window.get_data(self.HANDLER_NAME)
        window.disconnect(handler_id)
        window.set_data(self.HANDLER_NAME, None)
        for document in window.get_documents():
            handler_id = document.get_data(self.WHITESPACE_HANDLER_NAME)
            if handler_id is not None:
                document.disconnect(handler_id)
                document.set_data(self.HANDLER_NAME, None)

    def set_config(self, window):
        """Get EditorConfig properties for file and change settings"""

        tab = window.get_active_tab()
        document = tab.get_document()
        view = tab.get_view()

        props = self.get_document_properties(document)
        self.process_properties(props)
        self.set_indentation(view,
                             props.get('indent_style'),
                             props.get('indent_size'),
                             props.get('tab_width'))
        self.set_end_of_line(document, props.get('end_of_line'))
        self.set_trim_trailing_whitespace(document,
                props.get('trim_trailing_whitespace'))

    def get_properties_from_filename(self, filename):
        """Retrieve dict of EditorConfig properties for the given filename"""
        try:
            return get_properties(filename)
        except EditorConfigError as e:
            logging.error("Error reading EditorConfig file", exc_info=True)
            return {}

    def process_properties(self, properties):
        """Process property values and remove invalid properties"""

        # Convert tab_width to a number
        if 'tab_width' in properties:
            try:
                properties['tab_width'] = int(properties['tab_width'])
            except ValueError:
                del properties['tab_width']

        # Convert indent_size to a number or set equal to tab_width
        if 'indent_size' in properties:
            if properties['indent_size'] == "tab" and 'tab_width' in properties:
                properties['indent_size'] = properties['tab_width']
            else:
                try:
                    properties['indent_size'] = int(properties['indent_size'])
                except ValueError:
                    del properties['indent_size']

        if properties.get('trim_trailing_whitespace') == 'true':
            properties['trim_trailing_whitespace'] = True
        else:
            properties['trim_trailing_whitespace'] = False

    def set_end_of_line(self, document, end_of_line):
        """Set line ending style based on given end_of_line property"""
        if end_of_line == "lf":
            document.set_property('newline-type', 0)
        elif end_of_line == "cr":
            document.set_property('newline-type', 1)
        elif end_of_line == "crlf":
            document.set_property('newline-type', 2)

    def set_indentation(self, view, indent_style, indent_size, tab_width):
        """Set indentation style for given view based on given properties"""

        if indent_style == 'space':
            view.set_insert_spaces_instead_of_tabs(True)
            if indent_size:
                view.set_tab_width(indent_size)
        elif indent_style == 'tab':
            view.set_insert_spaces_instead_of_tabs(False)
            if tab_width:
                view.set_tab_width(tab_width)

    def set_trim_trailing_whitespace(self, document, trim_trailing_whitespace):
        """Create/delete file save handler for trimming trailing whitespace"""
        def trim_whitespace_on_save(document, *args):
            document.begin_user_action()
            self.trim_trailing_whitespace(document)
            document.end_user_action()
        handler_id = document.get_data(self.WHITESPACE_HANDLER_NAME)
        if trim_trailing_whitespace:
            # If no handler when trim_trailing_whitespace is "true", create one
            if handler_id is None:
                handler_id = document.connect('save', trim_whitespace_on_save)
                document.set_data(self.WHITESPACE_HANDLER_NAME, handler_id)
        elif handler_id is not None:
            # Disconnect handler when trim_trailing_whitespace is "false"
            document.disconnect(handler_id)
            document.set_data(self.WHITESPACE_HANDLER_NAME, None)

    def trim_trailing_whitespace(self, document):
        """Trim trailing whitespace from each line of document"""
        for line in range(document.get_end_iter().get_line() + 1):
            end_of_line = document.get_iter_at_line(line)
            end_of_line.forward_to_line_end()
            whitespace_start = end_of_line.copy()
            while whitespace_start.backward_char():
                if not whitespace_start.get_char() in ' \t':
                    whitespace_start.forward_char()
                    break
            if not whitespace_start.equal(end_of_line):
                document.delete(whitespace_start, end_of_line)
{'fraction_non_alphanumeric': 0.08545709609539397, 'fraction_numerical': 0.0008183306055646482, 'mean_word_length': 2.4719967532467533, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3490578', 'n_tokens_mistral': 2387, 'n_tokens_neox': 2304, 'n_words': 504}