<commit_before># -*- coding: utf-8 -*-

#  Copyright 2014 <NAME>
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import os
import re

from pylint import lint


class Symbolizer(object):

    def __init__(
        self, start_location=os.getcwd(), column_width=79, handle_inline=False
    ):
        self.start_location = start_location
        self.column_width = column_width
        self.handle_inline = handle_inline

        self._pylint_disable = '# pylint: disable='
        self._mapping = self._initalize_mapping()
        self._pattern = re.compile(
            r'\b(' + '|'.join(self._mapping.keys()) + r')\b'
        )
        self._leading_whitespace = 0

    @staticmethod
    def _initalize_mapping():
        """
        Initialize Pylint with defaults and any plugins.

        Returns a mapping to be used by regex for finding and replacing
        Pylint ids with Pylint symbols

        :rtpe: dict
        """
        linter = lint.PyLinter()
        linter.load_defaults()
        linter.load_default_plugins()

        mapping = {}
        for symbol, msg in linter._messages.iteritems():
            mapping[msg.msgid] = symbol

        return mapping

    def __call__(self):
        self.perform_symbolization()

    def _get_files(self):
        """
        Generates a list of files with extension in the directory structure

        :yields: str (filepath)
        """
        for dirpath, __, filenames in os.walk(self.start_location):
            for file_ in filenames:
                if file_.endswith('.py'):
                    yield "{0}/{1}".format(dirpath, file_)

    def perform_symbolization(self):  # pragma: no cover
        """
        Cycle through python files, reading the contents and processing each
        line and writing out to the same file
        """
        for filename in self._get_files():
            print u"Processing file -- {0}".format(filename)
            updated_file_text = u''
            with open(filename, 'r') as fin:
                for line in fin.readlines():
                    new_line = self.replace_id_with_symbol(line)

                    if not updated_file_text:
                        updated_file_text = new_line
                    else:
                        updated_file_text += new_line

            with open(filename, 'w') as fout:
                fout.write(updated_file_text)

    def replace_id_with_symbol(self, line):
        """
        Given a line, check to see if any of the Pylint message ids are
        in the line. If it is replace with the Pylint symbolic name.

        If the line has been changed verify the length is within the standard
        monitor width

        :param line: A line of text
        :param type: str

        :rtype: str
        """
        new_line = self._pattern.sub(lambda x: self._mapping[x.group()], line)
        if line != new_line:
            if self.handle_inline:
                self._set_leading_whitespace(new_line)
            line = self._check_line_length(new_line)

            self._reset_leading_whitespace()

        return line

    def _check_line_length(self, line):
        """
        If the line is within the given character count, return it unchanged.
        Else split the line on `,` (the standard delimiter for multiple
        disabled arguments for Pylint) and build a string verifying that it
        is still within the given column width.

        Recursively calls itself until the line has been fully processed

        :param line: A line of tet
        :rtype: str
        """
        if len(line) < self.column_width:
            return line
        else:
            count = 0
            first_line_lst = []
            second_line_lst = []
            for item in line.split(','):
                if count + len(item) < self.column_width:
                    first_line_lst.append(item)
                    count += len(item)
                else:
                    second_line_lst.append(item)

            if first_line_lst and second_line_lst:
                return self._build_stuff(first_line_lst, second_line_lst)

    def _build_stuff(self, first_line_lst, second_line_lst):
        new_line = ','.join(first_line_lst)

        second_line_lst = self._insert_pylint_disable(second_line_lst)
        second_line_lst = self._insert_leading_whitespace(
            second_line_lst)

        second_line = self._check_line_length(','.join(second_line_lst))
        second_line = second_line.replace(u'disable=,', 'disable=')
        new_line = '{0}\n{1}'.format(
            new_line, second_line
        )

        return new_line

    def _insert_pylint_disable(self, line_list):
        """
        Checks if `# pylint: disable=` is in the first argument in the list.
        If it is not, inserts the above in the first index

        :param line_list: List of strings
        :param type: list
        :rtype: list
        """
        if not line_list[0].startswith(self._pylint_disable):
            line_list.insert(0, self._pylint_disable)

        return line_list

    def _insert_leading_whitespace(self, line_list):
        """
        If the handle_inline flag has been set and we have calculated
        leading whitespace, insert the leading whitespace into the first
        position of the list

        :param line_list: List of string
        :param type: list
        :rtpe: list
        """
        if self.handle_inline and self._leading_whitespace:
            line_list[0] = '{0}{1}'.format(
                self._leading_whitespace, line_list[0]
            )

        return line_list

    def _set_leading_whitespace(self, line):
        """
        Get the leading whitespace and append either four spaces to the
        whitespace or a tab for the next logical line

        :param line: A string to with leading whitespace
        :rtype: str
        """
        whitespace = u''
        indentation = u''

        if line.index(self._pylint_disable):
            indentation = ' ' * 4
            whitespace = line[:-len(line.lstrip())]
            if '\t' in whitespace:
                indentation = '\t'

        self._leading_whitespace = whitespace + indentation

    def _reset_leading_whitespace(self):
        self._leading_whitespace = 0<commit_msg>Handling inline disables and fix bug with "disable=,"<commit_after># -*- coding: utf-8 -*-

#  Copyright 2014 <NAME>
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.

import os
import re

from pylint import lint


class Symbolizer(object):

    def __init__(
        self, start_location=os.getcwd(), column_width=79, handle_inline=False
    ):
        self.start_location = start_location
        self.column_width = column_width
        self.handle_inline = handle_inline

        self._pylint_disable = u'# pylint: disable='
        self._mapping = self._initalize_mapping()
        self._pattern = re.compile(
            ur'\b(' + '|'.join(self._mapping.keys()) + ur')\b'
        )
        self._leading_whitespace = 0

    @staticmethod
    def _initalize_mapping():
        """
        Initialize Pylint with defaults and any plugins.

        Returns a mapping to be used by regex for finding and replacing
        Pylint ids with Pylint symbols

        :rtpe: dict
        """
        linter = lint.PyLinter()
        linter.load_defaults()
        linter.load_default_plugins()

        mapping = {}
        for symbol, msg in linter._messages.iteritems():
            mapping[msg.msgid] = symbol

        return mapping

    def __call__(self):
        self.perform_symbolization()

    def _get_files(self):
        """
        Generates a list of files with extension in the directory structure

        :yields: str (filepath)
        """
        for dirpath, __, filenames in os.walk(self.start_location):
            for file_ in filenames:
                if file_.endswith(u'.py'):
                    yield "{0}/{1}".format(dirpath, file_)

    def perform_symbolization(self):  # pragma: no cover
        """
        Cycle through python files, reading the contents and processing each
        line and writing out to the same file
        """
        for filename in self._get_files():
            print u"Processing file -- {0}".format(filename)
            updated_file_text = u''
            with open(filename, 'r') as fin:
                for line in fin.readlines():
                    new_line = self.replace_id_with_symbol(line)

                    if not updated_file_text:
                        updated_file_text = new_line
                    else:
                        updated_file_text += new_line

            with open(filename, 'w') as fout:
                fout.write(updated_file_text)

    def replace_id_with_symbol(self, line):
        """
        Given a line, check to see if any of the Pylint message ids are
        in the line. If it is replace with the Pylint symbolic name.

        If the line has been changed verify the length is within the standard
        monitor width

        :param line: A line of text
        :param type: str

        :rtype: str
        """
        new_line = self._pattern.sub(lambda x: self._mapping[x.group()], line)
        if line != new_line:
            if self.handle_inline:
                self._set_leading_whitespace(new_line)
            line = self._check_line_length(new_line)

            self._reset_leading_whitespace()

        return line

    def _check_line_length(self, line):
        """
        If the line is within the given character count, return it unchanged.
        Else split the line on `,` (the standard delimiter for multiple
        disabled arguments for Pylint) and build a string verifying that it
        is still within the given column width.

        Recursively calls itself until the line has been fully processed

        :param line: A line of tet
        :rtype: str
        """
        if len(line) < self.column_width:
            return line
        else:
            count = 0
            first_line_lst = []
            second_line_lst = []
            for item in line.split(u','):
                if count + len(item) < self.column_width:
                    first_line_lst.append(item)
                    count += len(item)
                else:
                    second_line_lst.append(item)

            if first_line_lst and second_line_lst:
                return self._build_stuff(first_line_lst, second_line_lst)

    def _build_stuff(self, first_line_lst, second_line_lst):
        new_line = u','.join(first_line_lst)

        second_line_lst = self._insert_pylint_disable(second_line_lst)
        second_line_lst = self._insert_leading_whitespace(
            second_line_lst)

        second_line = self._check_line_length(','.join(second_line_lst))
        second_line = second_line.replace(u'disable=,', 'disable=')
        new_line = u'{0}\n{1}'.format(
            new_line, second_line
        )

        return new_line

    def _insert_pylint_disable(self, line_list):
        """
        Checks if `# pylint: disable=` is in the first argument in the list.
        If it is not, inserts the above in the first index

        :param line_list: List of strings
        :param type: list
        :rtype: list
        """
        if not line_list[0].startswith(self._pylint_disable):
            line_list.insert(0, self._pylint_disable)

        return line_list

    def _insert_leading_whitespace(self, line_list):
        """
        If the handle_inline flag has been set and we have calculated
        leading whitespace, insert the leading whitespace into the first
        position of the list

        :param line_list: List of string
        :param type: list
        :rtpe: list
        """
        if self.handle_inline and self._leading_whitespace:
            line_list[0] = u'{0}{1}'.format(
                self._leading_whitespace, line_list[0]
            )

        return line_list

    def _set_leading_whitespace(self, line):
        """
        Get the leading whitespace and append either four spaces to the
        whitespace or a tab for the next logical line

        :param line: A string to with leading whitespace
        :rtype: str
        """
        whitespace = u''
        indentation = u''

        if line.index(self._pylint_disable):
            indentation = u' ' * 4
            whitespace = line[:-len(line.lstrip())]
            if u'\t' in whitespace:
                indentation = u'\t'

        self._leading_whitespace = whitespace + indentation

    def _reset_leading_whitespace(self):
        self._leading_whitespace = 0
{'fraction_non_alphanumeric': 0.0794361647456134, 'fraction_numerical': 0.003817634534909331, 'mean_word_length': 2.0325022261798753, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1518752', 'n_tokens_mistral': 3851, 'n_tokens_neox': 3712, 'n_words': 1172}