<commit_before># -*- coding: utf-8 -*-
from __future__ import print_function
from nose.suite import ContextList
import re
import sys
import os
import codecs
import doctest
from nose.plugins.base import Plugin
from nose.util import tolist, anyp
from nose.plugins.doctests import Doctest, log, DocFileCase

ALLOW_UNICODE = doctest.register_optionflag('ALLOW_UNICODE')


class _UnicodeOutputChecker(doctest.OutputChecker):
    _literal_re = re.compile(r"(\W|^)[uU]([rR]?[\'\"])", re.UNICODE)

    def _remove_u_prefixes(self, txt):
        return re.sub(self._literal_re, r'\1\2', txt)

    def check_output(self, want, got, optionflags):
        res = doctest.OutputChecker.check_output(self, want, got, optionflags)
        if res:
            return True
        if not (optionflags & ALLOW_UNICODE):
            return False

        # ALLOW_UNICODE is active and want != got
        cleaned_want = self._remove_u_prefixes(want)
        cleaned_got = self._remove_u_prefixes(got)
        res = doctest.OutputChecker.check_output(self, cleaned_want, cleaned_got, optionflags)
        return res


_checker = _UnicodeOutputChecker()

<commit_msg>pylint cleanup<commit_after># -*- coding: utf-8 -*-
from __future__ import print_function
import re
import sys
import os
import codecs
import doctest
from nose.util import tolist, anyp
from nose.plugins.base import Plugin
from nose.suite import ContextList
from nose.plugins.doctests import Doctest, log, DocFileCase

ALLOW_UNICODE = doctest.register_optionflag('ALLOW_UNICODE')


class _UnicodeOutputChecker(doctest.OutputChecker):
    _literal_re = re.compile(r"(\W|^)[uU]([rR]?[\'\"])", re.UNICODE)

    def _remove_u_prefixes(self, txt):
        return re.sub(self._literal_re, r'\1\2', txt)

    def check_output(self, want, got, optionflags):
        res = doctest.OutputChecker.check_output(self, want, got, optionflags)
        if res:
            return True
        if not (optionflags & ALLOW_UNICODE):
            return False

        # ALLOW_UNICODE is active and want != got
        cleaned_want = self._remove_u_prefixes(want)
        cleaned_got = self._remove_u_prefixes(got)
        res = doctest.OutputChecker.check_output(self, cleaned_want, cleaned_got, optionflags)
        return res


_checker = _UnicodeOutputChecker()


{'fraction_non_alphanumeric': 0.09777389786119599, 'fraction_numerical': 0.0026189436927106066, 'mean_word_length': 3.639676113360324, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5264573', 'n_tokens_mistral': 751, 'n_tokens_neox': 740, 'n_words': 179}