<commit_before>"""Stuff that differs in different Python versions and platform
distributions."""
from __future__ import absolute_import

# flake8: noqa

import os
import imp
import sys
import site

uses_pycache = hasattr(imp, 'cache_from_source')


class NeverUsedException(Exception):
    """this exception should never be raised"""

try:
    #new in Python 3.3
    PermissionError = PermissionError
except NameError:
    PermissionError = NeverUsedException

console_encoding = sys.__stdout__.encoding


try:
    from logging.config import dictConfig as logging_dictConfig
except ImportError:
    from pip.compat.dictconfig import dictConfig as logging_dictConfig


if sys.version_info >= (3,):
    from io import StringIO
    from urllib.error import URLError, HTTPError
    from urllib.request import url2pathname, urlretrieve, pathname2url
    import urllib.parse as urllib
    import urllib.request as urllib2
    import urllib.parse as urlparse

    def cmp(a, b):
        return (a > b) - (a < b)

    def console_to_str(s):
        try:
            return s.decode(console_encoding)
        except UnicodeDecodeError:
            return s.decode('utf_8')

    def native_str(s, replace=False):
        if isinstance(s, bytes):
            return s.decode('utf-8', 'replace' if replace else 'strict')
        return s<commit_msg>Removed the PermissionError exception as it was never actually used<commit_after>"""Stuff that differs in different Python versions and platform
distributions."""
from __future__ import absolute_import

# flake8: noqa

import os
import imp
import sys
import site

uses_pycache = hasattr(imp, 'cache_from_source')


console_encoding = sys.__stdout__.encoding


try:
    from logging.config import dictConfig as logging_dictConfig
except ImportError:
    from pip.compat.dictconfig import dictConfig as logging_dictConfig


if sys.version_info >= (3,):
    from io import StringIO
    from urllib.error import URLError, HTTPError
    from urllib.request import url2pathname, urlretrieve, pathname2url
    import urllib.parse as urllib
    import urllib.request as urllib2
    import urllib.parse as urlparse

    def cmp(a, b):
        return (a > b) - (a < b)

    def console_to_str(s):
        try:
            return s.decode(console_encoding)
        except UnicodeDecodeError:
            return s.decode('utf_8')

    def native_str(s, replace=False):
        if isinstance(s, bytes):
            return s.decode('utf-8', 'replace' if replace else 'strict')
        return s
{'fraction_non_alphanumeric': 0.07352941176470588, 'fraction_numerical': 0.006359300476947536, 'mean_word_length': 3.4003496503496504, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '200049', 'n_tokens_mistral': 751, 'n_tokens_neox': 729, 'n_words': 239}