<commit_before>import re
import six
import subprocess

import sys


class CommandError(Exception):
    """
    Error raised when a command being excuted returns an error
    """
    pass


def execute(command):
    """Execute provided command returning the stdout
    Args:
        command (list[str]): list of tokens to execute as your command.
        subprocess_mod (module): Defaults to pythons subprocess module but you can optionally pass in
        another. This is mostly for testing purposes
    Returns:
        str - Stdout of the command passed in. This will be Unicode for python < 3. Str for python 3
    Raises:
        ValueError if there is a error running the command
    """
    stdout_pipe = subprocess.PIPE
    process = subprocess.Popen(
        command, stdout=stdout_pipe,
        stderr=stdout_pipe
    )
    try:
        stdout, stderr = process.communicate()
    except OSError:
        sys.stderr.write(" ".join(
                [cmd.decode(sys.getfilesystemencoding())
                 if isinstance(cmd, bytes) else cmd
                 for cmd in command])
        )
        raise

    stderr = _ensure_unicode(stderr)
    # after version 1.8.0 pylint writes a message to stderr:
    # Using config file {}
    stderr = re.sub(r'^Using config file .*$', '', stderr).strip()
    # If we get a non-empty output to stderr, raise an exception
    if bool(stderr) and process.returncode:
        raise CommandError(stderr)

    return _ensure_unicode(stdout), stderr


def run_command_for_code(command):
    """
    Returns command's exit code.
    """
    process = subprocess.Popen(
        command, stdout=subprocess.PIPE, stderr=subprocess.PIPE
    )
    process.communicate()
    exit_code = process.returncode
    return exit_code


def _ensure_unicode(text):
    """
    Ensures the text passed in becomes unicode
    Args:
        text (str|unicode)
    Returns:
        unicode
    """
    if isinstance(text, six.binary_type):
        return text.decode(sys.getfilesystemencoding(), 'replace')
    else:<commit_msg>Fix pylint 1.8.0 compatibility<commit_after>import re
import six
import subprocess

import sys


class CommandError(Exception):
    """
    Error raised when a command being excuted returns an error
    """
    pass


def execute(command):
    """Execute provided command returning the stdout
    Args:
        command (list[str]): list of tokens to execute as your command.
        subprocess_mod (module): Defaults to pythons subprocess module but you can optionally pass in
        another. This is mostly for testing purposes
    Returns:
        str - Stdout of the command passed in. This will be Unicode for python < 3. Str for python 3
    Raises:
        ValueError if there is a error running the command
    """
    stdout_pipe = subprocess.PIPE
    process = subprocess.Popen(
        command, stdout=stdout_pipe,
        stderr=stdout_pipe
    )
    try:
        stdout, stderr = process.communicate()
    except OSError:
        sys.stderr.write(" ".join(
                [cmd.decode(sys.getfilesystemencoding())
                 if isinstance(cmd, bytes) else cmd
                 for cmd in command])
        )
        raise

    stderr = _ensure_unicode(stderr)
    # after version 1.8.0 pylint writes a message to stderr:
    # Using config file {}
    # OR
    # No config file found, using default configuration
    stderr = re.sub(r'^Using config file .*$', '', stderr).strip()
    stderr = re.sub(r'^No config file found, using default configuration$', '', stderr).strip()
    # If we get a non-empty output to stderr, raise an exception
    if bool(stderr) and process.returncode:
        raise CommandError(stderr)

    return _ensure_unicode(stdout), stderr


def run_command_for_code(command):
    """
    Returns command's exit code.
    """
    process = subprocess.Popen(
        command, stdout=subprocess.PIPE, stderr=subprocess.PIPE
    )
    process.communicate()
    exit_code = process.returncode
    return exit_code


def _ensure_unicode(text):
    """
    Ensures the text passed in becomes unicode
    Args:
        text (str|unicode)
    Returns:
        unicode
    """
    if isinstance(text, six.binary_type):
        return text.decode(sys.getfilesystemencoding(), 'replace')
    else:
{'fraction_non_alphanumeric': 0.0838785046728972, 'fraction_numerical': 0.0030373831775700934, 'mean_word_length': 2.5824267782426777, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6316636', 'n_tokens_mistral': 1234, 'n_tokens_neox': 1173, 'n_words': 401}