<commit_before># -*- coding: utf-8 -*-

import os
import re
import subprocess
import sys
import threading
import time
import tempfile
import platform

from nimp.utilities.logging import *

if os.name is "nt":
    from nimp.utilities.windows_utilities import *

#-------------------------------------------------------------------------------
def _default_log_callback(line, default_log_function):
    default_log_function(line)

#-------------------------------------------------------------------------------
def _sanitize_command(command):
    # Perforce paths should be immune
    if command[0].lower() in ['p4', 'p4.exe']:
        return command

    # If we’re running under MSYS, leading slashes in command line arguments
    # will be treated as a path, so we need to escape them
    if os.environ.get('MSYSTEM') == 'MSYS':
        return [re.sub('^/', '//', x) for x in command]

    return command

#-------------------------------------------------------------------------------
def capture_process_output(directory, command, input = None):
<commit_msg>Don't sanitize command line argument that are indeed a file<commit_after># -*- coding: utf-8 -*-

import os
import re
import subprocess
import sys
import threading
import time
import tempfile
import platform

from nimp.utilities.logging import *

if os.name is "nt":
    from nimp.utilities.windows_utilities import *

#-------------------------------------------------------------------------------
def _default_log_callback(line, default_log_function):
    default_log_function(line)

#-------------------------------------------------------------------------------
def _sanitize_command(command):
    # Perforce paths should be immune
    if command[0].lower() in ['p4', 'p4.exe']:
        return command

    # If we’re running under MSYS, leading slashes in command line arguments
    # will be treated as a path, so we need to escape them, except if the given
    # argument is indeed a file
    if os.environ.get('MSYSTEM') == 'MSYS':
        sanitized_command = []
        for x in command:
            if os.path.isfile(x):
                sanitized_command += [x]
            else:
                sanitized_command += [re.sub('^/', '//', x)]
        return sanitized_command

    return command

#-------------------------------------------------------------------------------
def capture_process_output(directory, command, input = None):

{'fraction_non_alphanumeric': 0.2755905511811024, 'fraction_numerical': 0.003315375051802735, 'mean_word_length': 1.590128755364807, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2332134', 'n_tokens_mistral': 605, 'n_tokens_neox': 594, 'n_words': 200}