<commit_before>import sys
import os
import shutil
import glob
import socket
import hashlib
import time
import uuid as libuuid

PY3 = sys.version_info >= (3,)
if PY3:
    basestring = str

from .log import LoggerFactory

logger = LoggerFactory.create_logger('daemon')


# Make a directory with recursive creation if need
# Can send IOError if a file already exists with the name
def make_dir(path):
    # Already exists as a directory
    # NOTICE: will always ends as the root dir will exists
    # note: '' if for windows, as I am not sure about the root directory
    if os.path.isdir(path) or path == '':
        return
    # Do not exists, but the path is already there by a file!
    if os.path.exists(path):
        raise IOError('Asking to create the directory %s but a file already exits there' % path)
    # assert that the parent directory does exists
    parent, dir_name = os.path.split(path)
    if dir_name == '':  # was ending by a /, skip this loop, dir_name is useless
        return make_dir(parent)
    # assert that the parent does exists
    make_dir(parent)
    # so now we can create the son
    os.mkdir(path)


def exec_command(cmd):
    import subprocess
    # If we have a list, we should not call with a shell
    shell = False if isinstance(cmd, list) else True
    close_fds = True
    # windows do not manage close fds
    if os.name == 'nt':
        close_fds = False
    # There is no setsid function on windows
    preexec_fn = getattr(os, 'setsid', None)
    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=close_fds, preexec_fn=preexec_fn, shell=shell)
    stdout, stderr = p.communicate()
    stdout = bytes_to_unicode(stdout)
    stderr = bytes_to_unicode(stderr)
    return p.returncode, stdout, stderr


def my_sort(lst, cmp_f):
    if not PY3:
        lst = sorted(lst, cmp=cmp_f)
    else:
        from functools import cmp_to_key<commit_msg>Enh: Do not close fds on subprocess calls, because it's too cpu consuming.<commit_after>import sys
import os
import shutil
import glob
import socket
import hashlib
import time
import uuid as libuuid

PY3 = sys.version_info >= (3,)
if PY3:
    basestring = str

from .log import LoggerFactory

logger = LoggerFactory.create_logger('daemon')


# Make a directory with recursive creation if need
# Can send IOError if a file already exists with the name
def make_dir(path):
    # Already exists as a directory
    # NOTICE: will always ends as the root dir will exists
    # note: '' if for windows, as I am not sure about the root directory
    if os.path.isdir(path) or path == '':
        return
    # Do not exists, but the path is already there by a file!
    if os.path.exists(path):
        raise IOError('Asking to create the directory %s but a file already exits there' % path)
    # assert that the parent directory does exists
    parent, dir_name = os.path.split(path)
    if dir_name == '':  # was ending by a /, skip this loop, dir_name is useless
        return make_dir(parent)
    # assert that the parent does exists
    make_dir(parent)
    # so now we can create the son
    os.mkdir(path)


def exec_command(cmd):
    import subprocess
    # If we have a list, we should not call with a shell
    shell = False if isinstance(cmd, list) else True
    # close_fds: I used to set True on unix (False on Windows because it do not manage it)
    # but it's just tooooooo cpu consuming, calling 65K time close()
    # so... nop
    close_fds = False
    # There is no setsid function on windows
    preexec_fn = getattr(os, 'setsid', None)
    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=close_fds, preexec_fn=preexec_fn, shell=shell)
    stdout, stderr = p.communicate()
    stdout = bytes_to_unicode(stdout)
    stderr = bytes_to_unicode(stderr)
    return p.returncode, stdout, stderr


def my_sort(lst, cmp_f):
    if not PY3:
        lst = sorted(lst, cmp=cmp_f)
    else:
        from functools import cmp_to_key
{'fraction_non_alphanumeric': 0.07593984962406015, 'fraction_numerical': 0.002506265664160401, 'mean_word_length': 3.2367303609341826, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3985188', 'n_tokens_mistral': 1278, 'n_tokens_neox': 1239, 'n_words': 491}