<commit_before>## Copyright (c) 2012 Aldebaran Robotics. All rights reserved.
## Use of this source code is governed by a BSD-style license that can be
## found in the COPYING file.

""" This module contains few functions around subprocess

"""

import os
import sys
import contextlib
import logging
import subprocess
import threading
import Queue

import qibuild

LOGGER = logging.getLogger(__name__)


# Quick hack: in order to be able to configure how
# qibuild.command works, we have to use this
# global variable
CONFIG = dict()

class ProcessThread(threading.Thread):
    """ A simple way to run commands.

    The thread will terminate when the command terminates

    The full log is available in self.out, and the subprocess.Popen
    object in self.process

    """
    def __init__(self, cmd, name=None, cwd=None, env=None):
        if name is None:
            thread_name = "ProcessThread"
        else:
            thread_name = "ProcessThread<%s>" % name
        threading.Thread.__init__(self, name=thread_name)
        self.cmd = cmd
        self.cwd = cwd
        self.env = env
        self.out = ""
        self.process = None
        self.exception = ""

    def run(self):
        try:
            self.process = subprocess.Popen(self.cmd,
                stdout=subprocess.PIPE,
                stderr=subprocess.STDOUT,
                cwd=self.cwd,
                env=self.env)
        except Exception, e:
            self.exception = e
            return

        while self.process.poll() is None:
            self.out += self.process.stdout.readline()


class CommandFailedException(Exception):
    """Custom exception """
    def __init__(self, cmd, returncode, cwd=None):
        self.cmd = cmd
        if cwd is None:
            self.cwd = os.getcwd()
        else:
            self.cwd = cwd
        self.returncode = returncode

    def __str__(self):
        mess  = """The following command failed
{cmd}
Return code is {returncode}
Working dir was {cwd}
"""
        return mess.format(cmd=self.cmd, returncode=self.returncode, cwd=self.cwd)


class ProcessCrashedError(Exception):
    """An other custom exception, used by call_background """
    def __init__(self, cmd):
        self.cmd = cmd

    def __str__(self):
        mess = "%s crashed!\n" % os.path.basename(self.cmd[0])
        mess += "Full command: %s" % self.cmd
        return mess

class NotInPath(Exception):
    """Custom exception """
    def __init__(self, executable, env=None):
        self.executable = executable
        self.env = env

    def __str__(self):
        if self.env:
            path_env = self.env.get("PATH")
        else:
            path_env = os.environ["PATH"]
        mess  = "Could not find executable: %s\n" % self.executable
        mess += "Looked in:\n"
        mess += "\n".join(path_env.split(os.pathsep))
        return mess


def find_program(executable, env=None):
    """Get the full path of an executable by
    looking at PATH environment variable
    (and PATHEXT on windows)

    :return: None if program was not found,
      the full path to executable otherwize
    """
    full_path = None
    if env:
        env_path = env.get("PATH", "")
    else:
        env_path = os.environ["PATH"]
    for path in env_path.split(os.pathsep):
        path = qibuild.sh.to_native_path(path)
        full_path = os.path.join(path, executable)
        if os.access(full_path, os.X_OK) and os.path.isfile(full_path):
            return full_path
        pathext = os.environ.get("PATHEXT")
        if pathext:
            for ext in pathext.split(";"):
                with_ext = full_path + ext
                if os.access(with_ext, os.X_OK):
                    return qibuild.sh.to_native_path(with_ext)
    return None


def check_is_in_path(executable, build_env=None):
    """Check that the given executable is to be found in %PATH%"""
    if find_program(executable, env=build_env) is None:
        raise NotInPath(executable, env=build_env)


def call(cmd, cwd=None, env=None, ignore_ret_code=False):
    """ Execute a command line.

    If ignore_ret_code is False:
        raise CommandFailedException if returncode is None.
    Else:
        simply returns the returncode of the process

    Note: first arg of the cmd is assumed to be something
    inside %PATH%. (or in env[PATH] if env is not None)

    Note: the shell= argument of the subprocess.Popen
    call will always be False.

    can raise:
      * CommandFailedException if ignore_ret_code is False
        and returncode is non zero
      * NotInPath  if first arg of cmd is not in %PATH%
      * And a normal exception if cwd is given and is not
        an existing directory.

    If sys.stdout or sys.stderr are not a tty, only write
    the last 300 lines of the process to sys.stdout if the
    returncode is not zero, else write everything.

    Note: this trick with sys.stderr, sys.stdout and subprocess
    does not work on windows with python < 2.7, so it is simply
    disabled, and you have a normal behavior instead.

    """
    exe_full_path = find_program(cmd[0], env=env)
    if not exe_full_path:
        raise NotInPath(cmd[0], env=env)
    cmd[0] = exe_full_path

    if cwd:
        if not os.path.exists(cwd):
            # We know we are likely to have a problem on windows here,
            # so always raise.
            raise Exception("Trying to to run %s in non-existing %s" %
                (" ".join(cmd), cwd))

    ring_buffer = RingBuffer(300)

    returncode = 0
    quiet_command = CONFIG.get("quiet", False)
    # This code won't work on windows with python < 2.7,
    # so quiet will be ignored
    if sys.platform.startswith("win") and sys.version_info < (2, 7):
        quiet_command = False
    if not quiet_command:
        returncode = subprocess.call(cmd, env=env, cwd=cwd)
    else:
        cmdline = CommandLine(cmd, cwd=cwd, env=env)
        for(out, err) in cmdline.execute():
            if out is not None:
                ring_buffer.append(out)
            if err is not None:
                ring_buffer.append(err)
        returncode = cmdline.returncode

    if ignore_ret_code:
        return returncode

    if returncode != 0:
        if quiet_command:
            lines = ring_buffer.get()
            for line in lines:
                sys.stdout.write(line)
                sys.stdout.flush()<commit_msg>qibuild.command: re-add call to LOGGER.debug<commit_after>## Copyright (c) 2012 Aldebaran Robotics. All rights reserved.
## Use of this source code is governed by a BSD-style license that can be
## found in the COPYING file.

""" This module contains few functions around subprocess

"""

import os
import sys
import contextlib
import logging
import subprocess
import threading
import Queue

import qibuild

LOGGER = logging.getLogger(__name__)


# Quick hack: in order to be able to configure how
# qibuild.command works, we have to use this
# global variable
CONFIG = dict()

class ProcessThread(threading.Thread):
    """ A simple way to run commands.

    The thread will terminate when the command terminates

    The full log is available in self.out, and the subprocess.Popen
    object in self.process

    """
    def __init__(self, cmd, name=None, cwd=None, env=None):
        if name is None:
            thread_name = "ProcessThread"
        else:
            thread_name = "ProcessThread<%s>" % name
        threading.Thread.__init__(self, name=thread_name)
        self.cmd = cmd
        self.cwd = cwd
        self.env = env
        self.out = ""
        self.process = None
        self.exception = ""

    def run(self):
        try:
            self.process = subprocess.Popen(self.cmd,
                stdout=subprocess.PIPE,
                stderr=subprocess.STDOUT,
                cwd=self.cwd,
                env=self.env)
        except Exception, e:
            self.exception = e
            return

        while self.process.poll() is None:
            self.out += self.process.stdout.readline()


class CommandFailedException(Exception):
    """Custom exception """
    def __init__(self, cmd, returncode, cwd=None):
        self.cmd = cmd
        if cwd is None:
            self.cwd = os.getcwd()
        else:
            self.cwd = cwd
        self.returncode = returncode

    def __str__(self):
        mess  = """The following command failed
{cmd}
Return code is {returncode}
Working dir was {cwd}
"""
        return mess.format(cmd=self.cmd, returncode=self.returncode, cwd=self.cwd)


class ProcessCrashedError(Exception):
    """An other custom exception, used by call_background """
    def __init__(self, cmd):
        self.cmd = cmd

    def __str__(self):
        mess = "%s crashed!\n" % os.path.basename(self.cmd[0])
        mess += "Full command: %s" % self.cmd
        return mess

class NotInPath(Exception):
    """Custom exception """
    def __init__(self, executable, env=None):
        self.executable = executable
        self.env = env

    def __str__(self):
        if self.env:
            path_env = self.env.get("PATH")
        else:
            path_env = os.environ["PATH"]
        mess  = "Could not find executable: %s\n" % self.executable
        mess += "Looked in:\n"
        mess += "\n".join(path_env.split(os.pathsep))
        return mess


def find_program(executable, env=None):
    """Get the full path of an executable by
    looking at PATH environment variable
    (and PATHEXT on windows)

    :return: None if program was not found,
      the full path to executable otherwize
    """
    full_path = None
    if env:
        env_path = env.get("PATH", "")
    else:
        env_path = os.environ["PATH"]
    for path in env_path.split(os.pathsep):
        path = qibuild.sh.to_native_path(path)
        full_path = os.path.join(path, executable)
        if os.access(full_path, os.X_OK) and os.path.isfile(full_path):
            return full_path
        pathext = os.environ.get("PATHEXT")
        if pathext:
            for ext in pathext.split(";"):
                with_ext = full_path + ext
                if os.access(with_ext, os.X_OK):
                    return qibuild.sh.to_native_path(with_ext)
    return None


def check_is_in_path(executable, build_env=None):
    """Check that the given executable is to be found in %PATH%"""
    if find_program(executable, env=build_env) is None:
        raise NotInPath(executable, env=build_env)


def call(cmd, cwd=None, env=None, ignore_ret_code=False):
    """ Execute a command line.

    If ignore_ret_code is False:
        raise CommandFailedException if returncode is None.
    Else:
        simply returns the returncode of the process

    Note: first arg of the cmd is assumed to be something
    inside %PATH%. (or in env[PATH] if env is not None)

    Note: the shell= argument of the subprocess.Popen
    call will always be False.

    can raise:
      * CommandFailedException if ignore_ret_code is False
        and returncode is non zero
      * NotInPath  if first arg of cmd is not in %PATH%
      * And a normal exception if cwd is given and is not
        an existing directory.

    If sys.stdout or sys.stderr are not a tty, only write
    the last 300 lines of the process to sys.stdout if the
    returncode is not zero, else write everything.

    Note: this trick with sys.stderr, sys.stdout and subprocess
    does not work on windows with python < 2.7, so it is simply
    disabled, and you have a normal behavior instead.

    """
    exe_full_path = find_program(cmd[0], env=env)
    if not exe_full_path:
        raise NotInPath(cmd[0], env=env)
    cmd[0] = exe_full_path

    if cwd:
        if not os.path.exists(cwd):
            # We know we are likely to have a problem on windows here,
            # so always raise.
            raise Exception("Trying to to run %s in non-existing %s" %
                (" ".join(cmd), cwd))

    LOGGER.debug("Calling %s", ", ".join(cmd))
    ring_buffer = RingBuffer(300)

    returncode = 0
    quiet_command = CONFIG.get("quiet", False)
    # This code won't work on windows with python < 2.7,
    # so quiet will be ignored
    if sys.platform.startswith("win") and sys.version_info < (2, 7):
        quiet_command = False
    if not quiet_command:
        returncode = subprocess.call(cmd, env=env, cwd=cwd)
    else:
        cmdline = CommandLine(cmd, cwd=cwd, env=env)
        for(out, err) in cmdline.execute():
            if out is not None:
                ring_buffer.append(out)
            if err is not None:
                ring_buffer.append(err)
        returncode = cmdline.returncode

    if ignore_ret_code:
        return returncode

    if returncode != 0:
        if quiet_command:
            lines = ring_buffer.get()
            for line in lines:
                sys.stdout.write(line)
                sys.stdout.flush()
{'fraction_non_alphanumeric': 0.09168094718803552, 'fraction_numerical': 0.0034273251285246924, 'mean_word_length': 2.2836317135549873, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2068343', 'n_tokens_mistral': 3905, 'n_tokens_neox': 3770, 'n_words': 1254}