<commit_before>#!/usr/bin/env python
# Copyright (c) 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.


"""Run a command and report its results in machine-readable format."""


import collections
import optparse
import os
import pickle
import pprint
import socket
import subprocess
import sys

buildbot_path = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                             os.pardir))
sys.path.append(os.path.join(buildbot_path))

from site_config import slave_hosts_cfg


class CommandResults(collections.namedtuple('CommandResults',
                                            'stdout, stderr, returncode')):

  # We print this string before and after the important output from the command.
  # This makes it easy to ignore output from SSH, shells, etc.
  BOOKEND_STR = '@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@'

  def encode(self):
    """Convert the results into a machine-readable string.

    Returns:
        A hex-encoded string, bookended by BOOKEND_STR for easy parsing.
    """
    return (CommandResults.BOOKEND_STR +
            pickle.dumps(self.__dict__).encode('hex') +
            CommandResults.BOOKEND_STR)

  @staticmethod
  def decode(results_str):
    """Convert a machine-readable string into a CommandResults instance.

    Args:
        results_str: string; output from "run" or one of its siblings.
    Returns:
        A dictionary of results.
    """
    return CommandResults(**pickle.loads(
        results_str.split(CommandResults.BOOKEND_STR)[1].decode('hex')))

  @staticmethod
  def make(stdout='', stderr='', returncode=1):
    """Create CommandResults for a command.

    Args:
        stdout: string; stdout from a command.
        stderr: string; stderr from a command.
        returncode: string; return code of a command.
    """
    return CommandResults(stdout=stdout,
                          stderr=stderr,
                          returncode=returncode)

  @property
  def __dict__(self):
    """Return a dictionary representation of this CommandResults instance.

    Since collections.NamedTuple.__dict__ returns an OrderedDict, we have to
    create this wrapper to get a normal dict.
    """
    return dict(super(CommandResults, self).__dict__)

  def print_results(self, pretty=False):
    """Print the results of a command.

    Args:
        pretty: bool; whether or not to print in human-readable format.
    """
    if pretty:
      print pprint.pformat(self.__dict__)
    else:
      print repr(self.encode())


class ResolvableCommandElement(object):
  """Base class for elements of commands which have different string values
  depending on the properties of the host."""

  def resolve(self, slave_host_name):
    """Resolve this ResolvableCommandElement as appropriate.

    Args:
        slave_host_name: string; name of the slave host.
    Returns:
        string whose value depends on the given slave_host_name in some way.
    """
    raise NotImplementedError


class BuildbotPath(ResolvableCommandElement):
  """Path to the buildbot scripts checkout on a slave host machine."""

  def resolve(self, slave_host_name):
    """Return the resolved path to the buildbot checkout on a slave_host.

    Args:
        slave_host_name: string; name of the slave host.
    Returns:
        string; the path to the buildbot checkout.
    """
    host_data = slave_hosts_cfg.get_slave_host_config(slave_host_name)
    return host_data.path_module.join(*host_data.path_to_buildbot)


class ResolvablePath(ResolvableCommandElement):
  """Represents a path."""

  def __init__(self, *path_elems):
    """Instantiate this ResolvablePath.

    Args:
        path_elems: strings or ResolvableCommandElements which will be joined to
            form a path.
    """
    super(ResolvablePath, self).__init__()
    self._path_elems = list(*path_elems)

  def resolve(self, slave_host_name):
    """Resolve this ResolvablePath as appropriate.

    Args:
        slave_host_name: string; name of the slave host.
    Returns:
        string whose value depends on the given slave_host_name in some way.
    """
    host_data = slave_hosts_cfg.get_slave_host_config(slave_host_name)
    fixed_path_elems = _fixup_cmd(self._path_elems, slave_host_name)
    return host_data.path_module.join(*fixed_path_elems)

  @staticmethod
  def buildbot_path(*path_elems):
    """Convenience method; returns a path relative to the buildbot checkout."""
    return ResolvablePath([BuildbotPath()] + list(path_elems))


def _fixup_cmd(cmd, slave_host_name):
  """Resolve the command into a list of strings.

  Args:
      cmd: list containing strings or ResolvableCommandElements.
      slave_host_name: string; the name of the relevant slave host machine.
  """
  new_cmd = []
  for elem in cmd:
    if isinstance(elem, ResolvableCommandElement):
      resolved_elem = elem.resolve(slave_host_name)
      new_cmd.append(resolved_elem)
    else:
      new_cmd.append(elem)
  return new_cmd


def _launch_cmd(cmd):
  """Launch the given command. Non-blocking.

  Args:
      cmd: list of strings; command to run.
  Returns:
      subprocess.Popen instance.
  """
  return subprocess.Popen(cmd, shell=False, stderr=subprocess.PIPE,
                          stdout=subprocess.PIPE)


def _get_result(popen):
  """Get the results from a running process. Blocks until the process completes.

  Args:
      popen: subprocess.Popen instance.
  Returns:
      A dictionary with stdout, stderr, and returncode as keys.
  """
  stdout, stderr = popen.communicate()
  return CommandResults.make(stdout=stdout,
                             stderr=stderr,
                             returncode=popen.returncode)


def run(cmd):
  """Run the command, block until it completes, and return a results dictionary.

  Args:
      cmd: string or list of strings; the command to run.
  Returns:
      A dictionary with stdout, stderr, and returncode as keys.
  """<commit_msg>A little more print debugging<commit_after>#!/usr/bin/env python
# Copyright (c) 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.


"""Run a command and report its results in machine-readable format."""


import collections
import optparse
import os
import pickle
import pprint
import socket
import subprocess
import sys

buildbot_path = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                             os.pardir))
sys.path.append(os.path.join(buildbot_path))

from site_config import slave_hosts_cfg


class CommandResults(collections.namedtuple('CommandResults',
                                            'stdout, stderr, returncode')):

  # We print this string before and after the important output from the command.
  # This makes it easy to ignore output from SSH, shells, etc.
  BOOKEND_STR = '@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@'

  def encode(self):
    """Convert the results into a machine-readable string.

    Returns:
        A hex-encoded string, bookended by BOOKEND_STR for easy parsing.
    """
    return (CommandResults.BOOKEND_STR +
            pickle.dumps(self.__dict__).encode('hex') +
            CommandResults.BOOKEND_STR)

  @staticmethod
  def decode(results_str):
    """Convert a machine-readable string into a CommandResults instance.

    Args:
        results_str: string; output from "run" or one of its siblings.
    Returns:
        A dictionary of results.
    """
    return CommandResults(**pickle.loads(
        results_str.split(CommandResults.BOOKEND_STR)[1].decode('hex')))

  @staticmethod
  def make(stdout='', stderr='', returncode=1):
    """Create CommandResults for a command.

    Args:
        stdout: string; stdout from a command.
        stderr: string; stderr from a command.
        returncode: string; return code of a command.
    """
    return CommandResults(stdout=stdout,
                          stderr=stderr,
                          returncode=returncode)

  @property
  def __dict__(self):
    """Return a dictionary representation of this CommandResults instance.

    Since collections.NamedTuple.__dict__ returns an OrderedDict, we have to
    create this wrapper to get a normal dict.
    """
    return dict(super(CommandResults, self).__dict__)

  def print_results(self, pretty=False):
    """Print the results of a command.

    Args:
        pretty: bool; whether or not to print in human-readable format.
    """
    if pretty:
      print pprint.pformat(self.__dict__)
    else:
      print repr(self.encode())


class ResolvableCommandElement(object):
  """Base class for elements of commands which have different string values
  depending on the properties of the host."""

  def resolve(self, slave_host_name):
    """Resolve this ResolvableCommandElement as appropriate.

    Args:
        slave_host_name: string; name of the slave host.
    Returns:
        string whose value depends on the given slave_host_name in some way.
    """
    raise NotImplementedError


class BuildbotPath(ResolvableCommandElement):
  """Path to the buildbot scripts checkout on a slave host machine."""

  def resolve(self, slave_host_name):
    """Return the resolved path to the buildbot checkout on a slave_host.

    Args:
        slave_host_name: string; name of the slave host.
    Returns:
        string; the path to the buildbot checkout.
    """
    host_data = slave_hosts_cfg.get_slave_host_config(slave_host_name)
    return host_data.path_module.join(*host_data.path_to_buildbot)


class ResolvablePath(ResolvableCommandElement):
  """Represents a path."""

  def __init__(self, *path_elems):
    """Instantiate this ResolvablePath.

    Args:
        path_elems: strings or ResolvableCommandElements which will be joined to
            form a path.
    """
    super(ResolvablePath, self).__init__()
    self._path_elems = list(*path_elems)

  def resolve(self, slave_host_name):
    """Resolve this ResolvablePath as appropriate.

    Args:
        slave_host_name: string; name of the slave host.
    Returns:
        string whose value depends on the given slave_host_name in some way.
    """
    host_data = slave_hosts_cfg.get_slave_host_config(slave_host_name)
    fixed_path_elems = _fixup_cmd(self._path_elems, slave_host_name)
    return host_data.path_module.join(*fixed_path_elems)

  @staticmethod
  def buildbot_path(*path_elems):
    """Convenience method; returns a path relative to the buildbot checkout."""
    return ResolvablePath([BuildbotPath()] + list(path_elems))


def _fixup_cmd(cmd, slave_host_name):
  """Resolve the command into a list of strings.

  Args:
      cmd: list containing strings or ResolvableCommandElements.
      slave_host_name: string; the name of the relevant slave host machine.
  """
  new_cmd = []
  for elem in cmd:
    if isinstance(elem, ResolvableCommandElement):
      resolved_elem = elem.resolve(slave_host_name)
      new_cmd.append(resolved_elem)
    else:
      new_cmd.append(elem)
  return new_cmd


def _launch_cmd(cmd):
  """Launch the given command. Non-blocking.

  Args:
      cmd: list of strings; command to run.
  Returns:
      subprocess.Popen instance.
  """
  print ' '.join(cmd)
  return subprocess.Popen(cmd, shell=False, stderr=subprocess.PIPE,
                          stdout=subprocess.PIPE)


def _get_result(popen):
  """Get the results from a running process. Blocks until the process completes.

  Args:
      popen: subprocess.Popen instance.
  Returns:
      A dictionary with stdout, stderr, and returncode as keys.
  """
  stdout, stderr = popen.communicate()
  return CommandResults.make(stdout=stdout,
                             stderr=stderr,
                             returncode=popen.returncode)


def run(cmd):
  """Run the command, block until it completes, and return a results dictionary.

  Args:
      cmd: string or list of strings; the command to run.
  Returns:
      A dictionary with stdout, stderr, and returncode as keys.
  """
{'fraction_non_alphanumeric': 0.08802583636965883, 'fraction_numerical': 0.0009937065253395165, 'mean_word_length': 3.075936550793115, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4357856', 'n_tokens_mistral': 3519, 'n_tokens_neox': 3395, 'n_words': 1106}