<commit_before>#!/bin/bash

# Copyright (c) 2009 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# Script to run client or server tests on a live remote image.

# Load common constants.  This should be the first executable line.
# The path to common.sh should be relative to your script's location.

. "$(dirname $0)/common.sh"
. "$(dirname $0)/autotest_lib.sh"
. "$(dirname $0)/remote_access.sh"

DEFAULT_OUTPUT_FILE=test-output-$(date '+%Y%m%d.%H%M%S')

DEFINE_string build_desc "" "Build description used in database"
DEFINE_string chroot "${DEFAULT_CHROOT_DIR}" "alternate chroot location" c
DEFINE_boolean cleanup ${FLAGS_FALSE} "Clean up temp directory"
DEFINE_integer iterations 1 "Iterations to run every top level test" i
DEFINE_string machine_desc "" "Machine description used in database"
DEFINE_string output_file "${DEFAULT_OUTPUT_FILE}" "Test run output" o
DEFINE_string prepackaged_autotest "" "Use this prepackaged autotest dir"
DEFINE_string results_dir_root "" "alternate root results directory"
DEFINE_boolean update_db ${FLAGS_FALSE} "Put results in autotest database" u
DEFINE_boolean verbose ${FLAGS_FALSE} "Show verbose autoserv output" v

function cleanup() {
  if [[ $FLAGS_cleanup -eq ${FLAGS_TRUE} ]]; then
    rm -rf "${TMP}"
  else
    echo "Left temporary files at ${TMP}"
  fi
  cleanup_remote_access
}

# Returns an error if the test_result_file has text which indicates
# the test was not run successfully.
# Arguments:
#   $1 - file name of autotest status for to check for success
# Returns:
#   None
function is_successful_test() {
  local file="$1"
  # To be successful, must not have FAIL or BAD in the file.
  if egrep -q "(BAD|FAIL)" "${file}"; then
    return 1
  fi
  # To be successful, must have GOOD in the file.
  if ! grep -q GOOD "${file}"; then
    return 1
  fi
  return 0
}

# Removes single quotes around parameter
# Arguments:
#   $1 - string which optionally has surrounding quotes
# Returns:
#   None, but prints the string without quotes.
function remove_quotes() {
  echo "$1" | sed -e "s/^'//; s/'$//"
}

# Adds attributes to all tests run
# Arguments:
#   $1 - results directory
#   $2 - attribute name (key)
#   $3 - attribute value (value)
function add_test_attribute() {
  local results_dir="$1"
  local attribute_name="$2"
  local attribute_value="$3"
  if [[ -z "$attribute_value" ]]; then
    return;
  fi

  for status_file in $(echo "${results_dir}"/*/status); do
    local keyval_file=$(dirname $status_file)/keyval
    echo "Updating ${keyval_file}"
    echo "${attribute_name}=${attribute_value}" >> "${keyval_file}"
  done
}


# Ask the target what board it is
function learn_board() {
  if [[ -n "${FLAGS_board}" ]]; then
    return
  fi
  remote_sh grep CHROMEOS_RELEASE_BOARD /etc/lsb-release
  FLAGS_board=$(echo "${REMOTE_OUT}" | cut -d= -f2)
  if [[ -z "${FLAGS_board}" ]]; then
    check_board
  fi
  echo "Target reports board is ${FLAGS_board}"
}


function main() {
  assert_outside_chroot

  cd $(dirname "$0")

  FLAGS "$@" || exit 1

  if [[ -z "${FLAGS_ARGV}" ]]; then
    echo "Please specify tests to run, like:"
    echo "  $0 --remote=MyMachine SystemBootPerf"
    exit 1
  fi

  set -e

  # Set global TMP for remote_access.sh's sake
  TMP=$(mktemp -d /tmp/run_remote_tests.XXXX)

  rm -f "${FLAGS_output_file}"

  trap cleanup EXIT

  remote_access_init

  local autotest_dir=""
  if [[ -z "${FLAGS_prepackaged_autotest}" ]]; then
    learn_board
    # Always copy into installed autotest directory.  This way if a user
    # is just modifying scripts, they take effect without having to wait
    # for the laborious build_autotest.sh command.
    local original="${GCLIENT_ROOT}/src/third_party/autotest/files"
    autotest_dir="${FLAGS_chroot}/build/${FLAGS_board}/usr/local/autotest"
    update_chroot_autotest "${original}" "${autotest_dir}"
  else
    autotest_dir="${FLAGS_prepackaged_autotest}"
  fi

  local parse_cmd="${autotest_dir}/tko/parse.py"

  if [[ ${FLAGS_update_db} -eq ${FLAGS_TRUE} && ! -x "${parse_cmd}" ]]; then
    echo "Cannot find parser ${parse_cmd}"
    exit 1
  fi

  local autoserv="${autotest_dir}/server/autoserv"

  local control_files_to_run=""

  # Now search for tests which unambiguously include the given identifier
  local search_path=$(echo ${autotest_dir}/{client,server}/{tests,site_tests})
  for test_request in $FLAGS_ARGV; do
    test_request=$(remove_quotes "${test_request}")
    ! finds=$(find ${search_path} -type f -name control\* | \
      egrep "${test_request}")
    if [[ -z "${finds}" ]]; then
      echo "Can not find match for ${test_request}"
      exit 1<commit_msg>Add restrictions to files considered control files<commit_after>#!/bin/bash

# Copyright (c) 2009 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# Script to run client or server tests on a live remote image.

# Load common constants.  This should be the first executable line.
# The path to common.sh should be relative to your script's location.

. "$(dirname $0)/common.sh"
. "$(dirname $0)/autotest_lib.sh"
. "$(dirname $0)/remote_access.sh"

DEFAULT_OUTPUT_FILE=test-output-$(date '+%Y%m%d.%H%M%S')

DEFINE_string build_desc "" "Build description used in database"
DEFINE_string chroot "${DEFAULT_CHROOT_DIR}" "alternate chroot location" c
DEFINE_boolean cleanup ${FLAGS_FALSE} "Clean up temp directory"
DEFINE_integer iterations 1 "Iterations to run every top level test" i
DEFINE_string machine_desc "" "Machine description used in database"
DEFINE_string output_file "${DEFAULT_OUTPUT_FILE}" "Test run output" o
DEFINE_string prepackaged_autotest "" "Use this prepackaged autotest dir"
DEFINE_string results_dir_root "" "alternate root results directory"
DEFINE_boolean update_db ${FLAGS_FALSE} "Put results in autotest database" u
DEFINE_boolean verbose ${FLAGS_FALSE} "Show verbose autoserv output" v

function cleanup() {
  if [[ $FLAGS_cleanup -eq ${FLAGS_TRUE} ]]; then
    rm -rf "${TMP}"
  else
    echo "Left temporary files at ${TMP}"
  fi
  cleanup_remote_access
}

# Returns an error if the test_result_file has text which indicates
# the test was not run successfully.
# Arguments:
#   $1 - file name of autotest status for to check for success
# Returns:
#   None
function is_successful_test() {
  local file="$1"
  # To be successful, must not have FAIL or BAD in the file.
  if egrep -q "(BAD|FAIL)" "${file}"; then
    return 1
  fi
  # To be successful, must have GOOD in the file.
  if ! grep -q GOOD "${file}"; then
    return 1
  fi
  return 0
}

# Removes single quotes around parameter
# Arguments:
#   $1 - string which optionally has surrounding quotes
# Returns:
#   None, but prints the string without quotes.
function remove_quotes() {
  echo "$1" | sed -e "s/^'//; s/'$//"
}

# Adds attributes to all tests run
# Arguments:
#   $1 - results directory
#   $2 - attribute name (key)
#   $3 - attribute value (value)
function add_test_attribute() {
  local results_dir="$1"
  local attribute_name="$2"
  local attribute_value="$3"
  if [[ -z "$attribute_value" ]]; then
    return;
  fi

  for status_file in $(echo "${results_dir}"/*/status); do
    local keyval_file=$(dirname $status_file)/keyval
    echo "Updating ${keyval_file}"
    echo "${attribute_name}=${attribute_value}" >> "${keyval_file}"
  done
}


# Ask the target what board it is
function learn_board() {
  if [[ -n "${FLAGS_board}" ]]; then
    return
  fi
  remote_sh grep CHROMEOS_RELEASE_BOARD /etc/lsb-release
  FLAGS_board=$(echo "${REMOTE_OUT}" | cut -d= -f2)
  if [[ -z "${FLAGS_board}" ]]; then
    check_board
  fi
  echo "Target reports board is ${FLAGS_board}"
}


function main() {
  assert_outside_chroot

  cd $(dirname "$0")

  FLAGS "$@" || exit 1

  if [[ -z "${FLAGS_ARGV}" ]]; then
    echo "Please specify tests to run, like:"
    echo "  $0 --remote=MyMachine SystemBootPerf"
    exit 1
  fi

  set -e

  # Set global TMP for remote_access.sh's sake
  TMP=$(mktemp -d /tmp/run_remote_tests.XXXX)

  rm -f "${FLAGS_output_file}"

  trap cleanup EXIT

  remote_access_init

  local autotest_dir=""
  if [[ -z "${FLAGS_prepackaged_autotest}" ]]; then
    learn_board
    # Always copy into installed autotest directory.  This way if a user
    # is just modifying scripts, they take effect without having to wait
    # for the laborious build_autotest.sh command.
    local original="${GCLIENT_ROOT}/src/third_party/autotest/files"
    autotest_dir="${FLAGS_chroot}/build/${FLAGS_board}/usr/local/autotest"
    update_chroot_autotest "${original}" "${autotest_dir}"
  else
    autotest_dir="${FLAGS_prepackaged_autotest}"
  fi

  local parse_cmd="${autotest_dir}/tko/parse.py"

  if [[ ${FLAGS_update_db} -eq ${FLAGS_TRUE} && ! -x "${parse_cmd}" ]]; then
    echo "Cannot find parser ${parse_cmd}"
    exit 1
  fi

  local autoserv="${autotest_dir}/server/autoserv"

  local control_files_to_run=""

  # Now search for tests which unambiguously include the given identifier
  local search_path=$(echo ${autotest_dir}/{client,server}/{tests,site_tests})
  for test_request in $FLAGS_ARGV; do
    test_request=$(remove_quotes "${test_request}")
    ! finds=$(find ${search_path} -maxdepth 2 -type f -name control\* | \
      egrep "${test_request}")
    if [[ -z "${finds}" ]]; then
      echo "Can not find match for ${test_request}"
      exit 1
{'fraction_non_alphanumeric': 0.12032057365812507, 'fraction_numerical': 0.0060107560898449855, 'mean_word_length': 4.0021097046413505, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2598906', 'n_tokens_mistral': 3138, 'n_tokens_neox': 3008, 'n_words': 1017}