<commit_before>#!/bin/bash
#
# Runs all steps in vm_setup_slave.sh, executes record_wpr and copies the
# created archives to Google Storage.
#
# The script should be run from the skia-telemetry-slave GCE instance's
# /home/default/skia-repo/buildbot/compute_engine_scripts/telemetry/telemetry_slave_scripts
# directory.
#
# Copyright 2013 Google Inc. All Rights Reserved.
# Author: <EMAIL> (<NAME>)


if [ $# -ne 3 ]; then
  echo
  echo "Usage: `basename $0` 1 All a1234b-c5678d"
  echo
  echo "The first argument is the slave_num of this telemetry slave."
  echo "The second argument is the type of pagesets to create from the 1M list"\
       "Eg: All, Filtered, 100k, 10k, Deeplinks."
  echo "The third argument is the name of the directory where the chromium" \
       "build which will be used for this run is stored."
  echo
  exit 1
fi

SLAVE_NUM=$1
PAGESETS_TYPE=$2
CHROMIUM_BUILD_DIR=$3

source ../vm_config.sh
source vm_utils.sh

create_worker_file $RECORD_WPR_ACTIVITY

source vm_setup_slave.sh

# Create the webpages_archive directory.
mkdir -p /home/default/storage/webpages_archive/$PAGESETS_TYPE/
rm -rf /home/default/storage/webpages_archive/$PAGESETS_TYPE/*

for page_set in /home/default/storage/page_sets/$PAGESETS_TYPE/*; do
  if [[ -f $page_set ]]; then
    echo "========== Processing $page_set =========="
    pageset_basename=`basename $page_set`
    if [ "$PAGESETS_TYPE" == "Filtered" ]; then
      # Since the archive already exists in 'All' do not run record_wpr.
      pageset_filename="${pageset_basename%.*}"
      cp  /home/default/storage/webpages_archive/All/${pageset_filename}* /home/default/storage/webpages_archive/$PAGESETS_TYPE/
      echo "========== $page_set copied over from All =========="
    else
      check_and_run_xvfb
      # TODO(rmistry): Figure out why using the specific chromium build does not
      # work.
      # DISPLAY=:0 timeout 600 tools/perf/record_wpr --extra-browser-args=--disable-setuid-sandbox --browser-executable=/home/default/storage/chromium-builds/${CHROMIUM_BUILD_DIR}/chrome --browser=exact $page_set
      DISPLAY=:0 timeout 600 tools/perf/record_wpr --extra-browser-args=--disable-setuid-sandbox --browser-executable=/usr/bin/chromium --browser=exact $page_set
      if [ $? -eq 124 ]; then
        echo "========== $page_set timed out! =========="<commit_msg>Chromium builds are working on the new telemetry ubuntu images, use them for capturing archives.<commit_after>#!/bin/bash
#
# Runs all steps in vm_setup_slave.sh, executes record_wpr and copies the
# created archives to Google Storage.
#
# The script should be run from the skia-telemetry-slave GCE instance's
# /home/default/skia-repo/buildbot/compute_engine_scripts/telemetry/telemetry_slave_scripts
# directory.
#
# Copyright 2013 Google Inc. All Rights Reserved.
# Author: <EMAIL> (<NAME>)


if [ $# -ne 3 ]; then
  echo
  echo "Usage: `basename $0` 1 All a1234b-c5678d"
  echo
  echo "The first argument is the slave_num of this telemetry slave."
  echo "The second argument is the type of pagesets to create from the 1M list"\
       "Eg: All, Filtered, 100k, 10k, Deeplinks."
  echo "The third argument is the name of the directory where the chromium" \
       "build which will be used for this run is stored."
  echo
  exit 1
fi

SLAVE_NUM=$1
PAGESETS_TYPE=$2
CHROMIUM_BUILD_DIR=$3

source ../vm_config.sh
source vm_utils.sh

create_worker_file $RECORD_WPR_ACTIVITY

source vm_setup_slave.sh

# Create the webpages_archive directory.
mkdir -p /home/default/storage/webpages_archive/$PAGESETS_TYPE/
rm -rf /home/default/storage/webpages_archive/$PAGESETS_TYPE/*

for page_set in /home/default/storage/page_sets/$PAGESETS_TYPE/*; do
  if [[ -f $page_set ]]; then
    echo "========== Processing $page_set =========="
    pageset_basename=`basename $page_set`
    if [ "$PAGESETS_TYPE" == "Filtered" ]; then
      # Since the archive already exists in 'All' do not run record_wpr.
      pageset_filename="${pageset_basename%.*}"
      cp  /home/default/storage/webpages_archive/All/${pageset_filename}* /home/default/storage/webpages_archive/$PAGESETS_TYPE/
      echo "========== $page_set copied over from All =========="
    else
      check_and_run_xvfb
      DISPLAY=:0 timeout 600 tools/perf/record_wpr --extra-browser-args=--disable-setuid-sandbox --browser-executable=/home/default/storage/chromium-builds/${CHROMIUM_BUILD_DIR}/chrome --browser=exact $page_set
      if [ $? -eq 124 ]; then
        echo "========== $page_set timed out! =========="
{'fraction_non_alphanumeric': 0.12866666666666668, 'fraction_numerical': 0.015111111111111112, 'mean_word_length': 3.9900221729490024, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5220375', 'n_tokens_mistral': 1555, 'n_tokens_neox': 1456, 'n_words': 410}