<commit_before>#!/bin/bash
#
# Runs a Lua script from Google Storage on the SKP files on this slave.
#
# The script should be run from the cluster-telemetry-slave GCE instance's
# /b/skia-repo/buildbot/cluster_telemetry/telemetry_slave_scripts
# directory.
#
# Copyright 2013 Google Inc. All Rights Reserved.
# Author: <EMAIL> (<NAME>)

if [ $# -ne 5 ]; then
  echo
  echo "Usage: `basename $0` 1 gs://chromium-skia-gm/telemetry/lua_scripts/test.lua" \
    "rmistry-2013-05-24.07-34-05"
  echo
  echo "The first argument is the slave_num of this telemetry slave."
  echo "The second argument is the Google Storage location of the Lua script."
  echo "The third argument is a unique run id (typically requester + timestamp)."
  echo "The fourth argument is the type of pagesets to create from the 1M list" \
       "Eg: All, Filtered, 100k, 10k, Deeplinks."
  echo "The fifth argument is the name of the directory where the chromium" \
       "build which will be used for this run is stored."
  echo
  exit 1
fi

SLAVE_NUM=$1
LUA_SCRIPT_GS_LOCATION=$2
RUN_ID=$3
PAGESETS_TYPE=$4
CHROMIUM_BUILD_DIR=$5

source vm_utils.sh

WORKER_FILE=LUA.$RUN_ID
LUA_FILE=$RUN_ID.lua
LUA_OUTPUT_FILE=$RUN_ID.lua-output
LUA_LOG_FILE=$RUN_ID.lua.log
create_worker_file $WORKER_FILE

# Sync trunk.
cd /b/skia-repo/trunk
for i in {1..3}; do /b/depot_tools/gclient sync && break || sleep 2; done

# Build tools.
make clean
GYP_DEFINES="skia_warnings_as_errors=0" make tools BUILDTYPE=Release

# Download the SKP files from Google Storage if the local TIMESTAMP is out of date.
mkdir -p /b/storage/skps/$PAGESETS_TYPE/$CHROMIUM_BUILD_DIR/
are_timestamps_equal /b/storage/skps/$PAGESETS_TYPE/$CHROMIUM_BUILD_DIR gs://chromium-skia-gm/telemetry/skps/slave$SLAVE_NUM/$PAGESETS_TYPE/$CHROMIUM_BUILD_DIR
if [ $? -eq 1 ]; then
  gsutil -m cp gs://chromium-skia-gm/telemetry/skps/slave$SLAVE_NUM/$PAGESETS_TYPE/$CHROMIUM_BUILD_DIR/* /b/storage/skps/$PAGESETS_TYPE/$CHROMIUM_BUILD_DIR/
fi

# Copy the lua script from Google Storage to /tmp.
gsutil cp $LUA_SCRIPT_GS_LOCATION /tmp/$LUA_FILE

# Run lua_pictures.
cd out/Release
./lua_pictures --skpPath /b/storage/skps/$PAGESETS_TYPE/$CHROMIUM_BUILD_DIR/ --luaFile /tmp/$LUA_FILE > /tmp/$LUA_OUTPUT_FILE &> /tmp/$LUA_LOG_FILE

# Copy the outputs of the lua script to Google Storage.
gsutil cp /tmp/$LUA_OUTPUT_FILE gs://chromium-skia-gm/telemetry/lua-outputs/slave$SLAVE_NUM/$LUA_OUTPUT_FILE
gsutil cp /tmp/$LUA_LOG_FILE gs://chromium-skia-gm/telemetry/lua-logs/slave$SLAVE_NUM/$LUA_LOG_FILE

# Clean up logs and the worker file.
rm -rf /tmp/*${RUN_ID}*
delete_worker_file $WORKER_FILE
<commit_msg>Fix redirection in vm_run_lua_on_skps<commit_after>#!/bin/bash
#
# Runs a Lua script from Google Storage on the SKP files on this slave.
#
# The script should be run from the cluster-telemetry-slave GCE instance's
# /b/skia-repo/buildbot/cluster_telemetry/telemetry_slave_scripts
# directory.
#
# Copyright 2013 Google Inc. All Rights Reserved.
# Author: <EMAIL> (<NAME>)

if [ $# -ne 5 ]; then
  echo
  echo "Usage: `basename $0` 1 gs://chromium-skia-gm/telemetry/lua_scripts/test.lua" \
    "rmistry-2013-05-24.07-34-05"
  echo
  echo "The first argument is the slave_num of this telemetry slave."
  echo "The second argument is the Google Storage location of the Lua script."
  echo "The third argument is a unique run id (typically requester + timestamp)."
  echo "The fourth argument is the type of pagesets to create from the 1M list" \
       "Eg: All, Filtered, 100k, 10k, Deeplinks."
  echo "The fifth argument is the name of the directory where the chromium" \
       "build which will be used for this run is stored."
  echo
  exit 1
fi

SLAVE_NUM=$1
LUA_SCRIPT_GS_LOCATION=$2
RUN_ID=$3
PAGESETS_TYPE=$4
CHROMIUM_BUILD_DIR=$5

source vm_utils.sh

WORKER_FILE=LUA.$RUN_ID
LUA_FILE=$RUN_ID.lua
LUA_OUTPUT_FILE=$RUN_ID.lua-output
LUA_LOG_FILE=$RUN_ID.lua.log
create_worker_file $WORKER_FILE

# Sync trunk.
cd /b/skia-repo/trunk
for i in {1..3}; do /b/depot_tools/gclient sync && break || sleep 2; done

# Build tools.
make clean
GYP_DEFINES="skia_warnings_as_errors=0" make tools BUILDTYPE=Release

# Download the SKP files from Google Storage if the local TIMESTAMP is out of date.
mkdir -p /b/storage/skps/$PAGESETS_TYPE/$CHROMIUM_BUILD_DIR/
are_timestamps_equal /b/storage/skps/$PAGESETS_TYPE/$CHROMIUM_BUILD_DIR gs://chromium-skia-gm/telemetry/skps/slave$SLAVE_NUM/$PAGESETS_TYPE/$CHROMIUM_BUILD_DIR
if [ $? -eq 1 ]; then
  gsutil -m cp gs://chromium-skia-gm/telemetry/skps/slave$SLAVE_NUM/$PAGESETS_TYPE/$CHROMIUM_BUILD_DIR/* /b/storage/skps/$PAGESETS_TYPE/$CHROMIUM_BUILD_DIR/
fi

# Copy the lua script from Google Storage to /tmp.
gsutil cp $LUA_SCRIPT_GS_LOCATION /tmp/$LUA_FILE

# Run lua_pictures.
cd out/Release
./lua_pictures --skpPath /b/storage/skps/$PAGESETS_TYPE/$CHROMIUM_BUILD_DIR/ --luaFile /tmp/$LUA_FILE > /tmp/$LUA_OUTPUT_FILE 2> /tmp/$LUA_LOG_FILE

# Copy the outputs of the lua script to Google Storage.
gsutil cp /tmp/$LUA_OUTPUT_FILE gs://chromium-skia-gm/telemetry/lua-outputs/slave$SLAVE_NUM/$LUA_OUTPUT_FILE
gsutil cp /tmp/$LUA_LOG_FILE gs://chromium-skia-gm/telemetry/lua-logs/slave$SLAVE_NUM/$LUA_LOG_FILE

# Clean up logs and the worker file.
rm -rf /tmp/*${RUN_ID}*
delete_worker_file $WORKER_FILE

{'fraction_non_alphanumeric': 0.10905976398934146, 'fraction_numerical': 0.014655500570993529, 'mean_word_length': 4.428719008264463, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3018982', 'n_tokens_mistral': 2065, 'n_tokens_neox': 1961, 'n_words': 514}