<commit_before># Copyright 2014 Google Inc. All Rights Reserved.
# Author: <EMAIL> (<NAME>)
#
# Converts all JSON page_sets in a directory to Python page_sets.


if [ $# -ne 2 ]; then
  echo
  echo "Usage: `basename $0` 1 All"
  echo
  echo "The first argument is the slave_num of this telemetry slave."
  echo "The second argument is the type of pagesets to create from the 1M" \
       "list Eg: All, Filtered, 100k, 10k."
  echo
  exit 1
fi

SLAVE_NUM=$1
PAGESETS_TYPE=$2

source vm_utils.sh

# Download the page_sets from Google Storage if the local TIMESTAMP is out of
# date.
mkdir -p /b/storage/page_sets/$PAGESETS_TYPE/
are_timestamps_equal /b/storage/page_sets/$PAGESETS_TYPE gs://chromium-skia-gm/telemetry/page_sets/slave$SLAVE_NUM/$PAGESETS_TYPE
if [ $? -eq 1 ]; then
  gsutil -m cp gs://chromium-skia-gm/telemetry/page_sets/slave$SLAVE_NUM/$PAGESETS_TYPE/* \
    /b/storage/page_sets/$PAGESETS_TYPE/
fi

for page_set in /b/storage/page_sets/$PAGESETS_TYPE/*.py; do
  sed -i s/"from telemetry.page.actions.all_page_actions import \*"//g $page_set
  sed -i s/action_runner.RunAction\(ScrollAction\(\)\)/action_runner.ScrollElement\(\)/g $page_set
  sed -i s/"action_runner.RunAction(RepaintContinuouslyAction({'seconds': 5}))"/"action_runner.RepaintContinuously(seconds=5)"/g $page_set
done


# Copy the python page_sets into Google Storage.
gsutil -m cp /b/storage/page_sets/$PAGESETS_TYPE/*py gs://chromium-skia-gm/telemetry/page_sets/slave$SLAVE_NUM/$PAGESETS_TYPE/
# Create a TIMESTAMP file and copy it to Google Storage.
TIMESTAMP=`date +%s`
echo $TIMESTAMP > /tmp/$TIMESTAMP
cp /tmp/$TIMESTAMP /b/storage/page_sets/$PAGESETS_TYPE/TIMESTAMP
gsutil cp /tmp/$TIMESTAMP gs://chromium-skia-gm/telemetry/page_sets/slave$SLAVE_NUM/$PAGESETS_TYPE/TIMESTAMP
rm /tmp/$TIMESTAMP
<commit_msg>Add RunRepaint section to page sets to support the repaint benchmark<commit_after># Copyright 2014 Google Inc. All Rights Reserved.
# Author: <EMAIL> (<NAME>)
#
# Converts all JSON page_sets in a directory to Python page_sets.


if [ $# -ne 2 ]; then
  echo
  echo "Usage: `basename $0` 1 All"
  echo
  echo "The first argument is the slave_num of this telemetry slave."
  echo "The second argument is the type of pagesets to create from the 1M" \
       "list Eg: All, Filtered, 100k, 10k."
  echo
  exit 1
fi

SLAVE_NUM=$1
PAGESETS_TYPE=$2

source vm_utils.sh

# Download the page_sets from Google Storage if the local TIMESTAMP is out of
# date.
mkdir -p /b/storage/page_sets/$PAGESETS_TYPE/
are_timestamps_equal /b/storage/page_sets/$PAGESETS_TYPE gs://chromium-skia-gm/telemetry/page_sets/slave$SLAVE_NUM/$PAGESETS_TYPE
if [ $? -eq 1 ]; then
  gsutil -m cp gs://chromium-skia-gm/telemetry/page_sets/slave$SLAVE_NUM/$PAGESETS_TYPE/* \
    /b/storage/page_sets/$PAGESETS_TYPE/
fi

for page_set in /b/storage/page_sets/$PAGESETS_TYPE/*.py; do
  sed -i '21i\ \ def RunRepaint(self, action_runner):' $page_set
  sed -i '22i\ \ \ \ action_runner.RepaintContinuously(seconds=5)\n' $page_set
done


# Copy the python page_sets into Google Storage.
gsutil -m cp /b/storage/page_sets/$PAGESETS_TYPE/*py gs://chromium-skia-gm/telemetry/page_sets/slave$SLAVE_NUM/$PAGESETS_TYPE/
# Create a TIMESTAMP file and copy it to Google Storage.
TIMESTAMP=`date +%s`
echo $TIMESTAMP > /tmp/$TIMESTAMP
cp /tmp/$TIMESTAMP /b/storage/page_sets/$PAGESETS_TYPE/TIMESTAMP
gsutil cp /tmp/$TIMESTAMP gs://chromium-skia-gm/telemetry/page_sets/slave$SLAVE_NUM/$PAGESETS_TYPE/TIMESTAMP
rm /tmp/$TIMESTAMP

{'fraction_non_alphanumeric': 0.12003454231433507, 'fraction_numerical': 0.01180195739781232, 'mean_word_length': 4.289193302891933, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7487931', 'n_tokens_mistral': 1342, 'n_tokens_neox': 1277, 'n_words': 326}