<commit_before>#!/usr/bin/env bash

set -x # Show commands
set -eu # Errors/undefined vars are fatal
set -o pipefail # Check all commands in a pipeline

sudo apt-get update
sudo apt-get install -y git

# Livegrep (Bazel is needed for Livegrep builds)
sudo apt-get install -y unzip openjdk-8-jdk libssl-dev
# Install Bazel 0.7.0
rm -rf bazel
mkdir bazel
pushd bazel
# Note that bazel unzips itself so we can't just pipe it to sudo bash.
curl -sSfL -O https://github.com/bazelbuild/bazel/releases/download/0.7.0/bazel-0.7.0-without-jdk-installer-linux-x86_64.sh
chmod +x bazel-0.7.0-without-jdk-installer-linux-x86_64.sh
sudo ./bazel-0.7.0-without-jdk-installer-linux-x86_64.sh
popd

# pygit2
sudo apt-get install -y python-virtualenv python-dev libffi-dev cmake

# Other
sudo apt-get install -y parallel realpath unzip python-pip

# Nginx
sudo apt-get install -y nginx

# Install pkg-config (needed for Rust's OpenSSL wrappers)
sudo apt-get install pkg-config

# Install Rust.
curl -sSf https://static.rust-lang.org/rustup.sh | sh -s -- --channel=nightly

# Install codesearch.
rm -rf livegrep
git clone -b mozsearch-version https://github.com/mozsearch/livegrep
pushd livegrep
bazel build //src/tools:codesearch --incompatible_disallow_set_constructor=false
sudo install bazel-bin/src/tools/codesearch /usr/local/bin
popd

# Install AWS scripts.
sudo pip install boto3<commit_msg>Bug 1466404 - Add a gRPC library to the web server provisioning.<commit_after>#!/usr/bin/env bash

set -x # Show commands
set -eu # Errors/undefined vars are fatal
set -o pipefail # Check all commands in a pipeline

sudo apt-get update
sudo apt-get install -y git

# Livegrep (Bazel is needed for Livegrep builds)
sudo apt-get install -y unzip openjdk-8-jdk libssl-dev
# Install Bazel 0.7.0
rm -rf bazel
mkdir bazel
pushd bazel
# Note that bazel unzips itself so we can't just pipe it to sudo bash.
curl -sSfL -O https://github.com/bazelbuild/bazel/releases/download/0.7.0/bazel-0.7.0-without-jdk-installer-linux-x86_64.sh
chmod +x bazel-0.7.0-without-jdk-installer-linux-x86_64.sh
sudo ./bazel-0.7.0-without-jdk-installer-linux-x86_64.sh
popd

# pygit2
sudo apt-get install -y python-virtualenv python-dev libffi-dev cmake

# Other
sudo apt-get install -y parallel realpath unzip python-pip

# Nginx
sudo apt-get install -y nginx

# Install pkg-config (needed for Rust's OpenSSL wrappers)
sudo apt-get install pkg-config

# Install Rust.
curl -sSf https://static.rust-lang.org/rustup.sh | sh -s -- --channel=nightly

# Install codesearch.
rm -rf livegrep
git clone -b mozsearch-version https://github.com/mozsearch/livegrep
pushd livegrep
bazel build //src/tools:codesearch --incompatible_disallow_set_constructor=false
sudo install bazel-bin/src/tools/codesearch /usr/local/bin
popd

# Install gRPC python libs and generate the python modules to communicate with the codesearch server
sudo pip install grpcio grpcio-tools
rm -rf livegrep-grpc
mkdir livegrep-grpc
python -m grpc_tools.protoc --python_out=livegrep-grpc --grpc_python_out=livegrep-grpc -I livegrep/src/proto livegrep/src/proto/livegrep.proto
# Add the generated modules to the python path
SITEDIR=$(python -m site --user-site)
mkdir -p "$SITEDIR"
echo "$PWD/livegrep-grpc" > "$SITEDIR/livegrep.pth"

# Install AWS scripts.
sudo pip install boto3
{'fraction_non_alphanumeric': 0.10088119112731693, 'fraction_numerical': 0.02035855363111516, 'mean_word_length': 3.9803328290468984, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6329904', 'n_tokens_mistral': 1278, 'n_tokens_neox': 1189, 'n_words': 356}