<commit_before>#!/bin/bash

set -eux

CIRCLE_NODE_INDEX="${CIRCLE_NODE_INDEX-0}"
CIRCLE_NODE_TOTAL="${CIRCLE_NODE_TOTAL-1}"

function is_shard() {
  test $(($1 % $CIRCLE_NODE_TOTAL)) -eq $CIRCLE_NODE_INDEX
}

function fetch_docker() {
  local user="$1"
  local repo="${2}"
  local tag="${3}"
  local name="${user}/${repo}"
  local ref="${name}:${tag}"
  if ! docker images --format {{.Repository}}:{{.Tag}} | grep -q "${ref}"; then
    # If we have a saved image, load it.
    imgcache="${builder_dir}/${user}.${repo}.tar"
    if [[ -e "${imgcache}" ]]; then
      time docker load -i "${imgcache}"
    fi

    # If we still don't have the tag we want: pull it and save it.
    if ! docker images --format {{.Repository}}:{{.Tag}} | grep -q "${ref}"; then
      time docker pull "${ref}"
      time docker save -o "${imgcache}" "${ref}"
    fi
  fi

  if ! docker images --format {{.Repository}}:{{.Tag}} | grep -q "${name}:latest"; then
    docker tag "${ref}" "${name}:latest"
  fi
}

# This is mildly tricky: This script runs itself recursively. The
# first time it is run it does not take the if-branch below and
# executes on the host computer. As a final step it uses the
# builder.sh script to run itself inside of docker passing "docker" as
# the argument causing the commands in the if-branch to be executed
# within the docker container.
if [ "${1-}" = "docker" ]; then
  go get -u github.com/robfig/glock
  glock sync -n < GLOCKFILE

  # Be careful to keep the dependencies built for each shard in sync
  # with the dependencies used by each shard in circle-test.sh.

  # Note the ordering here and later in this script of the shard tests
  # is a bit odd. We perform the tests in this order to maintain the
  # same build order as the pre-parallel deps script.

  if is_shard 2; then
    # Symlink the cache into the source tree if they don't exist.
    # In circleci they never do, but this script can also be run
    # locally where they might.
    for i in bower_components node_modules typings; do
      if ! [ -e ui/$i ]; then
        # Create the cache directory to avoid errors on `ln` below.
        # `/uicache` is `~/uicache` on the host computer, which is cached
        # by circle-ci.
        mkdir -p /uicache/$i
        ln -s /uicache/$i ui/$i
      fi
    done

    time make -C ui npm.installed     || rm -rf ui/node_modules/{*,.bin} && make -C ui npm.installed
    time make -C ui bower.installed   || rm -rf ui/bower_components/*    && make -C ui bower.installed
    time make -C ui typings.installed || rm -rf ui/typings/*             && make -C ui typings.installed

    # Fix permissions on the ui/typings directory and subdirectories
    # (they lack the execute permission for group/other).
    find /uicache/typings -type d | xargs chmod 0755

    # TODO(pmattis): This works around the problem seen in
    # https://github.com/cockroachdb/cockroach/issues/4013 where
    # certain checks and code generation tools rely on compiled
    # packages. In particular, `stringer` definitely relies on
    # compiled packages for imports and it appears `go vet` is
    # similar. Would be nice to find a different solution, but simply
    # removing the out of date packages does not fix the problem (it
    # causes `stringer` to complain).
    time make build
  fi

  if is_shard 1; then
    time go test -race -v -i ./...
    # We need go2xunit on both shards that run go tests.
    time go install -v github.com/tebeka/go2xunit
  fi

  if is_shard 0; then
    time make install
    time go test -v -i ./...
    time go test -v -c -tags acceptance ./acceptance
  fi

  exit 0
fi

shard1_done="${HOME}/shard1"
shard2_done="${HOME}/shard2"

function notify() {
  if is_shard 1; then
    if ! is_shard 0; then
      time ssh node0 touch "${shard1_done}"
    else
      touch "${shard1_done}"
    fi
  fi

  if is_shard 2; then
    if ! is_shard 0; then
      time ssh node0 touch "${shard2_done}"
    else
      touch "${shard2_done}"
    fi
  fi
}

# Notify shard 0 that shard's 1 and 2 are done if they exit for any
# reason. This prevents shard 0 from spinning forever.
trap "notify" EXIT

# `~/builder` is cached by circle-ci.
builder_dir=~/builder
mkdir -p "${builder_dir}"
du -sh "${builder_dir}"
ls -lah "${builder_dir}"

# If the images below are updated, the new images will not be picked
# up until the tags listed here (or in builder.sh for the builder
# image) are updated.<commit_msg>build: remove outdated comment<commit_after>#!/bin/bash

set -eux

CIRCLE_NODE_INDEX="${CIRCLE_NODE_INDEX-0}"
CIRCLE_NODE_TOTAL="${CIRCLE_NODE_TOTAL-1}"

function is_shard() {
  test $(($1 % $CIRCLE_NODE_TOTAL)) -eq $CIRCLE_NODE_INDEX
}

function fetch_docker() {
  local user="$1"
  local repo="${2}"
  local tag="${3}"
  local name="${user}/${repo}"
  local ref="${name}:${tag}"
  if ! docker images --format {{.Repository}}:{{.Tag}} | grep -q "${ref}"; then
    # If we have a saved image, load it.
    imgcache="${builder_dir}/${user}.${repo}.tar"
    if [[ -e "${imgcache}" ]]; then
      time docker load -i "${imgcache}"
    fi

    # If we still don't have the tag we want: pull it and save it.
    if ! docker images --format {{.Repository}}:{{.Tag}} | grep -q "${ref}"; then
      time docker pull "${ref}"
      time docker save -o "${imgcache}" "${ref}"
    fi
  fi

  if ! docker images --format {{.Repository}}:{{.Tag}} | grep -q "${name}:latest"; then
    docker tag "${ref}" "${name}:latest"
  fi
}

# This is mildly tricky: This script runs itself recursively. The
# first time it is run it does not take the if-branch below and
# executes on the host computer. As a final step it uses the
# builder.sh script to run itself inside of docker passing "docker" as
# the argument causing the commands in the if-branch to be executed
# within the docker container.
if [ "${1-}" = "docker" ]; then
  go get -u github.com/robfig/glock
  glock sync -n < GLOCKFILE

  # Be careful to keep the dependencies built for each shard in sync
  # with the dependencies used by each shard in circle-test.sh.

  # Note the ordering here and later in this script of the shard tests
  # is a bit odd. We perform the tests in this order to maintain the
  # same build order as the pre-parallel deps script.

  if is_shard 2; then
    # Symlink the cache into the source tree if they don't exist.
    # In circleci they never do, but this script can also be run
    # locally where they might.
    for i in bower_components node_modules typings; do
      if ! [ -e ui/$i ]; then
        # Create the cache directory to avoid errors on `ln` below.
        # `/uicache` is `~/uicache` on the host computer, which is cached
        # by circle-ci.
        mkdir -p /uicache/$i
        ln -s /uicache/$i ui/$i
      fi
    done

    time make -C ui npm.installed     || rm -rf ui/node_modules/{*,.bin} && make -C ui npm.installed
    time make -C ui bower.installed   || rm -rf ui/bower_components/*    && make -C ui bower.installed
    time make -C ui typings.installed || rm -rf ui/typings/*             && make -C ui typings.installed

    # Fix permissions on the ui/typings directory and subdirectories
    # (they lack the execute permission for group/other).
    find /uicache/typings -type d | xargs chmod 0755

    # TODO(pmattis): This works around the problem seen in
    # https://github.com/cockroachdb/cockroach/issues/4013 where
    # certain checks and code generation tools rely on compiled
    # packages. In particular, `stringer` definitely relies on
    # compiled packages for imports and it appears `go vet` is
    # similar. Would be nice to find a different solution, but simply
    # removing the out of date packages does not fix the problem (it
    # causes `stringer` to complain).
    time make build
  fi

  if is_shard 1; then
    time go test -race -v -i ./...
    # We need go2xunit on both shards that run go tests.
    time go install -v github.com/tebeka/go2xunit
  fi

  if is_shard 0; then
    time make install
    time go test -v -i ./...
    time go test -v -c -tags acceptance ./acceptance
  fi

  exit 0
fi

shard1_done="${HOME}/shard1"
shard2_done="${HOME}/shard2"

function notify() {
  if is_shard 1; then
    if ! is_shard 0; then
      time ssh node0 touch "${shard1_done}"
    else
      touch "${shard1_done}"
    fi
  fi

  if is_shard 2; then
    if ! is_shard 0; then
      time ssh node0 touch "${shard2_done}"
    else
      touch "${shard2_done}"
    fi
  fi
}

# Notify shard 0 that shard's 1 and 2 are done if they exit for any
# reason. This prevents shard 0 from spinning forever.
trap "notify" EXIT

# `~/builder` is cached by circle-ci.
builder_dir=~/builder
mkdir -p "${builder_dir}"
du -sh "${builder_dir}"
ls -lah "${builder_dir}"

{'fraction_non_alphanumeric': 0.11234794583428964, 'fraction_numerical': 0.008951113151250861, 'mean_word_length': 2.844287604764005, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7516506', 'n_tokens_mistral': 2883, 'n_tokens_neox': 2731, 'n_words': 1148}