<commit_before>#!/bin/bash

set -eux

CIRCLE_NODE_INDEX="${CIRCLE_NODE_INDEX-0}"
CIRCLE_NODE_TOTAL="${CIRCLE_NODE_TOTAL-1}"

function is_shard() {
  test $(($1 % $CIRCLE_NODE_TOTAL)) -eq $CIRCLE_NODE_INDEX
}

function fetch_docker() {
  local user="$1"
  local repo="${2}"
  local tag="${3}"
  local name="${user}/${repo}"
  local ref="${name}:${tag}"
  if ! docker images --format {{.Repository}}:{{.Tag}} | grep -q "${ref}"; then
    # If we have a saved image, load it.
    imgcache="${builder_dir}/${user}.${repo}.tar"
    if [[ -e "${imgcache}" ]]; then
      time docker load -i "${imgcache}"
    fi

    # If we still don't have the tag we want: pull it and save it.
    if ! docker images --format {{.Repository}}:{{.Tag}} | grep -q "${ref}"; then
      time docker pull "${ref}"
      time docker save -o "${imgcache}" "${ref}"
    fi
  fi

  if ! docker images --format {{.Repository}}:{{.Tag}} | grep -q "${name}:latest"; then
    docker tag "${ref}" "${name}:latest"
  fi
}

# This is mildly tricky: This script runs itself recursively. The
# first time it is run it does not take the if-branch below and
# executes on the host computer. As a final step it uses the
# builder.sh script to run itself inside of docker passing "docker" as
# the argument causing the commands in the if-branch to be executed
# within the docker container.
if [ "${1-}" = "docker" ]; then
  go get -u github.com/robfig/glock
  glock sync -n < GLOCKFILE

  # Be careful to keep the dependencies built for each shard in sync
  # with the dependencies used by each shard in circle-test.sh.

  # Note the ordering here and later in this script of the shard tests
  # is a bit odd. We perform the tests in this order to maintain the
  # same build order as the pre-parallel deps script.
<commit_msg>build: comment out broken `grep` code, add TODOs<commit_after>#!/bin/bash

set -eux

CIRCLE_NODE_INDEX="${CIRCLE_NODE_INDEX-0}"
CIRCLE_NODE_TOTAL="${CIRCLE_NODE_TOTAL-1}"

function is_shard() {
  test $(($1 % $CIRCLE_NODE_TOTAL)) -eq $CIRCLE_NODE_INDEX
}

function fetch_docker() {
  local user="$1"
  local repo="${2}"
  local tag="${3}"
  local name="${user}/${repo}"
  local ref="${name}:${tag}"
  # TODO(tamird): uncomment when CircleCI supports `docker images --format`
  #
  # if ! docker images --format {{.Repository}}:{{.Tag}} | grep -q "${ref}"; then
    # If we have a saved image, load it.
    imgcache="${builder_dir}/${user}.${repo}.tar"
    if [[ -e "${imgcache}" ]]; then
      time docker load -i "${imgcache}"
    fi

    # TODO(tamird): uncomment when CircleCI supports `docker images --format`
    #
    # If we still don't have the tag we want: pull it and save it.
    # if ! docker images --format {{.Repository}}:{{.Tag}} | grep -q "${ref}"; then
      time docker pull "${ref}"
      time docker save -o "${imgcache}" "${ref}"
    # fi
  # fi


  # TODO(tamird): uncomment when CircleCI supports `docker images --format`
  #
  # if ! docker images --format {{.Repository}}:{{.Tag}} | grep -q "${name}:latest"; then
    docker tag "${ref}" "${name}:latest"
  # fi
}

# This is mildly tricky: This script runs itself recursively. The
# first time it is run it does not take the if-branch below and
# executes on the host computer. As a final step it uses the
# builder.sh script to run itself inside of docker passing "docker" as
# the argument causing the commands in the if-branch to be executed
# within the docker container.
if [ "${1-}" = "docker" ]; then
  go get -u github.com/robfig/glock
  glock sync -n < GLOCKFILE

  # Be careful to keep the dependencies built for each shard in sync
  # with the dependencies used by each shard in circle-test.sh.

  # Note the ordering here and later in this script of the shard tests
  # is a bit odd. We perform the tests in this order to maintain the
  # same build order as the pre-parallel deps script.

{'fraction_non_alphanumeric': 0.14060072501294665, 'fraction_numerical': 0.0036250647332988087, 'mean_word_length': 3.4453394706559264, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2807115', 'n_tokens_mistral': 1272, 'n_tokens_neox': 1209, 'n_words': 491}