<commit_before>#!/bin/bash
#
# Copyright 2016 The Kubernetes Authors All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# GLOBALS
JOB_CACHE_DIR=/tmp/buildresults-cache

##############################################################################
# Builds a cache mapping Jenkins run numbers to build versions including
# build numbers and associated hashes.
# Used by release::set_build_version()
#
# @param job - Jenkins job name
#
release::update_job_cache () {
  local job=$1
  local logroot="gs://kubernetes-jenkins/logs"
  local cache_limit=50
  local buildlog
  local j
  local i
  local run
  local cnt
  local last_value
  local -a JOB

  mkdir -p $JOB_CACHE_DIR

  job_file=$JOB_CACHE_DIR/$job

  if ((FLAGS_verbose)); then
    if [[ -f $job_file ]]; then
      logecho -n "Updating"
      source $job_file
    else
      logecho -n "Creating"
    fi
    logecho -r " $job cache..."
  fi

  run=$($GSUTIL cat $logroot/$job/latest-build.txt 2>/dev/null) || return

  for ((cnt=0;cnt<=$cache_limit;cnt++)); do
    # Once we hit one that is set, break out
    # the cache should be complete after that
    [[ -n ${JOB[$run]} ]] && break

    buildlog=$($GSUTIL cat $logroot/$job/$run/build-log.txt 2>/dev/null)
    if [[ "$buildlog" =~ Uploading\ build\ result:\ SUCCESS ]] &&
       [[ "$buildlog" =~ \
          (build_version=|Found Kubernetes version: )(${VER_REGEX[release]}\.${VER_REGEX[build]}) ]]; then
      JOB[$run]=${BASH_REMATCH[2]}
    fi
    ((run--))
  done

  # Write out cache file
  # Reverse sort the array
  for i in $(for n in ${!JOB[*]}; do echo $n; done|sort -rh); do
    # and dedup
    [[ ${JOB[$i]} != $last_value ]] && echo JOB[$i]=${JOB[$i]}
    last_value=${JOB[$i]}
  done > $job_file
}

##############################################################################
# Sets the JENKINS_BUILD_VERSION global by cross checking it against a set of
# critical jenkins build jobs
# @param branch - branch name
#
# TODO:
# * Ability to point to a particular primary job hash and validate it
#   - Useful if wanting to go back to last good from a few days ago
# * Make use of builder average time to find a better search algo
#   - find longest running green and use reasonable offsets to search for
#     dependents to reduce search time
# * e2e-gce: :20
# * e2e-gce-scalability: :42
# * e2e-gce-serial: 3:30
# * e2e-gce-slow: 1:30
# * kubemark-5-gce: 1:10
# * e2e-gce-reboot: :47
# * test-go: :45
# * e2e-gke: :37
# * e2e-gke-slow: :50
release::set_build_version () {
  local branch=$1
  local build_version
  local build_number
  local build_sha1
  local run
  local giveup_build_number=999999
  local job_count=0
  local max_job_length
  local other_job
  local good_job
  local branch_head
  local branch_suffix
  [[ $branch =~ release- ]] && branch_suffix="-$branch"
  local main_job="kubernetes-e2e-gce$branch_suffix"
  declare -a job_array good_jobs

  # Would be nice to pull/generate these jobs dynamically filtered through
  # a pattern like gce,gke so new/changed testing jobs don't require
  # updating here, but it turns out way too much hardcoding would need
  # to be done to make this 'dynamic', so just list these explicitly.
  #all_jobs=$($JCURL -s "http://kubekins.dls.corp.google.com/view/Critical%20Builds/api/json" |jq -r '.jobs[] | .name' | egrep -v '-soak-|-build')
  local -a gce_jobs=("kubernetes-e2e-gce-serial$branch_suffix"
                     "kubernetes-e2e-gce-slow$branch_suffix"
                     "kubernetes-kubemark-5-gce$branch_suffix"
                     "kubernetes-e2e-gce-reboot$branch_suffix"
                     "kubernetes-e2e-gce-scalability$branch_suffix"
                     "kubernetes-test-go$branch_suffix"
                    )

  # kubernetes-e2e-gke-subnet - Uses a branch version?
  # kubernetes-e2e-gke-test - Uses a branch version?
  #"kubernetes-e2e-gke-serial"
  local -a gke_jobs=(
                     "kubernetes-e2e-gke$branch_suffix"<commit_msg>Use *.json files rather than build-log.txt for building cache.<commit_after>#!/bin/bash
#
# Copyright 2016 The Kubernetes Authors All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# GLOBALS
JOB_CACHE_DIR=/tmp/buildresults-cache

##############################################################################
# Builds a cache mapping Jenkins run numbers to build versions including
# build numbers and associated hashes.
# Used by release::set_build_version()
#
# @param job - Jenkins job name
#
release::update_job_cache () {
  local job=$1
  local logroot="gs://kubernetes-jenkins/logs"
  local cache_limit=50
  local buildstate
  local j
  local i
  local run
  local cnt
  local last_value
  local -a JOB

  mkdir -p $JOB_CACHE_DIR

  job_file=$JOB_CACHE_DIR/$job

  if ((FLAGS_verbose)); then
    if [[ -f $job_file ]]; then
      logecho -n "Updating"
      source $job_file
    else
      logecho -n "Creating"
    fi
    logecho -r " $job cache..."
  fi

  run=$($GSUTIL cat $logroot/$job/latest-build.txt 2>/dev/null) || return

  for ((cnt=0;cnt<=$cache_limit;cnt++)); do
    # Once we hit one that is set, break out
    # the cache should be complete after that
    [[ -n ${JOB[$run]} ]] && break

    buildstate=$($GSUTIL cat $logroot/$job/$run/*.json 2>/dev/null)
    [[ $(echo "$buildstate" | jq -r '.result|values') == "SUCCESS" ]] &&
     JOB[$run]=$(echo "$buildstate" | jq -r '.version|values')

    ((run--))
  done

  # Write out cache file
  # Reverse sort the array
  for i in $(for n in ${!JOB[*]}; do echo $n; done|sort -rh); do
    # and dedup
    [[ ${JOB[$i]} != $last_value ]] && echo JOB[$i]=${JOB[$i]}
    last_value=${JOB[$i]}
  done > $job_file
}

##############################################################################
# Sets the JENKINS_BUILD_VERSION global by cross checking it against a set of
# critical jenkins build jobs
# @param branch - branch name
#
# TODO:
# * Ability to point to a particular primary job hash and validate it
#   - Useful if wanting to go back to last good from a few days ago
# * Make use of builder average time to find a better search algo
#   - find longest running green and use reasonable offsets to search for
#     dependents to reduce search time
# * e2e-gce: :20
# * e2e-gce-scalability: :42
# * e2e-gce-serial: 3:30
# * e2e-gce-slow: 1:30
# * kubemark-5-gce: 1:10
# * e2e-gce-reboot: :47
# * test-go: :45
# * e2e-gke: :37
# * e2e-gke-slow: :50
release::set_build_version () {
  local branch=$1
  local build_version
  local build_number
  local build_sha1
  local run
  local giveup_build_number=999999
  local job_count=0
  local max_job_length
  local other_job
  local good_job
  local branch_head
  local branch_suffix
  [[ $branch =~ release- ]] && branch_suffix="-$branch"
  local main_job="kubernetes-e2e-gce$branch_suffix"
  declare -a job_array good_jobs

  # Would be nice to pull/generate these jobs dynamically filtered through
  # a pattern like gce,gke so new/changed testing jobs don't require
  # updating here, but it turns out way too much hardcoding would need
  # to be done to make this 'dynamic', so just list these explicitly.
  #all_jobs=$($JCURL -s "http://kubekins.dls.corp.google.com/view/Critical%20Builds/api/json" |jq -r '.jobs[] | .name' | egrep -v '-soak-|-build')
  local -a gce_jobs=("kubernetes-e2e-gce-serial$branch_suffix"
                     "kubernetes-e2e-gce-slow$branch_suffix"
                     "kubernetes-kubemark-5-gce$branch_suffix"
                     "kubernetes-e2e-gce-reboot$branch_suffix"
                     "kubernetes-e2e-gce-scalability$branch_suffix"
                     "kubernetes-test-go$branch_suffix"
                    )

  # kubernetes-e2e-gke-subnet - Uses a branch version?
  # kubernetes-e2e-gke-test - Uses a branch version?
  local -a gke_jobs=(
                     "kubernetes-e2e-gke-serial$branch_suffix"
                     "kubernetes-e2e-gke$branch_suffix"
{'fraction_non_alphanumeric': 0.146804447938897, 'fraction_numerical': 0.014489497922048747, 'mean_word_length': 3.295224312590449, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7511769', 'n_tokens_mistral': 3025, 'n_tokens_neox': 2835, 'n_words': 888}