<commit_before>#!/bin/bash
# (C) 2014 <NAME>
# License: MPLv2 - see project dir
# Git repository: https://github.com/gunnarx/franca_install_automation
# pull requests welcome

# --------------------------------------------------------------------------
# functions.sh : HELPER FUNCTIONS SHARED BETWEEN ALL VARIANTS
# --------------------------------------------------------------------------

MD5SUM=md5sum   # On MacOS X, the binary is "md5"

debug() {
   $DEBUG && {
      printf '*DEBUG*: ' 1>&2
      echo $@ 1>&2
   }
}

die() {
   echo "Something went wrong.  Message: "
   echo "$@"
   [ -n "$ORIGDIR" ] && cd "$ORIGDIR"
   exit 1
}

# This function sets variable $VAGRANT if we are running under vagrant provisioning
test_vagrant() {
   VAGRANT=""
   echo "$0" | fgrep -q "vagrant-shell" && VAGRANT="yes"
}

# Run command only if Vagrant environment
if_vagrant() {
   [ -n "$VAGRANT" ] && eval $@
}

# Run command only if NOT Vagrant environment
unless_vagrant() {
   [ -z "$VAGRANT" ] && $@
}

# Print a section header
section() {
      # Using printf because "echo -n" is not fully portable
      printf '****************************************************************\n'
      printf '*** ' ; echo $@
      printf '****************************************************************\n'
}

# Print an operation with *** in front of it
step() {
      printf '*** ' ; echo "$@"
}

# Check condition is met or die
ensure() {
   $@ || die "Condition not met: $*"
}

# Check that variable(s) have been defined
defined() {
   for v in $* ; do
      [ -n "$v" ] || "Variable $v not defined in CONFIG?"
   done
}

# Print a warning and pause
# (it will not pause in Vagrant provisioning because there is no terminal)
warn() {
   echo "WARNING: $1"
   echo Hit return to continue
   unless_vagrant read
}

# Print a note, no pause
note() {
   echo "NOTE: $1"
}

# dereference variable:
# This is a kind of weird hack, but it evaluates the variable whose name is
# given by the input.  It is used extensively in this program.
# E.g.: If x=foo ; deref x returns foo and deref $x returns the value of $foo
deref() { eval echo \$$1 ; }

get_md5() {
   $MD5SUM "$1" | cut -b 1-32
}

match_md5() {
   f=$1
   expect_md5=$2
   if [ -z "$expect_md5" ] ; then
     echo "No MD5 defined - must assume it's OK"
     true
   elif [ "$(get_md5 $f)" = "$expect_md5" ] ; then
     true
   else
     false
   fi
}

md5_check() {
   item=$1
   file=$2
   arch=$3

   if [ -n "$arch" ] ; then
      wanted_md5=$(deref ${item}_MD5_${arch})
   else
      wanted_md5=$(deref ${item}_MD5)
   fi

   debug "Checking MD5 $wanted_md5 for $item"
   if match_md5 $file $wanted_md5 ; then
      echo "MD5 ok ($item)"
   else
      die "MD5 checksum ($md5) did not match predefined md5 ($wanted_md5) for item $item.  Check CONFIG file."
   fi
}

download() {
   cd "$DOWNLOAD_DIR"
   $DEBUG && set -x
   downloaded_file=
   outfile=$(basename "$1")
   expected_md5=$2

   if [ -n "$VAGRANT" ] ; then
      progress_flag="-q"     # Be absolutely quiet
   else
      progress_flag="-#"     # Use a small progress bar 
   fi

   # If already exists, we check md5 to know if it is complete and OK
   if [ -f "$outfile" ] ; then
      echo -n "File exists: $PWD/$outfile, checking..."
      if [ -n "$expected_md5" ] ; then
         if match_md5 $outfile $expected_md5 ; then
            echo "OK"
            downloaded_file="$outfile"<commit_msg>Introduce CACHE_DIR for faster CIAT automation<commit_after>#!/bin/bash
# (C) 2014 <NAME>
# License: MPLv2 - see project dir
# Git repository: https://github.com/gunnarx/franca_install_automation
# pull requests welcome

# --------------------------------------------------------------------------
# functions.sh : HELPER FUNCTIONS SHARED BETWEEN ALL VARIANTS
# --------------------------------------------------------------------------

MD5SUM=md5sum   # On MacOS X, the binary is "md5"

debug() {
   $DEBUG && {
      printf '*DEBUG*: ' 1>&2
      echo $@ 1>&2
   }
}

die() {
   echo "Something went wrong.  Message: "
   echo "$@"
   [ -n "$ORIGDIR" ] && cd "$ORIGDIR"
   exit 1
}

# This function sets variable $VAGRANT if we are running under vagrant provisioning
test_vagrant() {
   VAGRANT=""
   echo "$0" | fgrep -q "vagrant-shell" && VAGRANT="yes"
}

# Run command only if Vagrant environment
if_vagrant() {
   [ -n "$VAGRANT" ] && eval $@
}

# Run command only if NOT Vagrant environment
unless_vagrant() {
   [ -z "$VAGRANT" ] && $@
}

# Print a section header
section() {
      # Using printf because "echo -n" is not fully portable
      printf '****************************************************************\n'
      printf '*** ' ; echo $@
      printf '****************************************************************\n'
}

# Print an operation with *** in front of it
step() {
      printf '*** ' ; echo "$@"
}

# Check condition is met or die
ensure() {
   $@ || die "Condition not met: $*"
}

# Check that variable(s) have been defined
defined() {
   for v in $* ; do
      [ -n "$v" ] || "Variable $v not defined in CONFIG?"
   done
}

# Print a warning and pause
# (it will not pause in Vagrant provisioning because there is no terminal)
warn() {
   echo "WARNING: $1"
   echo Hit return to continue
   unless_vagrant read
}

# Print a note, no pause
note() {
   echo "NOTE: $1"
}

# dereference variable:
# This is a kind of weird hack, but it evaluates the variable whose name is
# given by the input.  It is used extensively in this program.
# E.g.: If x=foo ; deref x returns foo and deref $x returns the value of $foo
deref() { eval echo \$$1 ; }

get_md5() {
   $MD5SUM "$1" | cut -b 1-32
}

match_md5() {
   f=$1
   expect_md5=$2
   if [ -z "$expect_md5" ] ; then
     echo "No MD5 defined - must assume it's OK"
     true
   elif [ "$(get_md5 $f)" = "$expect_md5" ] ; then
     true
   else
     false
   fi
}

md5_check() {
   item=$1
   file=$2
   arch=$3

   if [ -n "$arch" ] ; then
      wanted_md5=$(deref ${item}_MD5_${arch})
   else
      wanted_md5=$(deref ${item}_MD5)
   fi

   debug "Checking MD5 $wanted_md5 for $item"
   if match_md5 $file $wanted_md5 ; then
      echo "MD5 ok ($item)"
   else
      die "MD5 checksum ($md5) did not match predefined md5 ($wanted_md5) for item $item.  Check CONFIG file."
   fi
}

download() {
   cd "$DOWNLOAD_DIR"
   $DEBUG && set -x
   downloaded_file=
   outfile=$(basename "$1")
   expected_md5=$2

   if [ -n "$VAGRANT" ] ; then
      progress_flag="-q"     # Be absolutely quiet
   else
      progress_flag="-#"     # Use a small progress bar 
   fi

   # If exists in cache (and not in downloads), copy it from there
   if [ ! -f "$outfile" -a -n "$CACHE_DIR" -a -f "$CACHE_DIR/$outfile" ] ; then
      echo "Found in cache, copying..."
      cp "$CACHE_DIR/$outfile" "$outfile"
   fi

   # If already exists, we check md5 to know if it is complete and OK
   if [ -f "$outfile" ] ; then
      echo -n "File exists: $PWD/$outfile, checking..."
      if [ -n "$expected_md5" ] ; then
         if match_md5 $outfile $expected_md5 ; then
            echo "OK"
            downloaded_file="$outfile"
{'fraction_non_alphanumeric': 0.2029598308668076, 'fraction_numerical': 0.015503875968992248, 'mean_word_length': 2.401725790987536, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5121554', 'n_tokens_mistral': 2425, 'n_tokens_neox': 2325, 'n_words': 787}