<commit_before>#!/bin/bash
#
# @author: <NAME>
#

set -e

# ===========================================================================
function contains() {
  STR=${1?} && shift # colon-separated
  TARGET=${1?}
  echo ${STR?} | tr ':' '\n' | awk '/^'"${TARGET?}"'$/'
}

# ===========================================================================
function fetch() {
  URL=${1?} && shift
  DESTINATION_DIR=${1?} && shift

  IS_GIT=$(echo "${URL?}" | awk '/\.git$/')
  if [ -n "${IS_GIT?}" ]; then
    fetch_git "${URL?}" "${DESTINATION_DIR?}"   
  else
    fetch_non_git "${URL?}" "${DESTINATION_DIR?}"
  fi
  
  find ${DESTINATION_DIR?} -maxdepth 1
}

# ---------------------------------------------------------------------------
function fetch_git() { # private
  URL=${1?} && shift
  DESTINATION_DIR=${1?} && shift
  
  git clone ${URL?} ${DESTINATION_DIR?} 2>&1
}

# ---------------------------------------------------------------------------
function fetch_non_git() { # private
  URL=${1?} && shift
  DESTINATION_DIR=${1?} && shift
  
   IS_TAR_GZ=$(echo "${URL?}" | awk '/\.tar.gz/ || /\.tgz/')
  IS_TAR_BZ2=$(echo "${URL?}" | awk '/\.tar.bz2/')
      IS_ZIP=$(echo "${URL?}" | awk '/\.zip/')

  TEMP_DIR=$(mktemp -d)
  
  # fetch tarball
  if [ -n "${IS_TAR_GZ?}" -o -n "${IS_TAR_BZ2?}" ]; then
    if [ -n "${IS_TAR_GZ?}" ]; then FLAG="z"; else FLAG="j"; fi
    wget -O- ${URL?} | tar ${FLAG?}x -C ${TEMP_DIR?} 2>&1 # TODO: can rename ouput dir too?  
    mv ${TEMP_DIR?}/* ${DESTINATION_DIR?}
    
  # fetch zip file
  elif [ -n "${IS_ZIP?}" ]; then # can't use stdin
    ZIP_FILE=${TEMP_DIR?}/tmp.zip  
    wget -O ${ZIP_FILE?} ${URL?} 2>&1
    unzip -d ${DESTINATION_DIR?} ${ZIP_FILE?}
    rm ${ZIP_FILE?}
    
  else
    echo "ERROR: unsupported extension for '${URL?}'"
    exit 1
  fi
  
  rmdir ${TEMP_DIR?}
}

# ===========================================================================
function generate_timestamp() {
  date '+%y%m%d%H%M%S'
}

# ===========================================================================
function section_downloading() {
  COMPONENT=${1?}
  section "downloading" ${COMPONENT?}
}

# ---------------------------------------------------------------------------
function section_configuring() {
  COMPONENT=${1?}
  section "configuring" ${COMPONENT?}
}

# ---------------------------------------------------------------------------
function section_cross_compiling() {
  COMPONENT=${1?}
  section "cross-compiling" ${COMPONENT?}
}

# ---------------------------------------------------------------------------
function section_compiling() {
  COMPONENT=${1?}
  section "compiling" ${COMPONENT?}
}

# ---------------------------------------------------------------------------
function section_installing() {
  COMPONENT=${1?}
  section "installing" ${COMPONENT?}
}

# ---------------------------------------------------------------------------
function section_hacking() {
  COMPONENT=${1?}
  section "hacking" ${COMPONENT?}
}

# ---------------------------------------------------------------------------
function section() {
  ACTION=${1?}
  TARGET=${2?}
  echo -e "\n\n"
  printf '=%.0s' {1..75}
  echo -e "\n${ACTION?}: ${TARGET?}\n\n"
}

# ===========================================================================
function export_cross_compiler() {
  export  CC=${GCC_CROSS_COMPILER?} # can't be readonly
  export CXX=${GXX_CROSS_COMPILER?} # can't be readonly

  # necessary for gmp and curl
  export PATH="$PATH:${CROSS_COMPILER_ROOT_DIR?}/bin"
}

# ---------------------------------------------------------------------------
function sanity_check_cross_compiler() {
  (echo $CC   | grep ${TARGET_ARCHITECTURE?} >&/dev/null && \
   echo $CXX  | grep ${TARGET_ARCHITECTURE?} >&/dev/null && \
   echo $PATH | grep ${TARGET_ARCHITECTURE?} >&/dev/null) || {
    echo "ERROR: CC and/or CXX and/or PATH are not properly configured: CC='$CC', CXX='$CXX', PATH='$PATH'"
    exit 1
  }
}

# ===========================================================================
function return_code() {
  CODE=${1?}
  echo -e "\nreturn code: '${CODE?}'"
}

# ===========================================================================
# populates CMake toolchain file (see http://www.vtk.org/Wiki/CMake_Cross_Compiling)
function get_cmake_toolchain_file_content() {
  cat << EOF
    SET(CMAKE_SYSTEM_NAME Linux)
    SET(CMAKE_SYSTEM_VERSION 1)
    SET(CMAKE_C_COMPILER     ${GCC_CROSS_COMPILER?})
    SET(CMAKE_CXX_COMPILER   ${GXX_CROSS_COMPILER?})
    SET(CMAKE_FIND_ROOT_PATH ${CROSS_COMPILER_ROOT_DIR?})
    SET(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
    SET(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY NEVER)
    SET(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE NEVER)
EOF
}

# ===========================================================================
function set_cmake_paths() {
  PACKAGES=${1?}
  set_cmake_library_path "${PACKAGES?}"
  set_cmake_include_path "${PACKAGES?}"
  check_cmake_paths
  format_cmake_paths
  export PATH="${CMAKE_INSTALL_DIR?}/usr/local/bin:$PATH"
}

# ---------------------------------------------------------------------------
# TODO: generalize
function set_cmake_library_path() { # private
  LIBS=${1?}
  unset CMAKE_LIBRARY_PATH
  
  if [ -n "$(contains ${LIBS?} ${LIBSCRYPT?})" ]; then
    CMAKE_LIBRARY_PATH="$CMAKE_LIBRARY_PATH:${LIBSCRYPT_INSTALL_DIR?}/lib"
  fi
  if [ -n "$(contains ${LIBS?} ${JSONCPP?})" ]; then
    CMAKE_LIBRARY_PATH="$CMAKE_LIBRARY_PATH:${JSONCPP_INSTALL_DIR?}/lib"
  fi
  if [ -n "$(contains ${LIBS?} ${BOOST?})" ]; then  
    CMAKE_LIBRARY_PATH="$CMAKE_LIBRARY_PATH:${BOOST_INSTALL_DIR?}/lib"
  fi<commit_msg>removed obsolete instruction<commit_after>#!/bin/bash
#
# @author: <NAME>
#

set -e

# ===========================================================================
function contains() {
  STR=${1?} && shift # colon-separated
  TARGET=${1?}
  echo ${STR?} | tr ':' '\n' | awk '/^'"${TARGET?}"'$/'
}

# ===========================================================================
function fetch() {
  URL=${1?} && shift
  DESTINATION_DIR=${1?} && shift

  IS_GIT=$(echo "${URL?}" | awk '/\.git$/')
  if [ -n "${IS_GIT?}" ]; then
    fetch_git "${URL?}" "${DESTINATION_DIR?}"   
  else
    fetch_non_git "${URL?}" "${DESTINATION_DIR?}"
  fi
  
  find ${DESTINATION_DIR?} -maxdepth 1
}

# ---------------------------------------------------------------------------
function fetch_git() { # private
  URL=${1?} && shift
  DESTINATION_DIR=${1?} && shift
  
  git clone ${URL?} ${DESTINATION_DIR?} 2>&1
}

# ---------------------------------------------------------------------------
function fetch_non_git() { # private
  URL=${1?} && shift
  DESTINATION_DIR=${1?} && shift
  
   IS_TAR_GZ=$(echo "${URL?}" | awk '/\.tar.gz/ || /\.tgz/')
  IS_TAR_BZ2=$(echo "${URL?}" | awk '/\.tar.bz2/')
      IS_ZIP=$(echo "${URL?}" | awk '/\.zip/')

  TEMP_DIR=$(mktemp -d)
  
  # fetch tarball
  if [ -n "${IS_TAR_GZ?}" -o -n "${IS_TAR_BZ2?}" ]; then
    if [ -n "${IS_TAR_GZ?}" ]; then FLAG="z"; else FLAG="j"; fi
    wget -O- ${URL?} | tar ${FLAG?}x -C ${TEMP_DIR?} 2>&1 # TODO: can rename ouput dir too?  
    mv ${TEMP_DIR?}/* ${DESTINATION_DIR?}
    
  # fetch zip file
  elif [ -n "${IS_ZIP?}" ]; then # can't use stdin
    ZIP_FILE=${TEMP_DIR?}/tmp.zip  
    wget -O ${ZIP_FILE?} ${URL?} 2>&1
    unzip -d ${DESTINATION_DIR?} ${ZIP_FILE?}
    rm ${ZIP_FILE?}
    
  else
    echo "ERROR: unsupported extension for '${URL?}'"
    exit 1
  fi
  
  rmdir ${TEMP_DIR?}
}

# ===========================================================================
function generate_timestamp() {
  date '+%y%m%d%H%M%S'
}

# ===========================================================================
function section_downloading() {
  COMPONENT=${1?}
  section "downloading" ${COMPONENT?}
}

# ---------------------------------------------------------------------------
function section_configuring() {
  COMPONENT=${1?}
  section "configuring" ${COMPONENT?}
}

# ---------------------------------------------------------------------------
function section_cross_compiling() {
  COMPONENT=${1?}
  section "cross-compiling" ${COMPONENT?}
}

# ---------------------------------------------------------------------------
function section_compiling() {
  COMPONENT=${1?}
  section "compiling" ${COMPONENT?}
}

# ---------------------------------------------------------------------------
function section_installing() {
  COMPONENT=${1?}
  section "installing" ${COMPONENT?}
}

# ---------------------------------------------------------------------------
function section_hacking() {
  COMPONENT=${1?}
  section "hacking" ${COMPONENT?}
}

# ---------------------------------------------------------------------------
function section() {
  ACTION=${1?}
  TARGET=${2?}
  echo -e "\n\n"
  printf '=%.0s' {1..75}
  echo -e "\n${ACTION?}: ${TARGET?}\n\n"
}

# ===========================================================================
function export_cross_compiler() {
  export  CC=${GCC_CROSS_COMPILER?} # can't be readonly
  export CXX=${GXX_CROSS_COMPILER?} # can't be readonly

  # necessary for gmp and curl
  export PATH="$PATH:${CROSS_COMPILER_ROOT_DIR?}/bin"
}

# ---------------------------------------------------------------------------
function sanity_check_cross_compiler() {
  (echo $CC   | grep ${TARGET_ARCHITECTURE?} >&/dev/null && \
   echo $CXX  | grep ${TARGET_ARCHITECTURE?} >&/dev/null && \
   echo $PATH | grep ${TARGET_ARCHITECTURE?} >&/dev/null) || {
    echo "ERROR: CC and/or CXX and/or PATH are not properly configured: CC='$CC', CXX='$CXX', PATH='$PATH'"
    exit 1
  }
}

# ===========================================================================
function return_code() {
  CODE=${1?}
  echo -e "\nreturn code: '${CODE?}'"
}

# ===========================================================================
# populates CMake toolchain file (see http://www.vtk.org/Wiki/CMake_Cross_Compiling)
function get_cmake_toolchain_file_content() {
  cat << EOF
    SET(CMAKE_SYSTEM_NAME Linux)
    SET(CMAKE_SYSTEM_VERSION 1)
    SET(CMAKE_C_COMPILER     ${GCC_CROSS_COMPILER?})
    SET(CMAKE_CXX_COMPILER   ${GXX_CROSS_COMPILER?})
    SET(CMAKE_FIND_ROOT_PATH ${CROSS_COMPILER_ROOT_DIR?})
    SET(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
    SET(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY NEVER)
    SET(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE NEVER)
EOF
}

# ===========================================================================
function set_cmake_paths() {
  PACKAGES=${1?}
  set_cmake_library_path "${PACKAGES?}"
  set_cmake_include_path "${PACKAGES?}"
  check_cmake_paths
  format_cmake_paths
}

# ---------------------------------------------------------------------------
# TODO: generalize
function set_cmake_library_path() { # private
  LIBS=${1?}
  unset CMAKE_LIBRARY_PATH
  
  if [ -n "$(contains ${LIBS?} ${LIBSCRYPT?})" ]; then
    CMAKE_LIBRARY_PATH="$CMAKE_LIBRARY_PATH:${LIBSCRYPT_INSTALL_DIR?}/lib"
  fi
  if [ -n "$(contains ${LIBS?} ${JSONCPP?})" ]; then
    CMAKE_LIBRARY_PATH="$CMAKE_LIBRARY_PATH:${JSONCPP_INSTALL_DIR?}/lib"
  fi
  if [ -n "$(contains ${LIBS?} ${BOOST?})" ]; then  
    CMAKE_LIBRARY_PATH="$CMAKE_LIBRARY_PATH:${BOOST_INSTALL_DIR?}/lib"
  fi
{'fraction_non_alphanumeric': 0.38798744957418196, 'fraction_numerical': 0.006454504706409681, 'mean_word_length': 2.490613266583229, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4984949', 'n_tokens_mistral': 3807, 'n_tokens_neox': 3540, 'n_words': 657}