<commit_before>#!/bin/bash

die() {
  echo "$@" 1>&2 ; popd 2>/dev/null; exit 1
}

build_install_zstd() {
  TEMP=`mktemp -d` || die "failed to create zstd tmp build dir"
  pushd $TEMP
  wget https://github.com/facebook/zstd/archive/v1.3.2.tar.gz
  tar xzf v1.3.2.tar.gz || die "failed to uncompress zstd repo"
  cd zstd-1.3.2/lib  && make install PREFIX='/usr' || die "zstd build install failed"
  popd && rm -rf $TEMP
}

build_install_blosc() {
  TEMP=`mktemp -d` || die "failed to create blosc tmp build dir"
  pushd $TEMP
  wget https://github.com/Blosc/c-blosc/archive/v1.12.1.tar.gz
  tar xzf v1.12.1.tar.gz || die "failed to uncompress blosc repo"
  cd c-blosc-1.12.1 && mkdir build && cd build
  cmake -DCMAKE_INSTALL_PREFIX='/usr' .. || die "blosc cmake failed"
  cmake --build . --target install || die "bloc build install failed"
  popd && rm -rf $TEMP
}

install_apt_pkgs() {
  apt-get -y install gcc g++ wget cmake \
    zlib1g-dev libbz2-dev liblz4-dev || die "could not install apt pkg dependencies"
}

install_yum_pkgs() {
  yum -y install epel-release &&
  yum -y install gcc gcc-c++ which wget cmake \
    zlib-devel bzip2-devel lz4-devel || die "could not install yum pkg dependencies"
}

install_brew_pkgs() {
  brew install wget cmake lzlib lz4 bzip2 zstd || die "could not install brew pkg dependencies"
  brew install tiledb-inc/stable/blosc || die "could not install blosc pkg dependency"
}

install_deps() {
  if [[ $OSTYPE == linux* ]]; then
    if [ -n "$(command -v apt-get)" ]; then
      install_apt_pkgs 
      # zstd is in later versions of Ubuntu
      if [[ $(apt-cache search libzstd-dev) ]]; then
        apt-get -y install libzstd-dev
      else
        build_install_zstd
      fi
      build_install_blosc
    elif [ -n "$(command -v yum)" ]; then
      install_yum_pkgs
      build_install_zstd
      build_install_blosc
    fi
  elif [[ $OSTYPE == darwin* ]]; then
    if [ -n "$(command -v brew)" ]; then
      install_brew_pkgs
    else
      die "homebrew is not installed!"
    fi
  else
    die "unsupported package management system"
  fi 
}

run() {
  install_deps
}

run <commit_msg>Install latest version of cmake in Linux-based systems<commit_after>#!/bin/bash

die() {
  echo "$@" 1>&2 ; popd 2>/dev/null; exit 1
}

build_install_cmake() {
wget -P /tmp https://cmake.org/files/v3.9/cmake-3.9.4-Linux-x86_64.tar.gz \
    && tar xzf /tmp/cmake-3.9.4-Linux-x86_64.tar.gz -C /tmp \
    && rm /tmp/cmake-3.9.4-Linux-x86_64.tar.gz \
    && cp /tmp/cmake-3.9.4-Linux-x86_64/bin/* /usr/local/bin \
    && cp -r /tmp/cmake-3.9.4-Linux-x86_64/share/cmake-3.9 /usr/local/share \
    && rm -rf /tmp/cmake-3.9.4-Linux-x86_64 || die "failed to build cmake"
}

build_install_zstd() {
  TEMP=`mktemp -d` || die "failed to create zstd tmp build dir"
  pushd $TEMP
  wget https://github.com/facebook/zstd/archive/v1.3.2.tar.gz
  tar xzf v1.3.2.tar.gz || die "failed to uncompress zstd repo"
  cd zstd-1.3.2/lib  && make install PREFIX='/usr' || die "zstd build install failed"
  popd && rm -rf $TEMP
}

build_install_blosc() {
  TEMP=`mktemp -d` || die "failed to create blosc tmp build dir"
  pushd $TEMP
  wget https://github.com/Blosc/c-blosc/archive/v1.12.1.tar.gz
  tar xzf v1.12.1.tar.gz || die "failed to uncompress blosc repo"
  cd c-blosc-1.12.1 && mkdir build && cd build
  cmake -DCMAKE_INSTALL_PREFIX='/usr' .. || die "blosc cmake failed"
  cmake --build . --target install || die "bloc build install failed"
  popd && rm -rf $TEMP
}

install_apt_pkgs() {
  apt-get -y install gcc g++ wget cmake \
    zlib1g-dev libbz2-dev liblz4-dev || die "could not install apt pkg dependencies"
}

install_yum_pkgs() {
  yum -y install epel-release &&
  yum -y install gcc gcc-c++ which wget cmake \
    zlib-devel bzip2-devel lz4-devel || die "could not install yum pkg dependencies"
}

install_brew_pkgs() {
  brew install wget cmake lzlib lz4 bzip2 zstd || die "could not install brew pkg dependencies"
  brew install tiledb-inc/stable/blosc || die "could not install blosc pkg dependency"
}

install_deps() {
  if [[ $OSTYPE == linux* ]]; then
    if [ -n "$(command -v apt-get)" ]; then
      install_apt_pkgs 
      build_install_cmake
      # zstd is in later versions of Ubuntu
      if [[ $(apt-cache search libzstd-dev) ]]; then
        apt-get -y install libzstd-dev
      else
        build_install_zstd
      fi
      build_install_blosc
    elif [ -n "$(command -v yum)" ]; then
      install_yum_pkgs
      build_install_cmake
      build_install_zstd
      build_install_blosc
    fi
  elif [[ $OSTYPE == darwin* ]]; then
    if [ -n "$(command -v brew)" ]; then
      install_brew_pkgs
    else
      die "homebrew is not installed!"
    fi
  else
    die "unsupported package management system"
  fi 
}

run() {
  install_deps
}

run 
{'fraction_non_alphanumeric': 0.13145245559038662, 'fraction_numerical': 0.022988505747126436, 'mean_word_length': 2.786392405063291, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '479552', 'n_tokens_mistral': 1891, 'n_tokens_neox': 1810, 'n_words': 500}