<commit_before>#!/usr/bin/env bash

set -ex

sudo apt-get install -y libdouble-conversion1 libgflags2 \
    libboost-program-options1.54.0 libboost-filesystem1.54.0 \
    libboost-system1.54.0 libboost-regex1.54.0 libboost-thread1.54.0 \
    libboost-context1.54.0 libgoogle-glog0 libevent-2.0-5

sudo apt-get purge -y gcc g++ ragel autoconf \
    git libtool python-dev libssl-dev libevent-dev \
    binutils-dev make libdouble-conversion-dev libgflags-dev \
    libgoogle-glog-dev libjemalloc-dev

sudo apt-get purge -y 'libboost.*-dev'
sudo apt-get autoremove --purge -y
sudo apt-get autoclean -y
sudo apt-get clean -y

if [[ "x$1" != "x" ]]; then
    PKG_DIR=$1/pkgs
    INSTALL_DIR=$1/install
    strip "$INSTALL_DIR"/bin/mcrouter
    rm -rf "$PKG_DIR"
    rm -rf "$INSTALL_DIR"/lib/*.a
    rm -rf "$INSTALL_DIR"/include
fi<commit_msg>adding libjemalloc, to fix "/usr/local/bin/mcrouter: error while load…<commit_after>#!/usr/bin/env bash

set -ex

sudo apt-get install -y libdouble-conversion1 libgflags2 \
    libboost-program-options1.54.0 libboost-filesystem1.54.0 \
    libboost-system1.54.0 libboost-regex1.54.0 libboost-thread1.54.0 \
    libboost-context1.54.0 libgoogle-glog0 libevent-2.0-5 libjemalloc1

sudo apt-get purge -y gcc g++ ragel autoconf \
    git libtool python-dev libssl-dev libevent-dev \
    binutils-dev make libdouble-conversion-dev libgflags-dev \
    libgoogle-glog-dev libjemalloc-dev

sudo apt-get purge -y 'libboost.*-dev'
sudo apt-get autoremove --purge -y
sudo apt-get autoclean -y
sudo apt-get clean -y

if [[ "x$1" != "x" ]]; then
    PKG_DIR=$1/pkgs
    INSTALL_DIR=$1/install
    strip "$INSTALL_DIR"/bin/mcrouter
    rm -rf "$PKG_DIR"
    rm -rf "$INSTALL_DIR"/lib/*.a
    rm -rf "$INSTALL_DIR"/include
fi
{'fraction_non_alphanumeric': 0.13028571428571428, 'fraction_numerical': 0.038285714285714284, 'mean_word_length': 3.1394799054373523, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2542081', 'n_tokens_mistral': 719, 'n_tokens_neox': 667, 'n_words': 153}