<commit_before># detect platform
unamestr=`uname`
if [ "$unamestr" = 'Darwin' ]; then
    brew install leveldb wget node
elif [ "$unamestr" = 'Linux' ]; then
    apt-get -y update
    apt-get install -y zip git vim htop bzip2 curl libleveldb-dev nodejs npm
    sudo ln -s /usr/bin/nodejs /usr/bin/node
    apt-get install -y s3cmd
    ulimit -n 1000000
fi

npm install<commit_msg>Smooth out install snag<commit_after># detect platform
unamestr=`uname`
if [ "$unamestr" = 'Darwin' ]; then
    brew install leveldb wget node
elif [ "$unamestr" = 'Linux' ]; then
    apt-get -y update
    apt-get install -y zip git vim htop bzip2 curl libleveldb-dev nodejs npm
    sudo ln -s /usr/bin/nodejs /usr/bin/node
    apt-get install -y s3cmd
    ulimit -n 1000000
fi

mkdir ldb fixed
npm install

{'fraction_non_alphanumeric': 0.10165184243964422, 'fraction_numerical': 0.022871664548919948, 'mean_word_length': 3.1041666666666665, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5657240', 'n_tokens_mistral': 312, 'n_tokens_neox': 287, 'n_words': 99}