<commit_before>DOWNLOAD_DIR=${HOME}/downloads

setup_elasticsearch() {
    es_version=0.90.13
    es_dir=${DOWNLOAD_DIR}/elasticsearch-${es_version}

    echo "Installing elasticsearch version ${es_version}"

    if [ ! -d ${es_dir} ]; then
        wget "https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-${es_version}.tar.gz" -O ${DOWNLOAD_DIR}/elasticsearch.tar.gz

        tar xvzf ${DOWNLOAD_DIR}/elasticsearch.tar.gz
    fi

    nohup bash -c "cd ${es_dir} && bin/elasticsearch &"
    sleep 5
}

setup_kafka() {
    kafka_dir=${DOWNLOAD_DIR}/kafka
    # kafka install, copied from https://github.com/wvanbergen/kafka/blob/master/.travis.yml
    if [ ! -d ${kafka_dir} ]; then
        wget http://www.us.apache.org/dist/kafka/0.8.2.1/kafka_2.10-0.8.2.1.tgz -O ${DOWNLOAD_DIR}/kafka.tgz
        mkdir -p ${kafka_dir} && tar xzf ${DOWNLOAD_DIR}/kafka.tgz -C ${kafka_dir} --strip-components 1<commit_msg>tar into DOWNLOAD_DIR<commit_after>DOWNLOAD_DIR=${HOME}/downloads

setup_elasticsearch() {
    es_version=0.90.13
    es_dir=${DOWNLOAD_DIR}/elasticsearch-${es_version}

    echo "Installing elasticsearch version ${es_version}"

    if [ ! -d ${es_dir} ]; then
        wget "https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-${es_version}.tar.gz" -O ${DOWNLOAD_DIR}/elasticsearch.tar.gz

        tar xvzf ${DOWNLOAD_DIR}/elasticsearch.tar.gz -C ${DOWNLOAD_DIR}
    fi

    nohup bash -c "cd ${es_dir} && bin/elasticsearch &"
    sleep 5
}

setup_kafka() {
    kafka_dir=${DOWNLOAD_DIR}/kafka
    # kafka install, copied from https://github.com/wvanbergen/kafka/blob/master/.travis.yml
    if [ ! -d ${kafka_dir} ]; then
        wget http://www.us.apache.org/dist/kafka/0.8.2.1/kafka_2.10-0.8.2.1.tgz -O ${DOWNLOAD_DIR}/kafka.tgz
        mkdir -p ${kafka_dir} && tar xzf ${DOWNLOAD_DIR}/kafka.tgz -C ${kafka_dir} --strip-components 1
{'fraction_non_alphanumeric': 0.15982951518380395, 'fraction_numerical': 0.019179541822056473, 'mean_word_length': 3.4292452830188678, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2434198', 'n_tokens_mistral': 711, 'n_tokens_neox': 695, 'n_words': 111}