<commit_before>#!/bin/bash

# Executed as root

apt-get install -y openjdk-7-jre-headless

kafka_url='http://www.eng.lsu.edu/mirrors/apache/kafka/0.8.1.1/kafka_2.10-0.8.1.1.tgz'
kafka='kafka_2.10-0.8.1.1'
kafka_bin="$kafka.tgz"

if [ ! -d "/opt/$kafka" ]; then
  mkdir -p /vagrant/files
  cd /vagrant/files
  if [ ! -f "/vagrant/files/$kafka_bin" ]; then
    wget $kafka_url
  fi
  tar -C /opt -xzf $kafka_bin
fi<commit_msg>Packages versions gets stalled so apt-get update needs to be called before doing any install<commit_after>#!/bin/bash

# Executed as root

ap-get update
apt-get install -y openjdk-7-jre-headless

kafka_url='http://www.eng.lsu.edu/mirrors/apache/kafka/0.8.1.1/kafka_2.10-0.8.1.1.tgz'
kafka='kafka_2.10-0.8.1.1'
kafka_bin="$kafka.tgz"

if [ ! -d "/opt/$kafka" ]; then
  mkdir -p /vagrant/files
  cd /vagrant/files
  if [ ! -f "/vagrant/files/$kafka_bin" ]; then
    wget $kafka_url
  fi
  tar -C /opt -xzf $kafka_bin
fi
{'fraction_non_alphanumeric': 0.17215727948990436, 'fraction_numerical': 0.040382571732199786, 'mean_word_length': 3.149779735682819, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 6, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6166111', 'n_tokens_mistral': 450, 'n_tokens_neox': 432, 'n_words': 81}