<commit_before>#!/bin/bash

SCRIPT_NAME=$(basename $BASH_SOURCE)
SCRIPT_LOGFILE="./logs/"$(basename -s .sh $BASH_SOURCE)".log"
mkdir -p ./logs && chmod 755 ./logs

echo "running "$SCRIPT_NAME

# tested: works on ubuntu 14.04.3 gnome


# todo: find better check
if hash beanstalkd 2>/dev/null; then
	echo "  beanstalk server already installed"
else
	echo "  beanstalk server not installed"

	sudo apt-get -y install beanstalkd &>> $SCRIPT_LOGFILE

	wget https://github.com/tyba/beanstool/releases/download/v0.1.0/beanstool_v0.1.0_linux_amd64.tar.gz
	tar -xvzf beanstool_v0.1.0_linux_amd64.tar.gz
	cp beanstool_v0.1.0_linux_amd64/beanstool /usr/local/bin/

fi


# todo: find better check
if hash beanstool 2>/dev/null; then
	echo "  beanstool already installed"
else
	echo "  beanstool not installed"

	mkdir -p ~/tmp/beanstool
	cd ~/tmp/beanstool

	wget https://github.com/tyba/beanstool/releases/download/v0.1.0/beanstool_v0.1.0_linux_amd64.tar.gz
	tar -xvzf beanstool_v0.1.0_linux_amd64.tar.gz
	sudo cp beanstool_v0.1.0_linux_amd64/beanstool /usr/local/bin/

	rm -rf ~/tmp/beanstool

fi
<commit_msg>updated beantools to 0.2, checked ubuntu 16.04 compatibility<commit_after>#!/bin/bash

# tested on ubuntu 16.04 (gnome)
# tested on ubuntu 14.04.3 (gnome)

SCRIPT_NAME=$(basename $BASH_SOURCE)
SCRIPT_LOGFILE="./logs/"$(basename -s .sh $BASH_SOURCE)".log"
mkdir -p ./logs && chmod 755 ./logs

echo "running "$SCRIPT_NAME


# todo: find better check
if hash beanstalkd 2>/dev/null; then
	echo "  beanstalk server already installed"
else
	echo "  beanstalk server not installed"

	sudo apt-get -y install beanstalkd &>> $SCRIPT_LOGFILE


	wget https://github.com/src-d/beanstool/releases/download/v0.2.0/beanstool_v0.2.0_linux_amd64.tar.gz
	tar -xvzf beanstool_v0.2.0_linux_amd64.tar.gz
	sudo cp beanstool_v0.2.0_linux_amd64/beanstool /usr/local/bin/


fi


# todo: find better check
if hash beanstool 2>/dev/null; then
	echo "  beanstool already installed"
else
	echo "  beanstool not installed"

	mkdir -p ~/tmp/beanstool
	cd ~/tmp/beanstool


	wget https://github.com/src-d/beanstool/releases/download/v0.2.0/beanstool_v0.2.0_linux_amd64.tar.gz
	tar -xvzf beanstool_v0.2.0_linux_amd64.tar.gz
	sudo cp beanstool_v0.2.0_linux_amd64/beanstool /usr/local/bin/

	rm -rf ~/tmp/beanstool

fi

{'fraction_non_alphanumeric': 0.12166301969365427, 'fraction_numerical': 0.04463894967177243, 'mean_word_length': 4.0131578947368425, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7017240', 'n_tokens_mistral': 1026, 'n_tokens_neox': 930, 'n_words': 198}