<commit_before># Install Language Pack
sudo apt-get install language-pack-en

# Install Git
sudo apt-get install git

# Install ZSH
sudo apt-get install zsh

# Install Oh-My-Zsh
sudo sh -c "$(curl -fsSL https://raw.githubusercontent.com/robbyrussell/oh-my-zsh/master/tools/install.sh)"

#Install HTOP
sudo apt-get install htop 



# Install Redis
sudo add-apt-repository ppa:chris-lea/redis-server
sudo apt-get update
sudo apt-get install redis-server

# Install Postgres
sudo apt-get install postgresql postgresql-contrib

sudo -i -u 
createdb crawler
# create user crawl_worker with password 'password';
# GRANT ALL ON DATABASE crawler to crawl_worker;


# Install NVM
curl https://raw.githubusercontent.com/creationix/nvm/v0.31.0/install.sh | bash
source .bashrc

# Install Node
nvm install 4.2.6
nvm alias default 4.2.6
nvm use 4.2.6
<commit_msg>updated master init script<commit_after>sudo apt-get update

sudo apt-get upgrade

# Install Required Packages
sudo apt-get install language-pack-en git zsh htop make g++ libpq-dev postgresql postgresql-contrib

# Install Oh-My-Zsh
sudo sh -c "$(curl -fsSL https://raw.githubusercontent.com/robbyrussell/oh-my-zsh/master/tools/install.sh)"

# Install Redis
sudo add-apt-repository ppa:chris-lea/redis-server
sudo apt-get update
sudo apt-get install redis-server
# Edit Redis conf to allow remote connections /etc/redis/redis.conf

sudo -i -u 
createdb crawler
# create database crawler;
# create user crawl_worker with password '<PASSWORD>';
# GRANT ALL ON DATABASE crawler to crawl_worker;
# Edit /var/lib/pgsql/data/pg_hba.conf to allow remote auth
# Edit /etc/postgresql/9.1/main/postgresql.conf to allow remote connections
# Reload postgresql service


# Install NVM
curl https://raw.githubusercontent.com/creationix/nvm/v0.31.0/install.sh | zsh
source .zshrc

# Install Node
nvm install 4.2.6
nvm alias default 4.2.6
nvm use 4.2.6

{'fraction_non_alphanumeric': 0.09544008483563096, 'fraction_numerical': 0.014846235418875928, 'mean_word_length': 4.184065934065934, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6114141', 'n_tokens_mistral': 699, 'n_tokens_neox': 648, 'n_words': 197}