<commit_before># Java Current for Elasticsearch
add-apt-repository ppa:webupd8team/java
apt-get update
apt-get install oracle-java8-installer 

# Elasticsearch
wget -qO - https://packages.elasticsearch.org/GPG-KEY-elasticsearch | sudo apt-key add -
echo 'deb http://packages.elasticsearch.org/elasticsearch/1.4/debian stable main' | sudo tee /etc/apt/sources.list.d/elasticsearch.list
apt-get install elasticsearch
sudo update-rc.d elasticsearch defaults 95 10

# MongoDB
sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10
echo 'deb http://downloads-distro.mongodb.org/repo/ubuntu-upstart dist 10gen' | sudo tee /etc/apt/sources.list.d/mongodb.list
apt-get install mongodb-10gen

# More Packages
apt-get install git memcached python-virtualenv build-essential python-dev libxml2-dev libxslt1-dev ghostscript poppler-utils libpng12-dev libfreetype6-dev protobuf-compiler libprotobuf-dev

# Users and dirs
adduser --quiet ris
mkdir /srv/www
mkdir /srv/www/ris-scraper
chown ris:ris -R /srv/www/ris-scraper/
mkdir /srv/www/ris-web
chown ris:ris -R /srv/www/ris-web/

# Webinterface
sudo -i -u ris
cd /srv/www/ris-web/
git clone https://github.com/OpenRuhr/ris-web.git
cp config_dist.py config.py
vim config.py # Set Values

virtualenv --no-site-packages venv
pip install -update setuptools pip
pip install numpy
pip install -r requirements.txt

# Scraper
sudo -i -u ris
cd /srv/www/ris-web/
git clone https://github.com/OpenRuhr/ris-scraper.git<commit_msg>cCorrect Git Source<commit_after># Java Current for Elasticsearch
add-apt-repository ppa:webupd8team/java
apt-get update
apt-get install oracle-java8-installer 

# Elasticsearch
wget -qO - https://packages.elasticsearch.org/GPG-KEY-elasticsearch | sudo apt-key add -
echo 'deb http://packages.elasticsearch.org/elasticsearch/1.4/debian stable main' | sudo tee /etc/apt/sources.list.d/elasticsearch.list
apt-get install elasticsearch
sudo update-rc.d elasticsearch defaults 95 10

# MongoDB
sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10
echo 'deb http://downloads-distro.mongodb.org/repo/ubuntu-upstart dist 10gen' | sudo tee /etc/apt/sources.list.d/mongodb.list
apt-get install mongodb-10gen

# More Packages
apt-get install git memcached python-virtualenv build-essential python-dev libxml2-dev libxslt1-dev ghostscript poppler-utils libpng12-dev libfreetype6-dev protobuf-compiler libprotobuf-dev

# Users and dirs
adduser --quiet ris
mkdir /srv/www
mkdir /srv/www/ris-scraper
chown ris:ris -R /srv/www/ris-scraper/
mkdir /srv/www/ris-web
chown ris:ris -R /srv/www/ris-web/

# Webinterface
sudo -i -u ris
cd /srv/www/ris-web/
git clone https://github.com/okfde/ris-web.git
cp config_dist.py config.py
vim config.py # Set Values

virtualenv --no-site-packages venv
pip install -update setuptools pip
pip install numpy
pip install -r requirements.txt

# Scraper
sudo -i -u ris
cd /srv/www/ris-web/
git clone https://github.com/okfde/ris-scraper.git
{'fraction_non_alphanumeric': 0.1103588354773189, 'fraction_numerical': 0.015572105619498984, 'mean_word_length': 4.103626943005182, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 6, 'lorem ipsum': 0, 'www.': 12, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7074182', 'n_tokens_mistral': 1111, 'n_tokens_neox': 1031, 'n_words': 269}