<commit_before>FROM ubuntu:14.04
MAINTAINER <NAME> <<EMAIL>>

# RUN echo 'deb http://us.archive.ubuntu.com/ubuntu/ precise universe' >> /etc/apt/sources.list
RUN apt-get -y update
RUN apt-get -y upgrade

# Install package  that provides ADD-apt-repository
RUN apt-get -y install software-properties-common
#RUN apt-get install -y python-software-properties

# Dependencies
RUN add-apt-repository -y ppa:chris-lea/node.js
RUN apt-get -y update

RUN apt-get -y install build-essential python-dev python-pip python-virtualenv \
               libffi-dev libcairo2 python-cairo gunicorn \
	       supervisor nginx-light nodejs git wget curl

# Elasticsearch
# fake fuse
RUN  apt-get install libfuse2 &&\
     cd /tmp ; apt-get download fuse &&\
     cd /tmp ; dpkg-deb -x fuse_* . &&\
     cd /tmp ; dpkg-deb -e fuse_* &&\
     cd /tmp ; rm fuse_*.deb &&\
     cd /tmp ; echo -en '#!/bin/bash\nexit 0\n' > DEBIAN/postinst &&\
     cd /tmp ; dpkg-deb -b . /fuse.deb &&\
     cd /tmp ; dpkg -i /fuse.deb
RUN cd ~ && wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.1.1.deb
RUN cd ~ && dpkg -i elasticsearch-1.1.1.deb && rm elasticsearch-1.1.1.deb
RUN apt-get -y install openjdk-7-jre
RUN /usr/share/elasticsearch/bin/plugin -install mobz/elasticsearch-head
RUN /usr/share/elasticsearch/bin/plugin -install royrusso/elasticsearch-HQ
RUN /usr/share/elasticsearch/bin/plugin -install lmenezes/elasticsearch-kopf

# Install statsd
RUN mkdir /src && git clone https://github.com/etsy/statsd.git /src/statsd

# Install Graphite dependencies
RUN pip install 'Twisted<12.0'
RUN pip install django==1.4.9 django-tagging

# Install Whisper, Carbon and Graphite-Web
RUN pip install whisper
RUN pip install --install-option="--prefix=/var/lib/graphite" \
    --install-option="--install-lib=/var/lib/graphite/lib" \
    carbon
RUN pip install --install-option="--prefix=/var/lib/graphite" \
    --install-option="--install-lib=/var/lib/graphite/webapp" \
    graphite-web

# Install Grafana
RUN mkdir -p /src/grafana && cd /src/grafana && \
    wget http://grafanarel.s3.amazonaws.com/grafana-1.5.4.tar.gz && \
    tar xzvf grafana-1.5.4.tar.gz --strip-components=1 && rm grafana-1.5.4.tar.gz

# Install Redis
RUN apt-get install -y redis-server
RUN mkdir -p /src/redis
RUN mkdir -p /var/lib/redis

# Install Logstash
RUN mkdir -p /src/logstash && \
    cd /src/logstash && \
    wget -q https://download.elasticsearch.org/logstash/logstash/logstash-1.4.1.tar.gz && \
    tar xzf logstash-1.4.1.tar.gz --strip-components=1 && rm logstash-1.4.1.tar.gz

# Install Kibana
RUN mkdir -p /src/kibana && \
    cd /src/kibana && \
    wget -q https://download.elasticsearch.org/kibana/kibana/kibana-3.1.0.tar.gz && \
    tar xzf kibana-3.1.0.tar.gz --strip-components=1 && rm kibana-3.1.0.tar.gz


# Configuration
# -------------

# hyperion
ADD ./hyperion /src/hyperion

# Elasticsearch
ADD ./elasticsearch/run /usr/local/bin/run_elasticsearch
ADD ./elasticsearch/logging.yml /usr/share/elasticsearch/config
#RUN chown -R elasticsearch:elasticsearch /var/lib/elasticsearch
#RUN mkdir -p /var/lib/elasticsearch/elasticsearch && chown elasticsearch:elasticsearch /tmp/elasticsearch

# statsd
ADD ./statsd/config.js /src/statsd/config.js

# Whisper, Carbon and Graphite-Web
ADD ./graphite/initial_data.json /var/lib/graphite/webapp/graphite/initial_data.json
ADD ./graphite/local_settings.py /var/lib/graphite/webapp/graphite/local_settings.py
ADD ./graphite/carbon.conf /var/lib/graphite/conf/carbon.conf
ADD ./graphite/storage-schemas.conf /var/lib/graphite/conf/storage-schemas.conf
RUN mkdir -p /var/lib/graphite/storage/whisper
RUN touch /var/lib/graphite/storage/graphite.db /var/lib/graphite/storage/index
RUN chown -R www-data /var/lib/graphite/storage
RUN chmod 0775 /var/lib/graphite/storage /var/lib/graphite/storage/whisper
RUN chmod 0664 /var/lib/graphite/storage/graphite.db
RUN cd /var/lib/graphite/webapp/graphite && python manage.py syncdb --noinput

# Grafana
ADD ./grafana/config.js /src/grafana/config.js
ADD ./grafana/hyperion.json /src/grafana/app/dashboards/default.json

# Nginx
ADD ./nginx/nginx.conf /etc/nginx/nginx.conf

# Supervisord
ADD ./supervisord/supervisord.conf /etc/supervisor/conf.d/supervisord.conf

# Redis
ADD ./redis/redis.conf /src/redis/redis.conf

# Logstash
RUN mkdir /src/logstash/conf.d
ADD ./logstash/hyperion.conf /src/logstash/conf.d/hyperion.conf
ADD ./logstash/elasticsearch.conf /src/logstash/conf.d/elasticsearch.conf
ADD ./logstash/nginx.conf /src/logstash/conf.d/nginx.conf
#ADD ./logstash/indexer.conf /src/logstash/conf.d/indexer.conf
<commit_msg>Add indexer conf<commit_after>FROM ubuntu:14.04
MAINTAINER <NAME> <<EMAIL>>

# RUN echo 'deb http://us.archive.ubuntu.com/ubuntu/ precise universe' >> /etc/apt/sources.list
RUN apt-get -y update
RUN apt-get -y upgrade

# Install package  that provides ADD-apt-repository
RUN apt-get -y install software-properties-common
#RUN apt-get install -y python-software-properties

# Dependencies
RUN add-apt-repository -y ppa:chris-lea/node.js
RUN apt-get -y update

RUN apt-get -y install build-essential python-dev python-pip python-virtualenv \
               libffi-dev libcairo2 python-cairo gunicorn \
	       supervisor nginx-light nodejs git wget curl

# Elasticsearch
# fake fuse
RUN  apt-get install libfuse2 &&\
     cd /tmp ; apt-get download fuse &&\
     cd /tmp ; dpkg-deb -x fuse_* . &&\
     cd /tmp ; dpkg-deb -e fuse_* &&\
     cd /tmp ; rm fuse_*.deb &&\
     cd /tmp ; echo -en '#!/bin/bash\nexit 0\n' > DEBIAN/postinst &&\
     cd /tmp ; dpkg-deb -b . /fuse.deb &&\
     cd /tmp ; dpkg -i /fuse.deb
RUN cd ~ && wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.1.1.deb
RUN cd ~ && dpkg -i elasticsearch-1.1.1.deb && rm elasticsearch-1.1.1.deb
RUN apt-get -y install openjdk-7-jre
RUN /usr/share/elasticsearch/bin/plugin -install mobz/elasticsearch-head
RUN /usr/share/elasticsearch/bin/plugin -install royrusso/elasticsearch-HQ
RUN /usr/share/elasticsearch/bin/plugin -install lmenezes/elasticsearch-kopf

# Install statsd
RUN mkdir /src && git clone https://github.com/etsy/statsd.git /src/statsd

# Install Graphite dependencies
RUN pip install 'Twisted<12.0'
RUN pip install django==1.4.9 django-tagging

# Install Whisper, Carbon and Graphite-Web
RUN pip install whisper
RUN pip install --install-option="--prefix=/var/lib/graphite" \
    --install-option="--install-lib=/var/lib/graphite/lib" \
    carbon
RUN pip install --install-option="--prefix=/var/lib/graphite" \
    --install-option="--install-lib=/var/lib/graphite/webapp" \
    graphite-web

# Install Grafana
RUN mkdir -p /src/grafana && cd /src/grafana && \
    wget http://grafanarel.s3.amazonaws.com/grafana-1.5.4.tar.gz && \
    tar xzvf grafana-1.5.4.tar.gz --strip-components=1 && rm grafana-1.5.4.tar.gz

# Install Redis
RUN apt-get install -y redis-server
RUN mkdir -p /src/redis
RUN mkdir -p /var/lib/redis

# Install Logstash
RUN mkdir -p /src/logstash && \
    cd /src/logstash && \
    wget -q https://download.elasticsearch.org/logstash/logstash/logstash-1.4.1.tar.gz && \
    tar xzf logstash-1.4.1.tar.gz --strip-components=1 && rm logstash-1.4.1.tar.gz

# Install Kibana
RUN mkdir -p /src/kibana && \
    cd /src/kibana && \
    wget -q https://download.elasticsearch.org/kibana/kibana/kibana-3.1.0.tar.gz && \
    tar xzf kibana-3.1.0.tar.gz --strip-components=1 && rm kibana-3.1.0.tar.gz


# Configuration
# -------------

# hyperion
ADD ./hyperion /src/hyperion

# Elasticsearch
ADD ./elasticsearch/run /usr/local/bin/run_elasticsearch
ADD ./elasticsearch/logging.yml /usr/share/elasticsearch/config
#RUN chown -R elasticsearch:elasticsearch /var/lib/elasticsearch
#RUN mkdir -p /var/lib/elasticsearch/elasticsearch && chown elasticsearch:elasticsearch /tmp/elasticsearch

# statsd
ADD ./statsd/config.js /src/statsd/config.js

# Whisper, Carbon and Graphite-Web
ADD ./graphite/initial_data.json /var/lib/graphite/webapp/graphite/initial_data.json
ADD ./graphite/local_settings.py /var/lib/graphite/webapp/graphite/local_settings.py
ADD ./graphite/carbon.conf /var/lib/graphite/conf/carbon.conf
ADD ./graphite/storage-schemas.conf /var/lib/graphite/conf/storage-schemas.conf
RUN mkdir -p /var/lib/graphite/storage/whisper
RUN touch /var/lib/graphite/storage/graphite.db /var/lib/graphite/storage/index
RUN chown -R www-data /var/lib/graphite/storage
RUN chmod 0775 /var/lib/graphite/storage /var/lib/graphite/storage/whisper
RUN chmod 0664 /var/lib/graphite/storage/graphite.db
RUN cd /var/lib/graphite/webapp/graphite && python manage.py syncdb --noinput

# Grafana
ADD ./grafana/config.js /src/grafana/config.js
ADD ./grafana/hyperion.json /src/grafana/app/dashboards/default.json

# Nginx
ADD ./nginx/nginx.conf /etc/nginx/nginx.conf

# Supervisord
ADD ./supervisord/supervisord.conf /etc/supervisor/conf.d/supervisord.conf

# Redis
ADD ./redis/redis.conf /src/redis/redis.conf

# Logstash
RUN mkdir /src/logstash/conf.d
ADD ./logstash/hyperion.conf /src/logstash/conf.d/hyperion.conf
ADD ./logstash/elasticsearch.conf /src/logstash/conf.d/elasticsearch.conf
ADD ./logstash/nginx.conf /src/logstash/conf.d/nginx.conf
ADD ./logstash/indexer.conf /src/logstash/conf.d/indexer.conf

{'fraction_non_alphanumeric': 0.14423493714781102, 'fraction_numerical': 0.013437364542696142, 'mean_word_length': 3.2746641963872163, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 15, 'https://': 8, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6138166', 'n_tokens_mistral': 3457, 'n_tokens_neox': 3398, 'n_words': 698}