<commit_before>FROM ubuntu:14.04

MAINTAINER <NAME> <<EMAIL>>

RUN echo "deb http://archive.ubuntu.com/ubuntu/ $(lsb_release -sc) main universe" >> /etc/apt/sources.list

RUN apt-get update && apt-get install -y git wget tar \
 build-essential python-dev libopenblas-dev python-numpy python-scipy
RUN apt-get install -y python-pip
COPY . /app/
RUN pip install -r /app/requirements.txt
RUN pip install git+https://github.com/openeventdata/petrarch2/

WORKDIR /app
RUN wget https://s3.amazonaws.com/mordecai-geo/GoogleNews-vectors-negative300.bin.gz
RUN wget https://s3.amazonaws.com/ahalterman-textdata/wiki_ar_word2vec.model
RUN wget https://s3.amazonaws.com/ahalterman-textdata/wiki_ar_word2vec.model.syn0.npy
RUN wget https://s3.amazonaws.com/ahalterman-textdata/wiki_ar_word2vec.model.syn1neg.npy
<commit_msg>Reorganize for faster builds<commit_after>FROM ubuntu:14.04

MAINTAINER <NAME> <<EMAIL>>

RUN echo "deb http://archive.ubuntu.com/ubuntu/ $(lsb_release -sc) main universe" >> /etc/apt/sources.list

RUN apt-get update && apt-get install -y git wget tar \
 build-essential python-dev libopenblas-dev python-numpy python-scipy
RUN apt-get install -y python-pip
COPY requirements.txt /tmp/
RUN pip install -r /tmp/requirements.txt
RUN pip install git+https://github.com/openeventdata/petrarch2/

WORKDIR /app
RUN wget https://s3.amazonaws.com/mordecai-geo/GoogleNews-vectors-negative300.bin.gz
RUN wget https://s3.amazonaws.com/ahalterman-textdata/wiki_ar_word2vec.model
RUN wget https://s3.amazonaws.com/ahalterman-textdata/wiki_ar_word2vec.model.syn0.npy
RUN wget https://s3.amazonaws.com/ahalterman-textdata/wiki_ar_word2vec.model.syn1neg.npy

#RUN mkdir /home/ubuntu
#WORKDIR /home/ubuntu

#RUN git clone https://github.com/mit-nlp/MITIE.git
#WORKDIR /home/ubuntu/MITIE
#
#RUN wget http://sourceforge.net/projects/mitie/files/binaries/MITIE-models-v0.2.tar.bz2
#RUN tar --no-same-owner -xjf MITIE-models-v0.2.tar.bz2
#RUN mkdir /home/ubuntu/MITIE/mitielib/build
#RUN cd mitielib/build; cmake ..
#RUN cd mitielib/build; cmake --build . --config Release --target install
#RUN pip install git+https://github.com/caerusassociates/mitie-py.git

ADD . /app/

{'fraction_non_alphanumeric': 0.1321626617375231, 'fraction_numerical': 0.018484288354898338, 'mean_word_length': 4.3722084367245655, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 12, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5024954', 'n_tokens_mistral': 833, 'n_tokens_neox': 797, 'n_words': 161}