<commit_before>FROM ubuntu:15.10
MAINTAINER <NAME> <<EMAIL>>

# Set tool versions
ENV CSVFIX_VERSION   1.6
ENV CVSKIT_VERSION   0.9.1
ENV GDAL_VERSION     1.11.2+dfsg-3ubuntu3
ENV GNUPLOT_VERSION  4.6.6-2
ENV HTTPIE_VERSION   0.9.2-0.1
ENV JQ_VERSION       1.5
ENV UCHARDET_VERSION 0.0.5

# Set TAAL variables
ENV HOME /taal

# Set environment variables
ENV DEBIAN_FRONTEND noninteractive
ENV LANG en_US.UTF-8
ENV LANGUAGE en_US.UTF-8:en.UTF-8
ENV LC_ALL en_US.UTF-8

# Install system packages as well as GDAL and gnuplot
RUN apt-get update                    \
    && apt-get install -y             \
        curl                          \
        gdal-bin=$GDAL_VERSION        \
        gnuplot                       \
        httpie=$HTTPIE_VERSION        \
        locales                       \
        p7zip                         \
        python-pip                    \
        sudo                          \
        unzip                         \
        vim                           \
        wget                          \
    && apt-get autoremove -y          \
    && apt-get clean                  \
    && rm -rf /var/lib/apt/lists/*

# Configure locales
RUN locale-gen en_US.UTF-8 \
    && dpkg-reconfigure locales

# Configure time
RUN echo "Etc/UTC" > /etc/timezone \
    && dpkg-reconfigure tzdata

# Configure sudo
RUN sed --in-place --expression="s/\%sudo\t\ALL=(ALL:ALL) ALL/\%sudo\tALL=(ALL) NOPASSWD:ALL/" /etc/sudoers

# Create taal user
RUN useradd --groups sudo --create-home --home-dir=$HOME --shell /bin/bash taal  \
    && echo "taal:taal" | chpasswd                                               \
    && touch $HOME/.sudo_as_admin_successful

# Configure Bash
COPY ./files/.bashrc /root/.bashrc
COPY ./files/.bashrc $HOME/.bashrc
RUN chown -R taal:taal $HOME

# Create placeholder folders
RUN mkdir --mode=777 $HOME/data         \
    && chown -R taal:taal $HOME/data    \
    && mkdir --mode=777 $HOME/scripts   \
    && chown -R taal:taal $HOME/scripts

# Install csvfix
RUN apt-get update                                             \
    && apt-get install -y g++ make mercurial                   \
    && hg clone https://bitbucket.org/neilb/csvfix /tmp/csvfix \
    && (cd /tmp/csvfix && hg up "version-$CSVFIX_VERSION")     \
    && make --directory=/tmp/csvfix lin                        \
    && cp /tmp/csvfix/csvfix/bin/csvfix /usr/local/bin         \
    && rm -rf /tmp/csvfix                                      \
    && apt-get remove -y g++ make mercurial                    \
    && apt-get autoremove -y                                   \
    && apt-get clean                                           \
    && rm -rf /var/lib/apt/lists/*

# Install csvkit
RUN pip install csvkit==$CVSKIT_VERSION \
    && rm -rf /tmp/pip_build_root

# Install jq and uchardet
RUN apt-get update                                                                                                                 \
    && apt-get install -y build-essential cmake                                                                                    \
    && wget --quiet --output-document=/usr/local/bin/jq https://github.com/stedolan/jq/releases/download/jq-$JQ_VERSION/jq-linux64 \
    && chmod +x /usr/local/bin/jq                                                                                                  \
    && wget --quiet -O - https://github.com/BYVoid/uchardet/archive/v$UCHARDET_VERSION.tar.gz | tar -xz -C /tmp/                   \
    && (cd /tmp/uchardet-$UCHARDET_VERSION && cmake . && make && make install)                                                     \
    && rm -rf /tmp/uchardet-$UCHARDET_VERSION                                                                                      \
    && apt-get remove -y build-essential cmake                                                                                     \
    && apt-get autoremove -y                                                                                                       \
    && apt-get clean                                                                                                               \
    && rm -rf /var/lib/apt/lists/*

# Run Bash
USER taal
WORKDIR /taal
ENTRYPOINT ["bash"]<commit_msg>Simplify Dockerfile<commit_after>FROM ubuntu:15.10
MAINTAINER Emplacement Idéal Labs <<EMAIL>>

# Set tool versions
ENV CSVFIX_VERSION   1.6
ENV CVSKIT_VERSION   0.9.1
ENV GDAL_VERSION     1.11.2+dfsg-3ubuntu3
ENV GNUPLOT_VERSION  4.6.6-2
ENV HTTPIE_VERSION   0.9.2-0.1
ENV JQ_VERSION       1.5
ENV UCHARDET_VERSION 0.0.5

# Set TAAL variables
ENV HOME /taal

# Set environment variables
ENV DEBIAN_FRONTEND noninteractive
ENV LANG en_US.UTF-8
ENV LANGUAGE en_US.UTF-8:en.UTF-8
ENV LC_ALL en_US.UTF-8

# Install system packages as well as GDAL and gnuplot
RUN apt-get update                    \
    && apt-get install -y             \
        build-essential               \
        cmake                         \
        curl                          \
        gdal-bin=$GDAL_VERSION        \
        gnuplot                       \
        httpie=$HTTPIE_VERSION        \
        libmysqlclient-dev            \
        libpq-dev                     \
        locales                       \
        mercurial                     \
        p7zip                         \
        python-dev                    \
        python-pip                    \
        sudo                          \
        unzip                         \
        vim                           \
        wget                          \
    && apt-get autoremove -y          \
    && apt-get clean                  \
    && rm -rf /var/lib/apt/lists/*

# Configure locales
RUN locale-gen en_US.UTF-8 \
    && dpkg-reconfigure locales

# Configure time
RUN echo "Etc/UTC" > /etc/timezone \
    && dpkg-reconfigure tzdata

# Configure sudo
RUN sed --in-place --expression="s/\%sudo\t\ALL=(ALL:ALL) ALL/\%sudo\tALL=(ALL) NOPASSWD:ALL/" /etc/sudoers

# Create taal user
RUN useradd --groups sudo --create-home --home-dir=$HOME --shell /bin/bash taal  \
    && echo "taal:taal" | chpasswd                                               \
    && touch $HOME/.sudo_as_admin_successful

# Configure Bash
COPY ./files/.bashrc /root/.bashrc
COPY ./files/.bashrc $HOME/.bashrc
RUN chown -R taal:taal $HOME

# Create placeholder folders
RUN mkdir --mode=777 $HOME/data         \
    && chown -R taal:taal $HOME/data    \
    && mkdir --mode=777 $HOME/scripts   \
    && chown -R taal:taal $HOME/scripts

# Install csvfix
RUN hg clone https://bitbucket.org/neilb/csvfix /tmp/csvfix \
    && (cd /tmp/csvfix && hg up "version-$CSVFIX_VERSION")  \
    && make --directory=/tmp/csvfix lin                     \
    && cp /tmp/csvfix/csvfix/bin/csvfix /usr/local/bin      \
    && rm -rf /tmp/csvfix

# Install csvkit
RUN pip install csvkit==$CVSKIT_VERSION \
    && rm -rf /tmp/pip_build_root

# Install jq and uchardet
RUN wget --quiet --output-document=/usr/local/bin/jq https://github.com/stedolan/jq/releases/download/jq-$JQ_VERSION/jq-linux64 \
    && chmod +x /usr/local/bin/jq                                                                                               \
    && wget --quiet -O - https://github.com/BYVoid/uchardet/archive/v$UCHARDET_VERSION.tar.gz | tar -xz -C /tmp/                \
    && (cd /tmp/uchardet-$UCHARDET_VERSION && cmake . && make && make install)                                                  \
    && rm -rf /tmp/uchardet-$UCHARDET_VERSION

# Run Bash
USER taal
WORKDIR /taal
ENTRYPOINT ["bash"]
{'fraction_non_alphanumeric': 0.10272703203600742, 'fraction_numerical': 0.011384696849351337, 'mean_word_length': 1.1044568245125348, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 10, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 48, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3538247', 'n_tokens_mistral': 2301, 'n_tokens_neox': 2157, 'n_words': 496}