<commit_before>FROM ubuntu:16.04

LABEL spark.version="{{ SPARK_VERSION }}" \
      hadoop.version="{{ HADOOP_FULL_VERSION }}" \
      scala.version="{{ SCALA_VERSION }}" \
      python.version="{{ PYTHON_VERSION }}" \
      python.hashseed="{{ PYTHON_HASHSEED }}"

ENV JAVA_HOME /usr/lib/jvm/java-{{ JDK_VERSION }}-openjdk-amd64

ENV CURL_CA_BUNDLE /etc/ssl/certs/ca-certificates.crt

ENV SPARK_HOME /home/{{ USERNAME }}/spark-{{ SPARK_VERSION }}

ENV PYSPARK_PYTHON /home/{{ USERNAME }}/anaconda{{ PYTHON_VERSION }}/bin/python
ENV PYTHON_HASHSEED {{ PYTHON_HASHSEED }}

# See https://github.com/phusion/baseimage-docker/issues/58
RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections

RUN apt-get update \
    && apt-get install -y wget \
    && apt-get install -y --no-install-recommends openjdk-{{ JDK_VERSION }}-jdk-headless \
    && apt-get install -y libatlas3-base libopenblas-base \
    && apt-get install -y bzip2 \
    && apt-get clean \
    && rm -rf /var/lib/apt/lists/*ce

RUN groupadd -r {{ USERNAME }} && useradd -r -g {{ USERNAME }} {{ USERNAME }} \
    && mkdir /home/{{ USERNAME }} \
    && chown -R {{ USERNAME }}:{{ USERNAME }} /home/{{ USERNAME }}

USER {{ USERNAME }}

WORKDIR /home/{{ USERNAME }}

ENV PATH $PATH:/home/{{ USERNAME }}/anaconda{{ PYTHON_VERSION }}/bin

RUN wget {{ ANACONDA_URL }}/{{ ANACONDA_INSTALLER }} \
    && bash {{ ANACONDA_INSTALLER }} -b -p /home/{{ USERNAME }}/anaconda{{ PYTHON_VERSION }} \
    && rm {{ ANACONDA_INSTALLER }} \
    && conda install -y curl {{ PYTHON_PACKAGES }}

{% if WITH_R %}
RUN conda install -y -c r r
{% endif %}

{% if HADOOP_PROVIDED %}
RUN wget {{ HADOOP_DIST_URL }}/hadoop-{{ HADOOP_FULL_VERSION }}/hadoop-{{ HADOOP_FULL_VERSION }}.tar.gz -O - | tar -xz

ENV HADOOP_HOME /home/{{ USERNAME }}/hadoop-{{ HADOOP_FULL_VERSION }}
ENV LD_LIBRARY_PATH $LD_LIBRARY_PATH:$HADOOP_HOME/lib/native
ENV PATH $PATH:$HADOOP_HOME/bin

RUN echo "export SPARK_DIST_CLASSPATH=$(hadoop classpath)" > .bashrc
{% endif %}

RUN wget {{ SPARK_DIST_URL }}/spark-{{ SPARK_VERSION }}/spark-{{ SPARK_VERSION }}.tgz -O - | tar -xz \
    && cd $SPARK_HOME \
    && dev/change-scala-version.sh {{ SCALA_VERSION }} \
    && build/mvn -Dscala-{{ SCALA_VERSION }} \
                 -Pnetlib-lgpl -DskipTests {{ MVN_PARAMS }} \
                 clean package

{% if MVN_ARTIFACTS %}
{% for ARTIFACT in MVN_ARTIFACTS %}

{% if loop.first %}RUN {% else %}    {% endif %}mvn dependency:get -Dartifact={{ ARTIFACT }} {% if not loop.last %}\{% endif %}

{% endfor %}
{% endif %}<commit_msg>Fix mvn path<commit_after>FROM ubuntu:16.04

LABEL spark.version="{{ SPARK_VERSION }}" \
      hadoop.version="{{ HADOOP_FULL_VERSION }}" \
      scala.version="{{ SCALA_VERSION }}" \
      python.version="{{ PYTHON_VERSION }}" \
      python.hashseed="{{ PYTHON_HASHSEED }}"

ENV JAVA_HOME /usr/lib/jvm/java-{{ JDK_VERSION }}-openjdk-amd64

ENV CURL_CA_BUNDLE /etc/ssl/certs/ca-certificates.crt

ENV SPARK_HOME /home/{{ USERNAME }}/spark-{{ SPARK_VERSION }}

ENV PYSPARK_PYTHON /home/{{ USERNAME }}/anaconda{{ PYTHON_VERSION }}/bin/python
ENV PYTHON_HASHSEED {{ PYTHON_HASHSEED }}

# See https://github.com/phusion/baseimage-docker/issues/58
RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections

RUN apt-get update \
    && apt-get install -y wget \
    && apt-get install -y --no-install-recommends openjdk-{{ JDK_VERSION }}-jdk-headless \
    && apt-get install -y libatlas3-base libopenblas-base \
    && apt-get install -y bzip2 \
    && apt-get clean \
    && rm -rf /var/lib/apt/lists/*ce

RUN groupadd -r {{ USERNAME }} && useradd -r -g {{ USERNAME }} {{ USERNAME }} \
    && mkdir /home/{{ USERNAME }} \
    && chown -R {{ USERNAME }}:{{ USERNAME }} /home/{{ USERNAME }}

USER {{ USERNAME }}

WORKDIR /home/{{ USERNAME }}

ENV PATH $PATH:/home/{{ USERNAME }}/anaconda{{ PYTHON_VERSION }}/bin

RUN wget {{ ANACONDA_URL }}/{{ ANACONDA_INSTALLER }} \
    && bash {{ ANACONDA_INSTALLER }} -b -p /home/{{ USERNAME }}/anaconda{{ PYTHON_VERSION }} \
    && rm {{ ANACONDA_INSTALLER }} \
    && conda install -y curl {{ PYTHON_PACKAGES }}

{% if WITH_R %}
RUN conda install -y -c r r
{% endif %}

{% if HADOOP_PROVIDED %}
RUN wget {{ HADOOP_DIST_URL }}/hadoop-{{ HADOOP_FULL_VERSION }}/hadoop-{{ HADOOP_FULL_VERSION }}.tar.gz -O - | tar -xz

ENV HADOOP_HOME /home/{{ USERNAME }}/hadoop-{{ HADOOP_FULL_VERSION }}
ENV LD_LIBRARY_PATH $LD_LIBRARY_PATH:$HADOOP_HOME/lib/native
ENV PATH $PATH:$HADOOP_HOME/bin

RUN echo "export SPARK_DIST_CLASSPATH=$(hadoop classpath)" > .bashrc
{% endif %}

RUN wget {{ SPARK_DIST_URL }}/spark-{{ SPARK_VERSION }}/spark-{{ SPARK_VERSION }}.tgz -O - | tar -xz \
    && cd $SPARK_HOME \
    && dev/change-scala-version.sh {{ SCALA_VERSION }} \
    && build/mvn -Dscala-{{ SCALA_VERSION }} \
                 -Pnetlib-lgpl -DskipTests {{ MVN_PARAMS }} \
                 clean package

{% if MVN_ARTIFACTS %}
{% for ARTIFACT in MVN_ARTIFACTS %}

{% if loop.first %}RUN {% else %}    {% endif %}$SPARK_HOME/build/mvn dependency:get -Dartifact={{ ARTIFACT }} {% if not loop.last %}\{% endif %}

{% endfor %}
{% endif %}
{'fraction_non_alphanumeric': 0.17280342879407753, 'fraction_numerical': 0.0038963569062926166, 'mean_word_length': 3.281901584653878, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6607386', 'n_tokens_mistral': 2126, 'n_tokens_neox': 1953, 'n_words': 420}