<commit_before>FROM debian:stretch
MAINTAINER <<EMAIL>>

RUN apt-get update && apt-get upgrade -y
RUN apt-get install -y openjdk-11-jdk curl procps
RUN curl -s https://archive.apache.org/dist/spark/spark-2.4.0/spark-2.4.0-bin-hadoop2.7.tgz | tar -xz -C /tmp
RUN ln -s /tmp /tmp/spark-2.4.0-bin-hadoop2.7/logs

WORKDIR /tmp/spark-2.4.0-bin-hadoop2.7

ENV SPARK_MASTER_HOST 127.0.0.1

USER root
EXPOSE 7077

CMD /tmp/spark-2.4.0-bin-hadoop2.7/sbin/start-master.sh -h 0.0.0.0; \
    /tmp/spark-2.4.0-bin-hadoop2.7/sbin/start-slave.sh -c 1 -m 16G spark://`hostname`:7077; \
    while true; do sleep 3600; done <commit_msg>Use openjdk image. And less noisy spark.<commit_after>FROM openjdk:11-jdk
MAINTAINER <<EMAIL>>

RUN apt-get update && apt-get upgrade -y
RUN apt-get install -y curl procps
RUN curl -s https://archive.apache.org/dist/spark/spark-2.4.0/spark-2.4.0-bin-hadoop2.7.tgz | tar -xz -C /tmp
RUN cp -pi /tmp/spark-2.4.0-bin-hadoop2.7/conf/log4j.properties.template /tmp/spark-2.4.0-bin-hadoop2.7/conf/log4j.properties
RUN sed -e "s/log4j.rootCategory=INFO/log4j.rootCategory=WARN/" -i /tmp/spark-2.4.0-bin-hadoop2.7/conf/log4j.properties
RUN ln -s /tmp /tmp/spark-2.4.0-bin-hadoop2.7/logs

WORKDIR /tmp/spark-2.4.0-bin-hadoop2.7

ENV SPARK_MASTER_HOST 127.0.0.1

USER root
EXPOSE 7077

CMD /tmp/spark-2.4.0-bin-hadoop2.7/sbin/start-master.sh -h 0.0.0.0; \
    /tmp/spark-2.4.0-bin-hadoop2.7/sbin/start-slave.sh -c 1 -m 16G spark://`hostname`:7077; \
    while true; do sleep 3600; done 
{'fraction_non_alphanumeric': 0.17202141900937082, 'fraction_numerical': 0.08701472556894244, 'mean_word_length': 2.9973262032085564, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3686335', 'n_tokens_mistral': 722, 'n_tokens_neox': 660, 'n_words': 126}