<commit_before>FROM centos:6
MAINTAINER <NAME> <<EMAIL>>

USER root

# install dev tools and java
RUN yum install -y \
    java-1.8.0-openjdk* \
    krb5-workstation \
    tar \
    wget \
    which

# download and extract hadoop
RUN wget -O - \
    http://archive.apache.org/dist/hadoop/common/hadoop-2.7.0/hadoop-2.7.0.tar.gz \
    | tar -xzpo --strip-components 1 -C /

# create hadoop groups and users
RUN groupadd hadoop
RUN groupadd supergroup
RUN useradd -g supergroup -G hadoop hdfs
RUN useradd -g supergroup -G hadoop yarn

# set java home, path, and hadoop env variables
ENV JAVA_HOME /usr/lib/jvm/jre-1.8.0
ENV PATH $JAVA_HOME/bin:$PATH
ENV HADOOP_CONF_DIR /etc/hadoop
ENV HADOOP_LIBEXEC_DIR /libexec

RUN su hdfs -c "hdfs namenode -format"<commit_msg>Remove hadoop:2.7.0 dependency on which<commit_after>FROM centos:6
MAINTAINER <NAME> <<EMAIL>>

USER root

# install dev tools and java
RUN yum install -y \
    java-1.8.0-openjdk* \
    krb5-workstation \
    tar \
    wget

# download and extract hadoop
RUN wget -O - \
    http://archive.apache.org/dist/hadoop/common/hadoop-2.7.0/hadoop-2.7.0.tar.gz \
    | tar -xzpo --strip-components 1 -C /

# create hadoop groups and users
RUN groupadd hadoop
RUN groupadd supergroup
RUN useradd -g supergroup -G hadoop hdfs
RUN useradd -g supergroup -G hadoop yarn

# set java home, path, and hadoop env variables
ENV JAVA_HOME /usr/lib/jvm/jre-1.8.0
ENV PATH $JAVA_HOME/bin:$PATH
ENV HADOOP_CONF_DIR /etc/hadoop
ENV HADOOP_LIBEXEC_DIR /libexec

RUN su hdfs -c "hdfs namenode -format"
{'fraction_non_alphanumeric': 0.1038961038961039, 'fraction_numerical': 0.02142857142857143, 'mean_word_length': 3.098404255319149, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '320078', 'n_tokens_mistral': 604, 'n_tokens_neox': 583, 'n_words': 174}