<commit_before>#
# NYT-based news tagger service
#

FROM dockermediacloud/base:latest

RUN \
    #
    # Install model fetch dependencies
    apt-get -y --no-install-recommends install brotli && \
    #
    # Create directory for annotator
    mkdir -p /usr/src/crappy-predict-news-labels/models/ && \
    #
    true

# Download and extract models
# (get them first so that every code change doesn't trigger huge model redownload)
WORKDIR /usr/src/crappy-predict-news-labels/models/
ENV MODEL_URL="https://mediacloud-nytlabels-data.s3.amazonaws.com/predict-news-labels-onnx"

RUN /dl_to_stdout.sh "$MODEL_URL/GoogleNews-vectors-negative300.shelve.br" | \
        brotli -d > GoogleNews-vectors-negative300.shelve

RUN /dl_to_stdout.sh "$MODEL_URL/scaler.onnx" > scaler.onnx

RUN /dl_to_stdout.sh "$MODEL_URL/all_descriptors.onnx.br" | \
        brotli -d > all_descriptors.onnx
RUN /dl_to_stdout.sh "$MODEL_URL/all_descriptors.txt.br" | \
        brotli -d > all_descriptors.txt

RUN /dl_to_stdout.sh "$MODEL_URL/descriptors_3000.onnx.br" | \
        brotli -d > descriptors_3000.onnx
RUN /dl_to_stdout.sh "$MODEL_URL/descriptors_3000.txt.br" | \
        brotli -d > descriptors_3000.txt

RUN /dl_to_stdout.sh "$MODEL_URL/descriptors_600.onnx.br" | \
        brotli -d > descriptors_600.onnx
RUN /dl_to_stdout.sh "$MODEL_URL/descriptors_600.txt.br" | \
        brotli -d > descriptors_600.txt

RUN /dl_to_stdout.sh "$MODEL_URL/descriptors_with_taxonomies.onnx.br" | \
        brotli -d > descriptors_with_taxonomies.onnx
RUN /dl_to_stdout.sh "$MODEL_URL/descriptors_with_taxonomies.txt.br" | \
        brotli -d > descriptors_with_taxonomies.txt

RUN /dl_to_stdout.sh "$MODEL_URL/just_taxonomies.onnx.br" | \
        brotli -d > just_taxonomies.onnx
RUN /dl_to_stdout.sh "$MODEL_URL/just_taxonomies.txt.br" | \
        brotli -d > just_taxonomies.txt

# Install Python
RUN \
    apt-get -y --no-install-recommends install \
        python3 \
        python3-dev \
        python3-pip \
        #
        # Needed by "shelve" module:
        python3-gdbm \
    && \
    true

# Install requirements
# (do this first so that minor changes in the annotator's code don't trigger a
# full module reinstall)
WORKDIR /usr/src/crappy-predict-news-labels/
COPY src/crappy-predict-news-labels/requirements.txt /usr/src/crappy-predict-news-labels/
RUN \
    pip3 install -r requirements.txt && \
    rm -rf /root/.cache/ && \
    true

# Install NLTK data
RUN \
    python3 -m nltk.downloader -d /usr/local/share/nltk_data punkt && \
    rm /usr/local/share/nltk_data/tokenizers/punkt.zip && \
    true

# Copy the rest of the source
COPY src/crappy-predict-news-labels/ /usr/src/crappy-predict-news-labels/

# Set PYTHONPATH and PATH so that PyCharm is able to resolve dependencies
ENV PYTHONPATH="/usr/src/crappy-predict-news-labels:${PYTHONPATH}" \
    PATH="/usr/src/crappy-predict-news-labels:${PATH}"

# Tagger port
EXPOSE 8080

# We can just kill -9 the thing
STOPSIGNAL SIGTERM

USER nobody
<commit_msg>Install only the required NLTK tokenizer<commit_after>#
# NYT-based news tagger service
#

FROM dockermediacloud/base:latest

RUN \
    #
    # Install model fetch dependencies
    apt-get -y --no-install-recommends install brotli && \
    #
    # Create directory for annotator
    mkdir -p /usr/src/crappy-predict-news-labels/models/ && \
    #
    true

# Download and extract models
# (get them first so that every code change doesn't trigger huge model redownload)
WORKDIR /usr/src/crappy-predict-news-labels/models/
ENV MODEL_URL="https://mediacloud-nytlabels-data.s3.amazonaws.com/predict-news-labels-onnx"

RUN /dl_to_stdout.sh "$MODEL_URL/GoogleNews-vectors-negative300.shelve.br" | \
        brotli -d > GoogleNews-vectors-negative300.shelve

RUN /dl_to_stdout.sh "$MODEL_URL/scaler.onnx" > scaler.onnx

RUN /dl_to_stdout.sh "$MODEL_URL/all_descriptors.onnx.br" | \
        brotli -d > all_descriptors.onnx
RUN /dl_to_stdout.sh "$MODEL_URL/all_descriptors.txt.br" | \
        brotli -d > all_descriptors.txt

RUN /dl_to_stdout.sh "$MODEL_URL/descriptors_3000.onnx.br" | \
        brotli -d > descriptors_3000.onnx
RUN /dl_to_stdout.sh "$MODEL_URL/descriptors_3000.txt.br" | \
        brotli -d > descriptors_3000.txt

RUN /dl_to_stdout.sh "$MODEL_URL/descriptors_600.onnx.br" | \
        brotli -d > descriptors_600.onnx
RUN /dl_to_stdout.sh "$MODEL_URL/descriptors_600.txt.br" | \
        brotli -d > descriptors_600.txt

RUN /dl_to_stdout.sh "$MODEL_URL/descriptors_with_taxonomies.onnx.br" | \
        brotli -d > descriptors_with_taxonomies.onnx
RUN /dl_to_stdout.sh "$MODEL_URL/descriptors_with_taxonomies.txt.br" | \
        brotli -d > descriptors_with_taxonomies.txt

RUN /dl_to_stdout.sh "$MODEL_URL/just_taxonomies.onnx.br" | \
        brotli -d > just_taxonomies.onnx
RUN /dl_to_stdout.sh "$MODEL_URL/just_taxonomies.txt.br" | \
        brotli -d > just_taxonomies.txt

# Install NLTK data
RUN \
    apt-get -y --no-install-recommends install unzip && \
    mkdir -p /usr/local/share/nltk_data/tokenizers/punkt/PY3/ && \
    /dl_to_stdout.sh "https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/packages/tokenizers/punkt.zip" \
        > /var/tmp/punkt.zip && \
    cd /var/tmp/ && \
    unzip punkt.zip && \
    rm punkt.zip && \
    cd /var/tmp/punkt/ && \
    cp english.pickle /usr/local/share/nltk_data/tokenizers/punkt/ && \
    cp PY3/english.pickle /usr/local/share/nltk_data/tokenizers/punkt/PY3/ && \
    cd / && \
    rm -rf /var/tmp/punkt/ && \
    apt-get -y remove unzip && \
    true

# Install Python
RUN \
    apt-get -y --no-install-recommends install \
        python3 \
        python3-dev \
        python3-pip \
        #
        # Needed by "shelve" module:
        python3-gdbm \
    && \
    true

# Install requirements
# (do this first so that minor changes in the annotator's code don't trigger a
# full module reinstall)
WORKDIR /usr/src/crappy-predict-news-labels/
COPY src/crappy-predict-news-labels/requirements.txt /usr/src/crappy-predict-news-labels/
RUN \
    pip3 install -r requirements.txt && \
    rm -rf /root/.cache/ && \
    true

# Copy the rest of the source
COPY src/crappy-predict-news-labels/ /usr/src/crappy-predict-news-labels/

# Set PYTHONPATH and PATH so that PyCharm is able to resolve dependencies
ENV PYTHONPATH="/usr/src/crappy-predict-news-labels:${PYTHONPATH}" \
    PATH="/usr/src/crappy-predict-news-labels:${PATH}"

# Tagger port
EXPOSE 8080

# We can just kill -9 the thing
STOPSIGNAL SIGTERM

USER nobody

{'fraction_non_alphanumeric': 0.12178598922247882, 'fraction_numerical': 0.01447267128560431, 'mean_word_length': 2.9156118143459917, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 28, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2996769', 'n_tokens_mistral': 2506, 'n_tokens_neox': 2431, 'n_words': 464}