<commit_before>#!/bin/bash
# vim: sts=4 sw=4 et
set -e
set -x
cd $(dirname "$0")
PARENT="$(readlink -f ..)"

if [[ "$#" -ge 1 ]]; then
    git remote update
    git checkout -qf $1
    exec flock $PARENT/update.lock "$0"
else
    ENV_LINK=$PARENT/env
    ENV_DIR=$(mktemp -d -p "$PARENT" env.XXXXXXXXXX)
    PYTHON3="$(which python3)"

    virtualenv --python="$PYTHON3" "$ENV_DIR"

    $ENV_DIR/bin/pip install --upgrade pip wheel
    $ENV_DIR/bin/pip install -r requirements.txt
    $ENV_DIR/bin/python -m textblob.download_corpora

    OLD_ENV_DIR="$(readlink -f "$ENV_LINK")"
    ln -s -T --force "$ENV_DIR" "$ENV_LINK"
    rm -rf "$OLD_ENV_DIR"
    #sudo systemctl restart fewerror-twitter
fi
<commit_msg>Update update<commit_after>#!/bin/bash
# vim: sts=4 sw=4 et
set -e
set -x
cd $(dirname "$0")
PARENT="$(readlink -f ..)"

if [[ "$#" -ge 1 ]]; then
    git remote update
    git checkout -qf $1
    exec flock $PARENT/update.lock "$0"
else
    pipenv install
    pipenv run python -m textblob.download_corpora
    #sudo systemctl restart fewerror-twitter
fi

{'fraction_non_alphanumeric': 0.15478424015009382, 'fraction_numerical': 0.014071294559099437, 'mean_word_length': 3.1196911196911197, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1612310', 'n_tokens_mistral': 433, 'n_tokens_neox': 396, 'n_words': 113}