<commit_before>#!/bin/bash
set -exuo pipefail

cd $WORKSPACE/tubular
pip install -r requirements.txt
pip install awscli

set +x

SESSIONID=$(date +"%s")

RESULT=(`aws sts assume-role --role-arn $ROLE_ARN \
            --role-session-name $SESSIONID \
            --query '[Credentials.AccessKeyId,Credentials.SecretAccessKey,Credentials.SessionToken]' \
            --output text`)

export AWS_ACCESS_KEY_ID=${RESULT[0]}
export AWS_SECRET_ACCESS_KEY=${RESULT[1]}
export AWS_SECURITY_TOKEN=${RESULT[2]}
export AWS_SESSION_TOKEN=${AWS_SECURITY_TOKEN}

set -x

NAME_TAG="${ENVIRONMENT}-${DEPLOYMENT}-mongo"
IP_ADDRESSES=`aws ec2 describe-instances\
               --filter Name=tag:Name,Values=$NAME_TAG\
               --output text --query 'Reservations[*].Instances[*].PrivateIpAddress'\
               --region us-east-1`

MONGO_IPS=`echo $IP_ADDRESSES | sed 's/ /,/g'`

python scripts/structures.py\
        --database-name ${DATABASE_NAME}\
        --connection "mongodb://${MONGO_USER}:${MONGO_PASSWORD}@${MONGO_IPS}/${DATABASE_NAME}" make_plan plan.json

python scripts/structures.py\
        --database-name ${DATABASE_NAME}\
        --connection "mongodb://${MONGO_USER}:${MONGO_PASSWORD}@${MONGO_IPS}/${DATABASE_NAME}" prune --delay 5000 plan.json
<commit_msg>Change number of retained intermediates by mongo pruner (#561)<commit_after>#!/bin/bash
set -exuo pipefail

cd $WORKSPACE/tubular
pip install -r requirements.txt
pip install awscli

set +x

SESSIONID=$(date +"%s")

RESULT=(`aws sts assume-role --role-arn $ROLE_ARN \
            --role-session-name $SESSIONID \
            --query '[Credentials.AccessKeyId,Credentials.SecretAccessKey,Credentials.SessionToken]' \
            --output text`)

export AWS_ACCESS_KEY_ID=${RESULT[0]}
export AWS_SECRET_ACCESS_KEY=${RESULT[1]}
export AWS_SECURITY_TOKEN=${RESULT[2]}
export AWS_SESSION_TOKEN=${AWS_SECURITY_TOKEN}

set -x

NAME_TAG="${ENVIRONMENT}-${DEPLOYMENT}-mongo"
IP_ADDRESSES=`aws ec2 describe-instances\
               --filter Name=tag:Name,Values=$NAME_TAG\
               --output text --query 'Reservations[*].Instances[*].PrivateIpAddress'\
               --region us-east-1`

MONGO_IPS=`echo $IP_ADDRESSES | sed 's/ /,/g'`

python scripts/structures.py\
        --database-name ${DATABASE_NAME}\
        --connection "mongodb://${MONGO_USER}:${MONGO_PASSWORD}@${MONGO_IPS}/${DATABASE_NAME}" make_plan --retain 10 plan.json

python scripts/structures.py\
        --database-name ${DATABASE_NAME}\
        --connection "mongodb://${MONGO_USER}:${MONGO_PASSWORD}@${MONGO_IPS}/${DATABASE_NAME}" prune --delay 5000 plan.json

{'fraction_non_alphanumeric': 0.15363881401617252, 'fraction_numerical': 0.008856372737774355, 'mean_word_length': 3.6310160427807485, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7566736', 'n_tokens_mistral': 927, 'n_tokens_neox': 874, 'n_words': 156}