<commit_before>#!/bin/bash

# Clear Postgres
sudo -u postgres bash <<EOF
dropdb rsr
createdb rsr -E UTF8 -O rsr -T template0
EOF

# Clear db dir
sudo -u rsr bash <<EOF
sudo rm -Rf /var/akvo/rsr/mediaroot/db
EOF

echo "Done loading new data!"<commit_msg>[#991] Progress on load script<commit_after>#!/bin/bash

WORKDIR=/tmp/rsr-load

# Make sure we have a clean working dir
sudo rm -rfv $WORKDIR
mkdir -p $WORKDIR

# Copy instead of curl
cp -rvf /var/akvo/rsr/code/data/dump/db $WORKDIR/db
cp -v /var/akvo/rsr/code/data/dump/rsr.dump $WORKDIR/

# User images
sudo -u rsr bash <<EOF
rm -rfv /var/akvo/rsr/mediaroot/db
rm -ffv /var/akvo/rsr/mediaroot/cache
cp -rv $WORKDIR/db /var/akvo/rsr/mediaroot/db

# Clean up thumbnails
cd /var/akvo/rsr
. venv/bin/activate
cd ./code
python ./manage.py thumbnail clear
deactivate
EOF


# Clear Postgres
sudo -u postgres bash <<EOF
dropdb rsr
createdb rsr -E UTF8 -O rsr -T template0
pg_restore -d rsr /tmp/rsr-load/rsr.dump
EOF

echo "Done loading new data!"
{'fraction_non_alphanumeric': 0.12072434607645875, 'fraction_numerical': 0.007042253521126761, 'mean_word_length': 3.4026548672566372, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4386208', 'n_tokens_mistral': 426, 'n_tokens_neox': 402, 'n_words': 127}