<commit_before>#!/bin/bash
set -e

manage='sudo -H -u rsr /var/akvo/rsr/venv/bin/python /var/akvo/rsr/code/manage.py'

# if we have already done the loading, don't do it again
if [ -e /etc/localdev_rsr_provisioned ]
then
    $manage syncdb --noinput
    $manage migrate
    $manage collectstatic --noinput
    exit 0
fi

# For new data update the DUMP_URL
DUMP_URL='http://files.support.akvo-ops.org/devdbs/rsr_dump.20150211_191922.tar.gz'
DUMPDIR='/var/akvo/rsr/code/data/dump'
mkdir -p $DUMPDIR
cd $DUMPDIR
curl -L $DUMP_URL > $DUMPDIR/rsr_dump.tar.gz
cd /var/akvo/rsr/code/data/ && ./load.sh

$manage migrate
$manage collectstatic --noinput
<commit_msg>[#1187] New data set!<commit_after>#!/bin/bash
set -e

manage='sudo -H -u rsr /var/akvo/rsr/venv/bin/python /var/akvo/rsr/code/manage.py'

# if we have already done the loading, don't do it again
if [ -e /etc/localdev_rsr_provisioned ]
then
    $manage syncdb --noinput
    $manage migrate
    $manage collectstatic --noinput
    exit 0
fi

# For new data update the DUMP_URL
DUMP_URL='http://files.support.akvo-ops.org/devdbs/rsr_dump.20150224_135110.tar.gz'
# DUMP_URL='http://files.support.akvo-ops.org/devdbs/rsr_dump.20150211_191922.tar.gz'
DUMPDIR='/var/akvo/rsr/code/data/dump'
mkdir -p $DUMPDIR
cd $DUMPDIR
curl -L $DUMP_URL > $DUMPDIR/rsr_dump.tar.gz
cd /var/akvo/rsr/code/data/ && ./load.sh

$manage migrate
$manage collectstatic --noinput

{'fraction_non_alphanumeric': 0.1422475106685633, 'fraction_numerical': 0.034139402560455195, 'mean_word_length': 3.6435643564356437, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1993223', 'n_tokens_mistral': 643, 'n_tokens_neox': 582, 'n_words': 124}