<commit_before>#!/bin/bash -x
OLDDIR=`pwd`

dropdb --if-exists dimcaldb
createdb --owner=dimcaluser dimcaldb

# Migrations use relative paths so it's important to run them from project root
cd ${TRAVIS_BUILD_DIR}/src/dimcal/
./manage.py migrate
uwsgi --master                                     \
      --socket 127.0.0.1:3031                      \
      --module=dimcal.wsgi:application             \
      --env DJANGO_SETTINGS_MODULE=dimcal.settings \
      --http :8000                                 \
      --processes 8 --threads 2                    \
      --stats 127.0.0.1:9191                       &
echo "Waiting 11sec for the server to start"
sleep 11s
echo "Starting crawl"
cd ${TRAVIS_BUILD_DIR}/wget_build/

# Thanks: http://stackoverflow.com/a/11850469/266387
./extract_links.py | xargs -n 1 -P 8 \
                 wget --recursive --no-host-directories --quiet

rm dim_calendar/about # This is configured on s3 with a redirect
cd ${OLDDIR}
echo **************************************** DONE! ****************************************

<commit_msg>Debug VI<commit_after>#!/bin/bash -x
OLDDIR=`pwd`

dropdb --if-exists dimcaldb
createdb --owner=dimcaluser dimcaldb

# Migrations use relative paths so it's important to run them from project root
cd ${TRAVIS_BUILD_DIR}/src/dimcal/
./manage.py migrate
uwsgi --master                                     \
      --socket 127.0.0.1:3031                      \
      --module=dimcal.wsgi:application             \
      --env DJANGO_SETTINGS_MODULE=dimcal.settings \
      --http :8000                                 \
      --processes 8 --threads 2                    \
      --stats 127.0.0.1:9191                       &
echo "Waiting 11sec for the server to start"
sleep 11s
echo "Starting crawl"
cd ${TRAVIS_BUILD_DIR}/wget_build/

# Thanks: http://stackoverflow.com/a/11850469/266387
./extract_links.py
./extract_links.py | xargs -n 1 -P 8 \
                 wget --recursive --no-host-directories --no-verbose

rm dim_calendar/about # This is configured on s3 with a redirect
cd ${OLDDIR}
echo **************************************** DONE! ****************************************


{'fraction_non_alphanumeric': 0.17230199166280685, 'fraction_numerical': 0.04353867531264474, 'mean_word_length': 1.946793997271487, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4862626', 'n_tokens_mistral': 720, 'n_tokens_neox': 631, 'n_words': 166}