<commit_before>#!/bin/bash

if [ ! -f /opt/solr/solr.tgz ]; then
sudo -u solr /bin/bash - <<"EOF"
curl https://archive.apache.org/dist/lucene/solr/4.4.0/solr-4.4.0.tgz > /opt/solr/solr.tgz
tar xvfz /opt/solr/solr.tgz -C /opt/solr
chmod g+w /opt/solr/ -R
# TODO: see https://bitbucket.org/siovene/astrobin/issue/257/migrate-to-haystack-2x
sed -i '/EnglishPorterFilterFactory/d' /opt/solr/solr-4.4.0/example/solr/collection1/conf/schema.xml
sed -i '/<\/fields>/i<field name="_version_" type="slong" indexed="true" stored="true" multiValued="false"\/>' /opt/solr/solr-4.4.0/example/solr/collection1/conf/schema.xml
EOF
fi

sudo -u astrobin /bin/bash - <<"EOF"
. /venv/astrobin/dev/bin/activate
. /var/www/astrobin/env/dev

/var/www/astrobin/manage.py build_solr_schema > /opt/solr/solr-4.4.0/example/solr/collection1/conf/schema.xml
EOF<commit_msg>(vagrant) Fix solr schema *after* its generation.<commit_after>#!/bin/bash

if [ ! -f /opt/solr/solr.tgz ]; then
sudo -u solr /bin/bash - <<"EOF"
curl https://archive.apache.org/dist/lucene/solr/4.4.0/solr-4.4.0.tgz > /opt/solr/solr.tgz
tar xvfz /opt/solr/solr.tgz -C /opt/solr
chmod g+w /opt/solr/ -R
EOF
fi

sudo -u astrobin /bin/bash - <<"EOF"
. /venv/astrobin/dev/bin/activate
. /var/www/astrobin/env/dev

/var/www/astrobin/manage.py build_solr_schema > /opt/solr/solr-4.4.0/example/solr/collection1/conf/schema.xml

# TODO: see https://bitbucket.org/siovene/astrobin/issue/257/migrate-to-haystack-2x
sed -i '/EnglishPorterFilterFactory/d' /opt/solr/solr-4.4.0/example/solr/collection1/conf/schema.xml
sed -i '/<\/fields>/i<field name="_version_" type="slong" indexed="true" stored="true" multiValued="false"\/>' /opt/solr/solr-4.4.0/example/solr/collection1/conf/schema.xml
EOF


{'fraction_non_alphanumeric': 0.19491035280508964, 'fraction_numerical': 0.025448235974551765, 'mean_word_length': 3.6133333333333333, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 11, 'https://': 4, 'lorem ipsum': 0, 'www.': 4, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1250905', 'n_tokens_mistral': 782, 'n_tokens_neox': 746, 'n_words': 107}