<commit_before>#!/bin/sh

# This script creates a backup tgz file containing the following assets:
#
# 1. Dumps from Postgres of the CKAN databases.
# 2. A copy of the SOLR data directory.
# 3. A copy of CKAN's filestore directory.

# Set some useful variables.
USER_HOME=$(eval echo ~${SUDO_USER})
backup_dir="$USER_HOME/backup"
hostname=$(hostname -s)
date=`date '+%Y-%m-%d_%H-%M-%S'`
filename="$date-$hostname.tgz"
backup_target="$USER_HOME/$filename"

# Clean up and ensure the script is in a good state.
sudo rm -rf $backup_dir
rm *.tgz
mkdir -p "$backup_dir"

# Use pg_dump on the databases.
/usr/lib/ckan/default/bin/paster --plugin=ckan db dump -c /etc/ckan/default/production.ini "$backup_dir/ckan_default.sql"
sudo su - postgres -c "pg_dump -Fc datastore_default" > "$backup_dir/datastore_default.dump"

# Copy the filestore directory into the backup directory. This will contain any
# newly uploaded data-source assets (it's just an extra source of safety in
# case the S3 push at the end fails).
sudo cp -r /var/lib/ckan/default "$backup_dir/default"

# Create the tgz
cd $backup_dir
tar cfz $backup_target .

# Upload to S3
cd $USER_HOME
python upload.py $backup_target
if [ $? -eq 0 ] then
  rm $backup_target
fi


# Push any new data-source assets to S3
sudo /usr/lib/ckan/default/bin/paster --plugin=ckanext-s3archive s3archive archive -c /etc/ckan/default/production.ini<commit_msg>Switches off s3-plugin paster command to push recently uploaded files from the filestore to S3. THIS BREAKS THE DATASTORE.<commit_after>#!/bin/sh

# This script creates a backup tgz file containing the following assets:
#
# 1. Dumps from Postgres of the CKAN databases.
# 2. A copy of the SOLR data directory.
# 3. A copy of CKAN's filestore directory.

# Set some useful variables.
USER_HOME=$(eval echo ~${SUDO_USER})
backup_dir="$USER_HOME/backup"
hostname=$(hostname -s)
date=`date '+%Y-%m-%d_%H-%M-%S'`
filename="$date-$hostname.tgz"
backup_target="$USER_HOME/$filename"

# Clean up and ensure the script is in a good state.
sudo rm -rf $backup_dir
rm *.tgz
mkdir -p "$backup_dir"

# Use pg_dump on the databases.
/usr/lib/ckan/default/bin/paster --plugin=ckan db dump -c /etc/ckan/default/production.ini "$backup_dir/ckan_default.sql"
sudo su - postgres -c "pg_dump -Fc datastore_default" > "$backup_dir/datastore_default.dump"

# Copy the filestore directory into the backup directory. This will contain any
# newly uploaded data-source assets (it's just an extra source of safety in
# case the S3 push at the end fails).
sudo cp -r /var/lib/ckan/default "$backup_dir/default"

# Create the tgz
cd $backup_dir
tar cfz $backup_target .

# Upload to S3
cd $USER_HOME
python upload.py $backup_target
if [ $? -eq 0 ] then
  rm $backup_target
fi


# Push any new data-source assets to S3
#sudo /usr/lib/ckan/default/bin/paster --plugin=ckanext-s3archive s3archive archive -c /etc/ckan/default/production.ini
{'fraction_non_alphanumeric': 0.11279229711141678, 'fraction_numerical': 0.0068775790921595595, 'mean_word_length': 4.269927536231884, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1747717', 'n_tokens_mistral': 1031, 'n_tokens_neox': 982, 'n_words': 343}