<commit_before>#!/bin/bash

# exit the script if any of the commands fail.
# further discussion: http://www.davidpashley.com/articles/writing-robust-shell-scripts/
set -e

# Do not run script as root.  Will fail installing pdfium if you do...
test $USER != 'root' || { echo do NOT run as root >&2; exit 1; }

USERNAME=ubuntu
CLI_ARG=$1
RAILS_ENVIRONMENT=${CLI_ARG:-"$RAILS_ENV"} 

# Install pdfium https://github.com/documentcloud/pdfshaver
# TODO Should we allow user to specifiy pdfium version?
cd /home/$USERNAME && sudo wget 'http://s3.documentcloud.org.s3.amazonaws.com/pdfium/libpdfium-dev_0.1%2Bgit20150311-1_amd64.deb'
sudo dpkg -i libpdfium-dev_0.1+git20150311-1_amd64.deb
sudo apt-get install libfreeimage-dev libfreetype6-dev

# approve github ssh host key
# run as ubuntu user so that we don't need to change permissions
sudo su -l ubuntu <<'EOF'
sudo grep -q github /home/ubuntu/.ssh/known_hosts 2>/dev/null || sudo ssh-keyscan -t rsa github.com > /home/ubuntu/.ssh/known_hosts
EOF
# Make sure you have your github keys authorized, installed, and chmod to 600!

test -e /home/$USERNAME/documentcloud || git clone git@github.com:documentcloud/documentcloud.git documentcloud
cd /home/$USERNAME/documentcloud
gem install bundler
bundle
# Don't forget to download your secrets file into documentcloud/secrets!

#./bin/rails runner -e production 'puts "You already have #{Document.count} documents"'

# if you would like to use git hooks to automatically bundle the app's gem dependencies:
# sudo ln -s /home/$USERNAME/documentcloud/config/server/files/git-post-merge.sh /home/$USERNAME/documentcloud/.git/hooks/post-merge<commit_msg>script setup complete<commit_after>#!/bin/bash

# exit the script if any of the commands fail.
# further discussion: http://www.davidpashley.com/articles/writing-robust-shell-scripts/
set -e

# Do not run script as root.  Will fail installing pdfium if you do...
test $USER != 'root' || { echo do NOT run as root >&2; exit 1; }

USERNAME=ubuntu
CLI_ARG=$1
RAILS_ENVIRONMENT=${CLI_ARG:-"$RAILS_ENV"} 

# Install pdfium https://github.com/documentcloud/pdfshaver
# TODO Should we allow user to specifiy pdfium version?
cd /home/$USERNAME && sudo wget 'http://s3.documentcloud.org.s3.amazonaws.com/pdfium/libpdfium-dev_0.1%2Bgit20150311-1_amd64.deb'
sudo dpkg -i libpdfium-dev_0.1+git20150311-1_amd64.deb
sudo apt-get install libfreeimage-dev libfreetype6-dev

# approve github ssh host key
# run as ubuntu user so that we don't need to change permissions
sudo su -l ubuntu <<EOF
sudo grep -q github /home/$USERNAME/.ssh/known_hosts 2>/dev/null || sudo ssh-keyscan -t rsa github.com > /home/$USERNAME/.ssh/known_hosts
EOF
# Make sure you have your github keys authorized, installed, and chmod to 600!

test -e /home/$USERNAME/documentcloud || git clone git@github.com:documentcloud/documentcloud.git documentcloud

sudo su -l ubuntu <<EOF
cd /home/$USERNAME/documentcloud
gem install bundler
bundle
EOF

# Don't forget to download your secrets file into documentcloud/secrets!

#./bin/rails runner -e production 'puts "You already have #{Document.count} documents"'

# if you would like to use git hooks to automatically bundle the app's gem dependencies:
# sudo ln -s /home/$USERNAME/documentcloud/config/server/files/git-post-merge.sh /home/$USERNAME/documentcloud/.git/hooks/post-merge
{'fraction_non_alphanumeric': 0.11087740384615384, 'fraction_numerical': 0.022235576923076924, 'mean_word_length': 4.51158940397351, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7569278', 'n_tokens_mistral': 1184, 'n_tokens_neox': 1068, 'n_words': 345}