<commit_before>#!/bin/bash

# This script is intended for setting up a development environment.
# Additional configuration is necessary for production.

set -e

if [ "$EUID" -ne 0 ]
then
    echo "You must run the install script as root!"
    exit
fi

function devconfig() {
    python3 -c "
import json
with open('/home/ubuntu/director/config/devconfig.json', 'r') as f:
    print(json.load(f)['$1'])"
}

export DEBIAN_FRONTEND=noninteractive

if ! grep -Fxq "openldap1.csl.tjhsst.edu" /etc/hosts
then
    echo 172.16.17.32 openldap1.csl.tjhsst.edu >> /etc/hosts
fi

DB_PASS='<PASSWORD>'

apt-get update
apt-get upgrade -y

apt-get install -y git
apt-get install -y php-fpm

apt-get install -y sudo python python-dev python3 python3-dev python3-pip virtualenv libnss-pgsql2 nodejs nodejs-legacy supervisor
apt-get install -y postgresql postgresql-contrib libpq-dev libmysqlclient-dev nginx

mkdir -p /etc/nginx/director.d/
mkdir -p /etc/supervisor/director.d/
mkdir -p /var/log/gunicorn/

cd director/

cp web3/settings/secret.sample web3/settings/secret.py

cp config/dev-nginx.conf /etc/nginx/nginx.conf
nginx -s reload

# Set the database password
sed -i 's/^DB_PASSWORD.*/DB_PASSWORD = "'"$<PASSWORD>"'"/' web3/settings/secret.py

# Turn on debug to see more detailed error messages
sed -i 's/^DEBUG.*/DEBUG = True/' web3/settings/secret.py

# Remove Raven logging
sed -i '/dsn/d' web3/settings/secret.py

sed -i 's/^SOCIAL_AUTH_ION_KEY.*/SOCIAL_AUTH_ION_KEY = "'"$(devconfig ion_key)"'"/' web3/settings/secret.py
sed -i 's/^SOCIAL_AUTH_ION_SECRET.*/SOCIAL_AUTH_ION_SECRET = "'"$(devconfig ion_secret)"'"/' web3/settings/secret.py

# Create web3 user and database
sudo -u postgres createuser -D -A "web3" || echo "User already exists"
sudo -u postgres psql -c "ALTER USER web3 WITH PASSWORD '$<PASSWORD>';"
sudo -u postgres createdb -O "web3" "web3" || echo "Database already exists"

# Create www-data user
useradd www-data || echo "www-data user already exists"

mkdir -p /web

if [ ! -d "venv" ]; then
    virtualenv --python python3 venv
fi
source venv/bin/activate
pip install -U -r requirements.txt

./manage.py migrate

# Set up nss-pgsql
apt-get -y install nscd
cp config/nss-pgsql.conf /etc/nss-pgsql.conf
chmod 644 /etc/nss-pgsql.conf
sed -i 's/^connectionstring.*/connectionstring = hostaddr=127.0.0.1 dbname=web3 user=web3 password='"<PASSWORD>"' connect_timeout=1/' /etc/nss-pgsql.conf
sed -i 's/^passwd:.*/passwd: compat pgsql/' /etc/nsswitch.conf
sed -i 's/^group:.*/group: compat pgsql/' /etc/nsswitch.conf
/usr/sbin/nscd -i group
/usr/sbin/nscd -i passwd

cp config/dev-supervisord.conf /etc/supervisor/supervisord.conf
supervisorctl reread
supervisorctl update

# Automatically start supervisor on boot
systemctl enable supervisor

supervisorctl restart director
supervisorctl restart directornode<commit_msg>separate sql user for nss<commit_after>#!/bin/bash

# This script is intended for setting up a development environment.
# Additional configuration is necessary for production.

set -e

if [ "$EUID" -ne 0 ]
then
    echo "You must run the install script as root!"
    exit
fi

function devconfig() {
    python3 -c "
import json
with open('/home/ubuntu/director/config/devconfig.json', 'r') as f:
    print(json.load(f)['$1'])"
}

export DEBIAN_FRONTEND=noninteractive

# Add DNS entry for LDAP server
if ! grep -Fxq "openldap1.csl.tjhsst.edu" /etc/hosts
then
    echo 172.16.17.32 openldap1.csl.tjhsst.edu >> /etc/hosts
fi

DB_PASS='<PASSWORD>'
NSS_PASS='<PASSWORD>'

apt-get update
apt-get upgrade -y

apt-get install -y git
apt-get install -y php-fpm

apt-get install -y sudo python python-dev python3 python3-dev python3-pip virtualenv libnss-pgsql2 nodejs nodejs-legacy supervisor
apt-get install -y postgresql postgresql-contrib libpq-dev libmysqlclient-dev nginx

mkdir -p /etc/nginx/director.d/
mkdir -p /etc/supervisor/director.d/
mkdir -p /var/log/gunicorn/

cd director/

cp web3/settings/secret.sample web3/settings/secret.py

cp config/dev-nginx.conf /etc/nginx/nginx.conf
nginx -s reload

# Set the database password
sed -i 's/^DB_PASSWORD.*/DB_PASSWORD = "'"$<PASSWORD>"'"/' web3/settings/secret.py

# Turn on debug to see more detailed error messages
sed -i 's/^DEBUG.*/DEBUG = True/' web3/settings/secret.py

# Remove Raven logging
sed -i '/dsn/d' web3/settings/secret.py

# Setup Ion OAuth
sed -i 's/^SOCIAL_AUTH_ION_KEY.*/SOCIAL_AUTH_ION_KEY = "'"$(devconfig ion_key)"'"/' web3/settings/secret.py
sed -i 's/^SOCIAL_AUTH_ION_SECRET.*/SOCIAL_AUTH_ION_SECRET = "'"$(devconfig ion_secret)"'"/' web3/settings/secret.py

# Create web3 user and database
sudo -u postgres createuser -D -A "web3" || echo "web3 user already exists"
sudo -u postgres psql -c "ALTER USER web3 WITH PASSWORD '$<PASSWORD>';"
sudo -u postgres createdb -O "web3" "web3" || echo "Database already exists"

# Create nss user and give access
sudo -u postgres createuser -D -A "nss" || echo "nss user already exists"
sudo -u postgres psql -c "ALTER USER nss WITH PASSWORD '$<PASSWORD>';"

# Give nss user specific permissions
sudo -u postgres psql -c "GRANT CONNECT ON DATABASE web3 TO nss;"
sudo -u postgres psql -d "web3" -c "GRANT SELECT ON TABLE users_user TO nss;"
sudo -u postgres psql -d "web3" -c "GRANT SELECT ON TABLE users_group TO nss;"
sudo -u postgres psql -d "web3" -c "GRANT SELECT ON TABLE users_group_users TO nss;"
sudo -u postgres psql -d "web3" -c "GRANT SELECT ON TABLE users_user_groups TO nss;"

# Create www-data user
useradd www-data || echo "www-data user already exists"

mkdir -p /web

if [ ! -d "venv" ]; then
    virtualenv --python python3 venv
fi
source venv/bin/activate
pip install -U -r requirements.txt

./manage.py migrate

# Setup nss-pgsql
apt-get -y install nscd
cp config/nss-pgsql.conf /etc/nss-pgsql.conf
chmod 644 /etc/nss-pgsql.conf
sed -i 's/^connectionstring.*/connectionstring = hostaddr=127.0.0.1 dbname=web3 user=nss password='"$<PASSWORD>"' connect_timeout=1/' /etc/nss-pgsql.conf
sed -i 's/^passwd:.*/passwd: compat pgsql/' /etc/nsswitch.conf
sed -i 's/^group:.*/group: compat pgsql/' /etc/nsswitch.conf
/usr/sbin/nscd -i group
/usr/sbin/nscd -i passwd

# Setup supervisor
cp config/dev-supervisord.conf /etc/supervisor/supervisord.conf
supervisorctl reread
supervisorctl update

# Automatically start supervisor on boot
systemctl enable supervisor

supervisorctl restart director
supervisorctl restart directornode
{'fraction_non_alphanumeric': 0.1242557192102789, 'fraction_numerical': 0.014258853024130366, 'mean_word_length': 4.0180817610062896, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6689466', 'n_tokens_mistral': 2439, 'n_tokens_neox': 2331, 'n_words': 693}