<commit_before>#!/usr/bin/env python

###############################################################################
#
# SageMathCloud: A collaborative web-based interface to Sage, IPython, LaTeX and the Terminal.
#
#    Copyright (C) 2014, 2015, <NAME>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################


"""

GS = [G]oogle Cloud Storage / [B]trfs - based project storage system

    export DEV=/dev/sdX; export MOUNT=/projects

    mkfs.btrfs $DEV && mkdir -p $MOUNT && mount -o compress-force=lzo,noatime $DEV $MOUNT && btrfs quota enable $MOUNT && chmod og-rw $MOUNT && chmod og+x $MOUNT && btrfs subvolume create $MOUNT/conf && chown salvus. $MOUNT/conf && btrfs subvolume create $MOUNT/.snapshots

    btrfs subvolume create $MOUNT/sagemathcloud && sudo rsync -LrxH --delete /home/salvus/salvus/salvus/local_hub_template/ $MOUNT/sagemathcloud/

# Worry about tmp, e.g.,

    btrfs subvolume create $MOUNT/tmp && chmod 1777 $MOUNT/tmp && mount -o bind $MOUNT/tmp /tmp/

# Start compute server (as user salvus)

    echo 'export SMC_BTRFS=/$MOUNT; export SMC_BUCKET=gs://smc-gb-storageXXX; export SMC_ARCHIVE=gs://smc-gb-archiveXXX' >> $HOME/.bashrc
    source $HOME/.bashrc; cd ~/salvus/salvus; . salvus-env; ./compute start

# Database entry:

    cd $MOUNT/conf && echo "update compute_servers set dc='us-central1-c', port=`cat compute.port`, secret='`cat compute.secret`' where host='`hostname`';"



# For dedup support:

    cd /tmp && rm -rf duperemove && git clone https://github.com/markfasheh/duperemove && cd duperemove && sudo make install && rm -rf /tmp/duperemove

"""

# used in naming streams -- changing this would break all existing data...
TO      = "-to-"

# appended to end of snapshot name to make it persistent (never automatically deleted)
PERSIST = "-persist"

UID_WHITELIST = "/root/smc-iptables/uid_whitelist"

TIMESTAMP_FORMAT = "%Y-%m-%d-%H%M%S"

# This is the quota for the .sagemathcloud directory; must be<commit_msg>improve comments about setting up filesystem on compute nodes<commit_after>#!/usr/bin/env python

###############################################################################
#
# SageMathCloud: A collaborative web-based interface to Sage, IPython, LaTeX and the Terminal.
#
#    Copyright (C) 2014, 2015, <NAME>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################


"""

GS = [G]oogle Cloud Storage / [B]trfs - based project storage system

Use fdisk to make /dev/sdb1 for swap (80GB) and /dev/sdb2 for /projects (rest):

    export SWAP=/dev/sdb1
    mkswap $SWAP && swapon $SWAP && echo "$SWAP swap swap defaults 0 0" >> /etc/fstab

    export DEV=/dev/sdb2; export MOUNT=/projects
    mkfs.btrfs $DEV && mkdir -p $MOUNT && mount -o compress-force=lzo,noatime $DEV $MOUNT && btrfs quota enable $MOUNT && chmod og-rw $MOUNT && chmod og+x $MOUNT && btrfs subvolume create $MOUNT/conf && chown salvus. $MOUNT/conf && btrfs subvolume create $MOUNT/.snapshots && btrfs subvolume create $MOUNT/sagemathcloud && sudo rsync -LrxH --delete /home/salvus/salvus/salvus/local_hub_template/ $MOUNT/sagemathcloud/ && btrfs subvolume create $MOUNT/tmp && chmod 1777 $MOUNT/tmp && mount -o bind $MOUNT/tmp /tmp/ && echo "$DEV $MOUNT btrfs defaults 0 2" >> /etc/fstab && echo "$MOUNT/tmp  /tmp bind 0 2" >> /etc/fstab 




# Start compute server (as user salvus)

    echo 'export SMC_BTRFS=/$MOUNT; export SMC_BUCKET=gs://smc-gb-storageXXX; export SMC_ARCHIVE=gs://smc-gb-archiveXXX' >> $HOME/.bashrc
    source $HOME/.bashrc; cd ~/salvus/salvus; . salvus-env; ./compute start

# Database entry:

    cd $MOUNT/conf && echo "update compute_servers set dc='us-central1-c', port=`cat compute.port`, secret='`cat compute.secret`' where host='`hostname`';"



# For dedup support:

    cd /tmp && rm -rf duperemove && git clone https://github.com/markfasheh/duperemove && cd duperemove && sudo make install && rm -rf /tmp/duperemove

"""

# used in naming streams -- changing this would break all existing data...
TO      = "-to-"

# appended to end of snapshot name to make it persistent (never automatically deleted)
PERSIST = "-persist"

UID_WHITELIST = "/root/smc-iptables/uid_whitelist"

TIMESTAMP_FORMAT = "%Y-%m-%d-%H%M%S"

# This is the quota for the .sagemathcloud directory; must be
{'fraction_non_alphanumeric': 0.16604444444444444, 'fraction_numerical': 0.0071111111111111115, 'mean_word_length': 3.91353711790393, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 17, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6485939', 'n_tokens_mistral': 1822, 'n_tokens_neox': 1725, 'n_words': 615}