<commit_before>#! /usr/bin/env python3
"""
Sets up an Ubuntu 14.04 x64 server to be a cSFRblock Federated Node.

NOTE: The system should be properly secured before running this script.

TODO: This is admittedly a (bit of a) hack. In the future, take this kind of functionality out to a .deb with
      a postinst script to do all of this, possibly.
"""
import os
import sys
import re
import time
import getopt
import logging
import shutil
import socket
import urllib
import zipfile
import platform
import collections
import tempfile
import subprocess

try: #ignore import errors on windows
    import pwd
    import grp
except ImportError:
    pass

from setup_util import *

USERNAME = "csfr"
DAEMON_USERNAME = "csfrd"
USER_HOMEDIR = "/home/csfr"
QUESTION_FLAGS = collections.OrderedDict({
    "op": ('u', 'r'),
    "role": ('csfrwallet', 'vendingmachine', 'blockexplorer', 'csfrd_only', 'btcpayescrow'),
    "branch": ('master', 'develop'),
    "run_mode": ('t', 'm', 'b'),
    "backend_rpc_mode": ('s', 'p'),
    "blockchain_service": ('b', 'i'),
    "security_hardening": ('y', 'n'),
    "csfrd_public": ('y', 'n'),
    "csfrwallet_support_email": None,
    "autostart_services": ('y', 'n'),
})

def do_base_setup(run_as_user, branch, base_path, dist_path):
    """This creates the csfr and csfrd users and checks out the csfrd_build system from git"""
    #change time to UTC
    runcmd("ln -sf /usr/share/zoneinfo/UTC /etc/localtime")

    #install some necessary base deps
    runcmd("apt-get update")
    runcmd("apt-get -y install git-core software-properties-common python-software-properties build-essential ssl-cert ntp runit")
    
    #install node-js
    #node-gyp building has ...issues out of the box on Ubuntu... use <NAME>'s nodejs build instead, which is newer
    runcmd("apt-get -y remove nodejs npm gyp")
    runcmd("add-apt-repository -y ppa:chris-lea/node.js")
    runcmd("apt-get update")
    runcmd("apt-get -y install nodejs") #includes npm
    gypdir = None
    try:
        import gyp
        gypdir = os.path.dirname(gyp.__file__)
    except:
        pass
    else:
        runcmd("mv %s %s_bkup" % (gypdir, gypdir))
        #^ fix for https://github.com/TooTallNate/node-gyp/issues/363

    #Create csfr user, under which the files will be stored, and who will own the files, etc
    try:
        pwd.getpwnam(USERNAME)
    except:
        logging.info("Creating user '%s' ..." % USERNAME)
        runcmd("adduser --system --disabled-password --shell /bin/false --group %s" % USERNAME)
        
    #Create csfrd user (to run csfrd, csfrblockd, insight, saffroncoind, nginx) if not already made
    try:
        pwd.getpwnam(DAEMON_USERNAME)
    except:
        logging.info("Creating user '%s' ..." % DAEMON_USERNAME)
        runcmd("adduser --system --disabled-password --shell /bin/false --ingroup nogroup --home %s %s" % (USER_HOMEDIR, DAEMON_USERNAME))
    
    #add the run_as_user to the csfr group
    runcmd("adduser %s %s" % (run_as_user, USERNAME))
    
    #Check out csfrd-build repo under this user's home dir and use that for the build
    git_repo_clone("csfrd_build", "https://github.com/saffroncoin/csfrd_build.git",
        os.path.join(USER_HOMEDIR, "csfrd_build"), branch, for_user=run_as_user)

    #enhance fd limits for the csfrd user
    runcmd("cp -af %s/linux/other/csfrd_security_limits.conf /etc/security/limits.d/" % dist_path)

def do_backend_rpc_setup(run_as_user, branch, base_path, dist_path, run_mode, backend_rpc_mode):
    """Installs and configures saffroncoind"""
    backend_rpc_password = <PASSWORD>()
    backend_rpc_password_testnet = <PASSWORD>()

    if backend_rpc_mode == 's': #saffroncoind
        #Install saffroncoind
        SAFFRONCOIND_VER = "1.3.4"
        runcmd("rm -rf /tmp/saffroncoind.tar.gz /tmp/saffroncoin-%s-linux" % SAFFRONCOIND_VER)
        runcmd("wget -O /tmp/saffroncoind.tar.gz https://github.com/saffroncoin/saffroncoin/releases/download/%s/saffroncoin-binary-%s.tar.gz" % (SAFFRONCOIND_VER, SAFFRONCOIND_VER))
        runcmd("tar -C /tmp -zxvf /tmp/saffroncoind.tar.gz")
        runcmd("cp -af /tmp/saffroncoind /usr/bin")
        runcmd("rm -rf /tmp/saffroncoind.tar.gz /tmp/saffroncoind")
    
        #Do basic inital saffroncoin config (for both testnet and mainnet)
        runcmd("mkdir -p ~%s/.saffroncoin ~%s/.saffroncoin-testnet" % (USERNAME, USERNAME))
        if not os.path.exists(os.path.join(USER_HOMEDIR, '.saffroncoin', 'saffroncoin.conf')):
            runcmd(r"""bash -c 'echo -e "rpcuser=rpc\nrpcpassword=%s\nserver=1\ndaemon=1\ntxindex=1" > ~%s/.saffroncoin/saffroncoin.conf'""" % (
                backend_rpc_password, USERNAME))
        else: #grab the existing RPC password
            backend_rpc_password = subprocess.check_output(
                r"""bash -c "cat ~%s/.saffroncoin/saffroncoin.conf | sed -n 's/.*rpcpassword=\([^ \n]*\).*/\1/p'" """ % USERNAME, shell=True).strip().decode('utf-8')
        if not os.path.exists(os.path.join(USER_HOMEDIR, '.saffroncoin-testnet', 'saffroncoin.conf')):
            runcmd(r"""bash -c 'echo -e "rpcuser=rpc\nrpcpassword=%s\nserver=1\ndaemon=1\ntxindex=1\ntestnet=1" > ~%s/.saffroncoin-testnet/saffroncoin.conf'""" % (
                backend_rpc_password_testnet, USERNAME))
        else:
            backend_rpc_password_testnet = subprocess.check_output(
                r"""bash -c "cat ~%s/.saffroncoin-testnet/saffroncoin.conf | sed -n 's/.*rpcpassword=\([^ \n]*\).*/\1/p'" """
                % USERNAME, shell=True).strip().decode('utf-8')

        #remove pyrpcwallet startup scripts if present
        runcmd("rm -f /etc/service/pyrpcwallet /etc/service/pyrpcwallet-testnet")
        #install logrotate file
        runcmd("cp -dRf --preserve=mode %s/linux/logrotate/saffroncoind /etc/logrotate.d/saffroncoind" % dist_path)
        #set permissions
        runcmd("chown -R %s:%s ~%s/.saffroncoin ~%s/.saffroncoin-testnet" % (DAEMON_USERNAME, USERNAME, USERNAME, USERNAME))
        #set up runit startup scripts
        config_runit_for_service(dist_path, "saffroncoind", enabled=run_mode in ['m', 'b'])
        config_runit_for_service(dist_path, "saffroncoind-testnet", enabled=run_mode in ['t', 'b'])
    else:
        assert backend_rpc_mode == 'p'
        #checkout pyrpcwallet
        git_repo_clone("pyrpcwallet", "https://github.com/saffroncoin/pyrpcwallet.git",
            os.path.join(USER_HOMEDIR, "pyrpcwallet"), branch="master", for_user=run_as_user, hash="bdeef54ca74ebd5990350d62b57280d970af3c04")
        #remove saffroncoind startup scripts if present
        runcmd("rm -f /etc/service/saffroncoind /etc/service/saffroncoind-testnet")
        #change passwords
        modify_config(r'^RPC_PASSWORD=.*?$', 'RPC_PASSWORD=' + backend_rpc_password, '/etc/sv/pyrpcwallet/run')
        modify_config(r'^RPC_PASSWORD=.*?$', 'RPC_PASSWORD=' + backend_rpc_password_testnet, '/etc/sv/pyrpcwallet-testnet/run')
        #set permissions
        runcmd("mkdir -p ~%s/pyrpcwallet/log ~%s/pyrpcwallet/log-testnet" % (USERNAME, USERNAME))
        runcmd("chown -R %s:%s ~%s/pyrpcwallet ~%s/pyrpcwallet/log ~%s/pyrpcwallet/log-testnet" % (DAEMON_USERNAME, USERNAME, USERNAME, USERNAME, USERNAME))
        #set up runit startup scripts
        config_runit_for_service(dist_path, "pyrpcwallet", enabled=run_mode in ['m', 'b'])
        config_runit_for_service(dist_path, "pyrpcwallet-testnet", enabled=run_mode in ['t', 'b'])
    
    return backend_rpc_password, backend_rpc_password_testnet

def do_csfrd_setup(role, run_as_user, docker, branch, base_path, dist_path, run_mode, backend_rpc_password,
backend_rpc_password_testnet, csfrpartyd_public, csfrwallet_support_email):
    """Installs and configures csfrd and csfrblockd"""
    csfrd_rpc_password = '<PASSWORD>' if role == 'csfrd_only' and csfrd_public == 'y' else pass_generator()
    csfrd_rpc_password_testnet = '<PASSWORD>' if role == 'csfrd_only' and csfrd_public == 'y' else pass_generator()
    
    #Run setup.py (as the cSFR user, who runs it sudoed) to install and set up csfrpartyd, csfrblockd
    # as -y is specified, this will auto install csfrblockd full node (mongo and redis) as well as setting
    # csfrpartyd/csfrblockd to start up at startup for both mainnet and testnet (we will override this as necessary
    # based on run_mode later in this function)
    runcmd("~%s/csfrd_build/setup.py --noninteractive --branch=%s %s --with-testnet --for-user=%s %s" % (
        USERNAME, branch, "--with-bootstrap-db" if not docker else '', USERNAME, '--with-csfrblockd' if role != 'csfrd_only' else ''))
    runcmd("cd ~%s/csfrd_build && git config core.sharedRepository group && find ~%s/csfrd_build -type d -print0 | xargs -0 chmod g+s" % (
        USERNAME, USERNAME)) #to allow for group git actions 
    runcmd("chown -R %s:%s ~%s/csfrd_build" % (USERNAME, USERNAME, USERNAME)) #just in case
    runcmd("chmod -R u+rw,g+rw,o+r,o-w ~%s/csfrd_build" % USERNAME) #just in case
    
    #now change the csfrpartyd directories to be owned by the csfrd user (and the csfr group),
    # so that the csfrd account can write to the database, saved image files (csfrblockd), log files, etc
    runcmd("mkdir -p ~%s/.config/csfrd ~%s/.config/csfrd-testnet" % (USERNAME, USERNAME))    
    runcmd("chown -R %s:%s ~%s/.config/csfrd ~%s/.config/csfrd-testnet" % (
        DAEMON_USERNAME, USERNAME, USERNAME, USERNAME))
    runcmd("sed -ri \"s/USER=%s/USER=%s/g\" /etc/sv/csfrd/run /etc/sv/csfrd-testnet/run" % (
        USERNAME, DAEMON_USERNAME))

    #modify the default stored saffroncoind passwords in csfrd.conf
    modify_cp_config(r'^(backend|saffroncoind)\-rpc\-password=.*?$',
        'backend-rpc-password=%s' % backend_rpc_password, config='csfrd', net='mainnet', for_user=USERNAME)<commit_msg>Fixes<commit_after>#! /usr/bin/env python3
"""
Sets up an Ubuntu 14.04 x64 server to be a cSFRblock Federated Node.

NOTE: The system should be properly secured before running this script.

TODO: This is admittedly a (bit of a) hack. In the future, take this kind of functionality out to a .deb with
      a postinst script to do all of this, possibly.
"""
import os
import sys
import re
import time
import getopt
import logging
import shutil
import socket
import urllib
import zipfile
import platform
import collections
import tempfile
import subprocess

try: #ignore import errors on windows
    import pwd
    import grp
except ImportError:
    pass

from setup_util import *

USERNAME = "csfr"
DAEMON_USERNAME = "csfrd"
USER_HOMEDIR = "/home/csfr"
QUESTION_FLAGS = collections.OrderedDict({
    "op": ('u', 'r'),
    "role": ('csfrwallet', 'vendingmachine', 'blockexplorer', 'csfrd_only', 'btcpayescrow'),
    "branch": ('master', 'develop'),
    "run_mode": ('t', 'm', 'b'),
    "backend_rpc_mode": ('s', 'p'),
    "blockchain_service": ('b', 'i'),
    "security_hardening": ('y', 'n'),
    "csfrd_public": ('y', 'n'),
    "csfrwallet_support_email": None,
    "autostart_services": ('y', 'n'),
})

def do_base_setup(run_as_user, branch, base_path, dist_path):
    """This creates the csfr and csfrd users and checks out the csfrd_build system from git"""
    #change time to UTC
    runcmd("ln -sf /usr/share/zoneinfo/UTC /etc/localtime")

    #install some necessary base deps
    runcmd("apt-get update")
    runcmd("apt-get -y install git-core software-properties-common python-software-properties build-essential ssl-cert ntp runit")
    
    #install node-js
    #node-gyp building has ...issues out of the box on Ubuntu... use <NAME>'s nodejs build instead, which is newer
    runcmd("apt-get -y remove nodejs npm gyp")
    runcmd("add-apt-repository -y ppa:chris-lea/node.js")
    runcmd("apt-get update")
    runcmd("apt-get -y install nodejs") #includes npm
    gypdir = None
    try:
        import gyp
        gypdir = os.path.dirname(gyp.__file__)
    except:
        pass
    else:
        runcmd("mv %s %s_bkup" % (gypdir, gypdir))
        #^ fix for https://github.com/TooTallNate/node-gyp/issues/363

    #Create csfr user, under which the files will be stored, and who will own the files, etc
    try:
        pwd.getpwnam(USERNAME)
    except:
        logging.info("Creating user '%s' ..." % USERNAME)
        runcmd("adduser --system --disabled-password --shell /bin/false --group %s" % USERNAME)
        
    #Create csfrd user (to run csfrd, csfrblockd, insight, saffroncoind, nginx) if not already made
    try:
        pwd.getpwnam(DAEMON_USERNAME)
    except:
        logging.info("Creating user '%s' ..." % DAEMON_USERNAME)
        runcmd("adduser --system --disabled-password --shell /bin/false --ingroup nogroup --home %s %s" % (USER_HOMEDIR, DAEMON_USERNAME))
    
    #add the run_as_user to the csfr group
    runcmd("adduser %s %s" % (run_as_user, USERNAME))
    
    #Check out csfrd-build repo under this user's home dir and use that for the build
    git_repo_clone("csfrd_build", "https://github.com/saffroncoin/csfrd_build.git",
        os.path.join(USER_HOMEDIR, "csfrd_build"), branch, for_user=run_as_user)

    #enhance fd limits for the csfrd user
    runcmd("cp -af %s/linux/other/csfrd_security_limits.conf /etc/security/limits.d/" % dist_path)

def do_backend_rpc_setup(run_as_user, branch, base_path, dist_path, run_mode, backend_rpc_mode):
    """Installs and configures saffroncoind"""
    backend_rpc_password = <PASSWORD>()
    backend_rpc_password_testnet = <PASSWORD>_generator()

    if backend_rpc_mode == 's': #saffroncoind
        #Install saffroncoind
        SAFFRONCOIND_VER = "1.3.4"
        runcmd("rm -rf /tmp/saffroncoind.tar.gz /tmp/saffroncoin-%s-linux" % SAFFRONCOIND_VER)
        runcmd("wget -O /tmp/saffroncoind.tar.gz https://github.com/saffroncoin/saffroncoin/releases/download/%s/saffroncoin-binary-%s.tar.gz" % (SAFFRONCOIND_VER, SAFFRONCOIND_VER))
        runcmd("tar -C /tmp -zxvf /tmp/saffroncoind.tar.gz")
        runcmd("cp -af /tmp/saffroncoind /usr/bin")
        runcmd("rm -rf /tmp/saffroncoind.tar.gz /tmp/saffroncoind")
    
        #Do basic inital saffroncoin config (for both testnet and mainnet)
        runcmd("mkdir -p ~%s/.saffroncoin ~%s/.saffroncoin-testnet" % (USERNAME, USERNAME))
        if not os.path.exists(os.path.join(USER_HOMEDIR, '.saffroncoin', 'saffroncoin.conf')):
            runcmd(r"""bash -c 'echo -e "rpcuser=rpc\nrpcpassword=%s\nserver=1\ndaemon=1\ntxindex=1" > ~%s/.saffroncoin/saffroncoin.conf'""" % (
                backend_rpc_password, USERNAME))
        else: #grab the existing RPC password
            backend_rpc_password = subprocess.check_output(
                r"""bash -c "cat ~%s/.saffroncoin/saffroncoin.conf | sed -n 's/.*rpcpassword=\([^ \n]*\).*/\1/p'" """ % USERNAME, shell=True).strip().decode('utf-8')
        if not os.path.exists(os.path.join(USER_HOMEDIR, '.saffroncoin-testnet', 'saffroncoin.conf')):
            runcmd(r"""bash -c 'echo -e "rpcuser=rpc\nrpcpassword=%s\nserver=1\ndaemon=1\ntxindex=1\ntestnet=1" > ~%s/.saffroncoin-testnet/saffroncoin.conf'""" % (
                backend_rpc_password_testnet, USERNAME))
        else:
            backend_rpc_password_testnet = subprocess.check_output(
                r"""bash -c "cat ~%s/.saffroncoin-testnet/saffroncoin.conf | sed -n 's/.*rpcpassword=\([^ \n]*\).*/\1/p'" """
                % USERNAME, shell=True).strip().decode('utf-8')

        #remove pyrpcwallet startup scripts if present
        runcmd("rm -f /etc/service/pyrpcwallet /etc/service/pyrpcwallet-testnet")
        #install logrotate file
        runcmd("cp -dRf --preserve=mode %s/linux/logrotate/saffroncoind /etc/logrotate.d/saffroncoind" % dist_path)
        #set permissions
        runcmd("chown -R %s:%s ~%s/.saffroncoin ~%s/.saffroncoin-testnet" % (DAEMON_USERNAME, USERNAME, USERNAME, USERNAME))
        #set up runit startup scripts
        config_runit_for_service(dist_path, "saffroncoind", enabled=run_mode in ['m', 'b'])
        config_runit_for_service(dist_path, "saffroncoind-testnet", enabled=run_mode in ['t', 'b'])
    else:
        assert backend_rpc_mode == 'p'
        #checkout pyrpcwallet
        git_repo_clone("pyrpcwallet", "https://github.com/saffroncoin/pyrpcwallet.git",
            os.path.join(USER_HOMEDIR, "pyrpcwallet"), branch="master", for_user=run_as_user, hash="bdeef54ca74ebd5990350d62b57280d970af3c04")
        #remove saffroncoind startup scripts if present
        runcmd("rm -f /etc/service/saffroncoind /etc/service/saffroncoind-testnet")
        #change passwords
        modify_config(r'^RPC_PASSWORD=.*?$', 'RPC_PASSWORD=' + backend_rpc_password, '/etc/sv/pyrpcwallet/run')
        modify_config(r'^RPC_PASSWORD=.*?$', 'RPC_PASSWORD=' + backend_rpc_password_testnet, '/etc/sv/pyrpcwallet-testnet/run')
        #set permissions
        runcmd("mkdir -p ~%s/pyrpcwallet/log ~%s/pyrpcwallet/log-testnet" % (USERNAME, USERNAME))
        runcmd("chown -R %s:%s ~%s/pyrpcwallet ~%s/pyrpcwallet/log ~%s/pyrpcwallet/log-testnet" % (DAEMON_USERNAME, USERNAME, USERNAME, USERNAME, USERNAME))
        #set up runit startup scripts
        config_runit_for_service(dist_path, "pyrpcwallet", enabled=run_mode in ['m', 'b'])
        config_runit_for_service(dist_path, "pyrpcwallet-testnet", enabled=run_mode in ['t', 'b'])
    
    return backend_rpc_password, backend_rpc_password_testnet

def do_csfr_setup(role, run_as_user, docker, branch, base_path, dist_path, run_mode, backend_rpc_password,
backend_rpc_password_testnet, csfrd_public, csfrwallet_support_email):
    """Installs and configures csfrd and csfrblockd"""
    csfrd_rpc_password = '<PASSWORD>' if role == 'csfrd_only' and csfrd_public == 'y' else pass_generator()
    csfrd_rpc_password_testnet = '<PASSWORD>' if role == 'csfrd_only' and csfrd_public == 'y' else pass_generator()
    
    #Run setup.py (as the cSFR user, who runs it sudoed) to install and set up csfrd, csfrblockd
    # as -y is specified, this will auto install csfrblockd full node (mongo and redis) as well as setting
    # csfrd/csfrblockd to start up at startup for both mainnet and testnet (we will override this as necessary
    # based on run_mode later in this function)
    runcmd("~%s/csfrd_build/setup.py --noninteractive --branch=%s %s --with-testnet --for-user=%s %s" % (
        USERNAME, branch, "--with-bootstrap-db" if not docker else '', USERNAME, '--with-csfrblockd' if role != 'csfrd_only' else ''))
    runcmd("cd ~%s/csfrd_build && git config core.sharedRepository group && find ~%s/csfrd_build -type d -print0 | xargs -0 chmod g+s" % (
        USERNAME, USERNAME)) #to allow for group git actions 
    runcmd("chown -R %s:%s ~%s/csfrd_build" % (USERNAME, USERNAME, USERNAME)) #just in case
    runcmd("chmod -R u+rw,g+rw,o+r,o-w ~%s/csfrd_build" % USERNAME) #just in case
    
    #now change the csfrd directories to be owned by the csfrd user (and the csfr group),
    # so that the csfrd account can write to the database, saved image files (csfrblockd), log files, etc
    runcmd("mkdir -p ~%s/.config/csfrd ~%s/.config/csfrd-testnet" % (USERNAME, USERNAME))    
    runcmd("chown -R %s:%s ~%s/.config/csfrd ~%s/.config/csfrd-testnet" % (
        DAEMON_USERNAME, USERNAME, USERNAME, USERNAME))
    runcmd("sed -ri \"s/USER=%s/USER=%s/g\" /etc/sv/csfrd/run /etc/sv/csfrd-testnet/run" % (
        USERNAME, DAEMON_USERNAME))

    #modify the default stored saffroncoind passwords in csfrd.conf
    modify_cp_config(r'^(backend|saffroncoind)\-rpc\-password=.*?$',
        'backend-rpc-password=%s' % backend_rpc_password, config='csfrd', net='mainnet', for_user=USERNAME)
{'fraction_non_alphanumeric': 0.1359462371107577, 'fraction_numerical': 0.005130046683424819, 'mean_word_length': 3.4989614585737363, 'pattern_counts': {'":': 20, '<': 13, '<?xml version=': 0, '>': 17, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2937518', 'n_tokens_mistral': 7032, 'n_tokens_neox': 6938, 'n_words': 1637}