<commit_before>#!/usr/bin/env python

###############################################################################
#
#    CoCalc: Collaborative Calculation in the Cloud
#
#    Copyright (C) 2016, Sagemath Inc.
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################


# used in naming streams -- changing this would break all existing data...
TO      = "-to-"

# appended to end of snapshot name to make it persistent (never automatically deleted)
PERSIST = "-persist"

TIMESTAMP_FORMAT = "%Y-%m-%d-%H%M%S"

# This is the quota for the .smc directory; must be
# significantly bigger than that directory, and hold user logs.
SMC_TEMPLATE_QUOTA = '1000m'

USER_SWAP_MB = 1000  # amount of swap users get

import errno, hashlib, json, math, os, platform, re, shutil, signal, socket, stat, sys, tempfile, time, uuid

from subprocess import Popen, PIPE

TIMESTAMP_FORMAT = "%Y-%m-%d-%H%M%S"
USER_SWAP_MB     = 1000  # amount of swap users get in addition to how much RAM they have.
PLATFORM         = platform.system().lower()
PROJECTS         = '/projects'

def quota_to_int(x):
    return int(math.ceil(x))

def log(s, *args):
    if args:
        try:
            s = str(s%args)
        except Exception, mesg:
            s = str(mesg) + str(s)
    sys.stderr.write(s+'\n')
    sys.stderr.flush()

def cmd(s, ignore_errors=False, verbose=2, timeout=None, stdout=True, stderr=True):
    if isinstance(s, list):
        s = [str(x) for x in s]
    if verbose >= 1:
        if isinstance(s, list):
            t = [x if len(x.split()) <=1  else "'%s'"%x for x in s]
            log(' '.join(t))
        else:
            log(s)
    t = time.time()

    mesg = "ERROR"
    if timeout:
        mesg = "TIMEOUT: running '%s' took more than %s seconds, so killed"%(s, timeout)
        def handle(*a):
            if ignore_errors:
                return mesg
            else:
                raise KeyboardInterrupt(mesg)
        signal.signal(signal.SIGALRM, handle)
        signal.alarm(timeout)
    try:
        out = Popen(s, stdin=PIPE, stdout=PIPE, stderr=PIPE, shell=not isinstance(s, list))
        x = out.stdout.read() + out.stderr.read()
        e = out.wait()  # this must be *after* the out.stdout.read(), etc. above or will hang when output large!
        if e:
            if ignore_errors:
                return (x + "ERROR").strip()
            else:
                raise RuntimeError(x)
        if verbose>=2:
            log("(%s seconds): %s", time.time()-t, x[:500])
        elif verbose >= 1:
            log("(%s seconds)", time.time()-t)
        return x.strip()
    except IOError:
        return mesg
    finally:
        if timeout:
            signal.signal(signal.SIGALRM, signal.SIG_IGN)  # cancel the alarm

def check_uuid(u):
    try:
        assert uuid.UUID(u).get_version() == 4
    except (AssertionError, ValueError), mesg:
        raise RuntimeError("invalid uuid (='%s')"%u)

def uid(project_id):
    # We take the sha-512 of the uuid just to make it harder to force a collision.  Thus even if a
    # user could somehow generate an account id of their choosing, this wouldn't help them get the
    # same uid as another user.
    # 2^31-1=max uid which works with FUSE and node (and Linux, which goes up to 2^32-2).
    # 2^29 was the biggest that seemed to work with Docker on my crostini pixelbook, so shrinking to that.
    # This is NOT used in production anymore, so should be fine.
    n = int(hashlib.sha512(project_id).hexdigest()[:8], 16)  # up to 2^32
    n //= 8  # up to 2^29  (floor div so will work with python3 too)
    return n if n>65537 else n+65537   # 65534 used by linux for user sync, etc.


def thread_map(callable, inputs):
    """
    Computing [callable(args) for args in inputs]
    in parallel using len(inputs) separate *threads*.

    If an exception is raised by any thread, a RuntimeError exception
    is instead raised.
    """
    log("Doing the following in parallel:\n%s", '\n'.join([str(x) for x in inputs]))
    from threading import Thread
    class F(Thread):
        def __init__(self, x):
            self._x = x
            Thread.__init__(self)
            self.start()
        def run(self):
            try:
                self.result = callable(self._x)
                self.fail = False
            except Exception, msg:
                self.result = msg
                self.fail = True
    results = [F(x) for x in inputs]
    for f in results: f.join()
    e = [f.result for f in results if f.fail]
    if e: raise RuntimeError(e)
    return [f.result for f in results]

class Project(object):
    def __init__(self,
                 project_id,          # v4 uuid string
                 dev           = False,  # if true, use special devel mode where everything run as same user (no sudo needed); totally insecure!
                 projects      = PROJECTS,
                 single        = False,
                 kucalc        = False
                ):
        self._dev    = dev
        self._single = single
        self._kucalc = kucalc
        if kucalc:
            projects = '/home'
        check_uuid(project_id)
        if not os.path.exists(projects):
            if self._dev:
                os.makedirs(projects)
            else:
                raise RuntimeError("mount point %s doesn't exist"%projects)
        self.project_id    = project_id
        self._projects     = projects
        if self._kucalc:
            self.project_path = os.environ['HOME']
        else:
            self.project_path = os.path.join(self._projects, project_id)
        self.smc_path      = os.path.join(self.project_path, '.smc')
        self.forever_path  = os.path.join(self.project_path, '.forever')
        self.uid           = uid(self.project_id)
        self.username      = self.project_id.replace('-','')
        self.open_fail_file = os.path.join(self.project_path, '.sagemathcloud-open-failed')

    def _log(self, name=""):
        def f(s='', *args):
            log("Project(project_id=%s).%s(...): "%(self.project_id, name) + s, *args)
        return f

    def cmd(self, *args, **kwds):
        log("Project(project_id=%s).cmd(...): ", self.project_id)
        return cmd(*args, **kwds)

    ###
    # Users Management
    ###

    def create_user(self, login_shell='/bin/bash'):
        if not os.path.exists(self.project_path):
            os.makedirs(self.project_path)
        # We used to only chown if just made; it's recursive and can be very expensive in general!
        # However, since we're changing the uid mapping, we really do have to do this every time,
        # or we break existing installs.
        self.chown(self.project_path)
        if self._dev:
            return
        cmd(['/usr/sbin/groupadd', '-g', self.uid, '-o', self.username], ignore_errors=True)
        cmd(['/usr/sbin/useradd',  '-u', self.uid, '-g', self.uid, '-o', self.username,
                  '-d', self.project_path, '-s', login_shell], ignore_errors=True)

    def delete_user(self):
        if self._dev:
            return
        cmd(['/usr/sbin/userdel',  self.username], ignore_errors=True)
        cmd(['/usr/sbin/groupdel', self.username], ignore_errors=True)
        if os.path.exists('/etc/cgrules.conf'):
            c = open("/etc/cgrules.conf").read()
            i = c.find(self.username)
            if i != -1:
                j = c[i:].find('\n')
                if j == -1:
                    j = len(c)
                else:
                    j += i
                open("/etc/cgrules.conf",'w').write(c[:i]+c[j+1:])

    def pids(self):
        return [int(x) for x in self.cmd(['pgrep', '-u', self.uid], ignore_errors=True).replace('ERROR','').split()]

    def num_procs(self):
        return len(self.pids())

    def killall(self, grace_s=0.5, max_tries=15):
        log = self._log('killall')
        if self._dev:
            self.dev_env()
            os.chdir(self.project_path)
            self.cmd("smc-local-hub stop")
            self.cmd("smc-console-server stop")
            self.cmd("smc-sage-server stop")
            return

        log("killing all processes by user with id %s"%self.uid)
        # we use both kill and pkill -- pkill seems better in theory, but I've definitely seen it get ignored.
        for i in range(max_tries):
            n = self.num_procs()
            log("kill attempt left %s procs"%n)
            if n == 0:
                return
            self.cmd(['/usr/bin/killall', '-u', self.username], ignore_errors=True)
            self.cmd(['/usr/bin/pkill', '-u', self.uid], ignore_errors=True)
            time.sleep(grace_s)
            self.cmd(['/usr/bin/killall', '-9', '-u', self.username], ignore_errors=True)
            self.cmd(['/usr/bin/pkill', '-9', '-u', self.uid], ignore_errors=True)
        log("WARNING: failed to kill all procs after %s tries"%max_tries)

    def chown(self, path, recursive=True):
        if self._dev:
            return
        if recursive:
            cmd(["chown", "%s:%s"%(self.uid, self.uid), '-R', path])
        else:
            cmd(["chown", "%s:%s"%(self.uid, self.uid), path])

    def ensure_file_exists(self, src, target):
        target = os.path.abspath(target)
        if not os.path.exists(target):
            self.makedirs(os.path.split(target)[0])
            shutil.copyfile(src, target)
            if USERNAME == "root":
                os.chown(target, self.uid, self.uid)

    def create_smc_path(self):
        if not os.path.exists(self.smc_path):
            os.makedirs(self.smc_path)
        self.chown(self.smc_path)
        self.ensure_conf_files_exist()

    def ensure_conf_files_exist(self):
        for filename in ['bashrc', 'bash_profile']:
            target = os.path.join(self.project_path, '.' + filename)
            if not os.path.exists(target):
                source = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'templates', PLATFORM, filename)
                if os.path.exists(source):
                    shutil.copyfile(source, target)
                    if not self._dev:
                        os.chown(target, self.uid, self.uid)

    def remove_forever_path(self):
        if os.path.exists(self.forever_path):
            shutil.rmtree(self.forever_path, ignore_errors=True)

    def remove_smc_path(self):
        # do our best to remove the smc path
        if os.path.exists(self.smc_path):
            shutil.rmtree(self.smc_path, ignore_errors=True)

    def disk_quota(self, quota=0):  # quota in megabytes
        try:
            quota = quota_to_int(quota)
            # requires quotas to be setup as explained nicely at
            # https://www.digitalocean.com/community/tutorials/how-to-enable-user-and-group-quotas
            # and https://askubuntu.com/questions/109585/quota-format-not-supported-in-kernel/165298#165298
            # This sets the quota on all mounted filesystems:
            cmd(['setquota', '-u', self.username, quota*1000, quota*1200, 1000000, 1100000, '-a'])
        except Exception, mesg:
            log("WARNING -- quota failure %s", mesg)

    def compute_quota(self, cores, memory, cpu_shares):
        """
        cores      - number of cores (float)
        memory     - megabytes of RAM (int)
        cpu_shares - determines relative share of cpu (e.g., 256=most users)
        """
        if self._dev:
            return
        cfs_quota = int(100000*cores)

        group = "memory,cpu:%s"%self.username
        try:
            self.cmd(["cgcreate", "-g", group])
        except:
            if os.system("cgcreate") != 0:
                # cgroups not installed
                return
            else:
                raise
        if memory:
            memory = quota_to_int(memory)
            open("/sys/fs/cgroup/memory/%s/memory.limit_in_bytes"%self.username,'w').write("%sM"%memory)
            open("/sys/fs/cgroup/memory/%s/memory.memsw.limit_in_bytes"%self.username,'w').write("%sM"%(USER_SWAP_MB + memory))
        if cpu_shares:
            cpu_shares = quota_to_int(cpu_shares)
            open("/sys/fs/cgroup/cpu/%s/cpu.shares"%self.username,'w').write(str(cpu_shares))
        if cfs_quota:
            open("/sys/fs/cgroup/cpu/%s/cpu.cfs_quota_us"%self.username,'w').write(str(cfs_quota))

        z = "\n%s  cpu,memory  %s\n"%(self.username, self.username)
        cur = open("/etc/cgrules.conf").read() if os.path.exists("/etc/cgrules.conf") else ''

        if z not in cur:
            open("/etc/cgrules.conf",'a').write(z)
        try:
            pids = self.cmd("ps -o pid -u %s"%self.username, ignore_errors=False).split()[1:]
            self.cmd(["cgclassify", "-g", group] + pids, ignore_errors=True)
            # ignore cgclassify errors, since processes come and go, etc.
        except:
            pass  # ps returns an error code if there are NO processes at all

    def cgclassify(self):
        try:
            pids = self.cmd("ps -o pid -u %s"%self.username, ignore_errors=False).split()[1:]
            self.cmd(["cgclassify"] + pids, ignore_errors=True)
            # ignore cgclassify errors, since processes come and go, etc.":
        except:
            # ps returns an error code if there are NO processes at all (a common condition).
            pids = []

    def create_project_path(self):
        if not os.path.exists(self.project_path):
            os.makedirs(self.project_path)
            if not self._dev:
                os.chown(self.project_path, self.uid, self.uid)

    def remove_snapshots_path(self):
        """
        Remove the ~/.snapshots path
        """
        p = os.path.join(self.project_path, '.snapshots')
        if os.path.exists(p):
            shutil.rmtree(p, ignore_errors=True)

    def ensure_bashrc(self):
        # ensure .bashrc has certain properties
        bashrc = os.path.join(self.project_path, '.bashrc')
        if not os.path.exists(bashrc):
            return
        s = open(bashrc).read()
        changed = False
        if '.sagemathcloud' in s:
            s = '\n'.join([y for y in s.splitlines() if '.sagemathcloud' not in y])
            changed = True
        if 'SAGE_ATLAS_LIB' not in s:
            s += '\nexport SAGE_ATLAS_LIB=/usr/lib/   # do not build ATLAS\n\n'
            changed = True
        if '$HOME/bin:$HOME/.local/bin' not in s:
            s += '\nexport PATH=$HOME/bin:$HOME/.local/bin:$PATH\n\n'
            changed = True
        if changed:
            open(bashrc,'w').write(s)

    def dev_env(self):
        os.environ['PATH'] = "{salvus_root}/smc-project/bin:{salvus_root}/smc_pyutil/smc_pyutil:{path}".format(
                                    salvus_root=os.environ['SALVUS_ROOT'], path=os.environ['PATH'])
        os.environ['PYTHONPATH'] = "{home}/.local/lib/python2.7/site-packages".format(home=os.environ['HOME'])
        os.environ['SMC_LOCAL_HUB_HOME'] = self.project_path
        os.environ['SMC_HOST'] = 'localhost'
        os.environ['SMC'] = self.smc_path

        # for development, the raw server, jupyter, etc., have to listen on localhost since that is where
        # the hub is running
        os.environ['SMC_PROXY_HOST'] = 'localhost'

    def start(self, cores, memory, cpu_shares, base_url):
        self.remove_smc_path()   # start can be prevented by massive logs in ~/.smc; if project not stopped via stop, then they will still be there.
        self.ensure_bashrc()
        self.remove_forever_path()    # probably not needed anymore
        self.remove_snapshots_path()
        self.create_user()
        self.create_smc_path()
        self.chown(self.project_path, False) # Sometimes /projects/[project_id] doesn't have group/owner equal to that of the project.

        os.environ['SMC_BASE_URL'] = base_url

        # When running CoCalc inside of CoCalc, this env variable
        # could cause trouble, e.g., confusing the sagews server.
        if 'COCALC_SECRET_TOKEN' in os.environ:
            del os.environ['COCALC_SECRET_TOKEN']
        if self._dev:
            self.dev_env()
            os.chdir(self.project_path)
            self.cmd("smc-local-hub start")
            def started():
                return os.path.exists("%s/local_hub/local_hub.port"%self.smc_path)
            i=0
            while not started():
                time.sleep(0.1)
                i += 1
                sys.stdout.flush()
                if i >= 100:
                    return
            return

        pid = os.fork()
        if pid == 0:
            try:
                os.nice(-os.nice(0))  # Reset nice-ness to 0
                os.setgroups([])      # Drops other groups, like root or sudoers
                os.setsid()           # Make it a session leader
                os.setgid(self.uid)
                os.setuid(self.uid)

                try:
                    # Fork a second child and exit immediately to prevent zombies.  This
                    # causes the second child process to be orphaned, making the init
                    # process responsible for its cleanup.
                    pid = os.fork()
                except OSError, e:
                    raise Exception, "%s [%d]" % (e.strerror, e.errno)

                if pid == 0:
                    os.environ['HOME'] = self.project_path
                    os.environ['SMC'] = self.smc_path
                    os.environ['USER'] = os.environ['USERNAME'] =  os.environ['LOGNAME'] = self.username
                    os.environ['MAIL'] = '/var/mail/%s'%self.username
                    # Needed to read code from system-wide installed location.
                    os.environ['NODE_PATH'] = '/cocalc/src/node_modules/smc-util:/cocalc/src/node_modules:/cocalc/src:/cocalc/src/smc-project/node_modules::/cocalc/src/smc-webapp/node_modules'
                    if self._single:
                        # In single-machine mode, everything is on localhost.
                        os.environ['SMC_HOST'] = 'localhost'
                    del os.environ['SUDO_COMMAND']; del os.environ['SUDO_UID']; del os.environ['SUDO_GID']; del os.environ['SUDO_USER']
                    os.chdir(self.project_path)
                    self.cmd("smc-start")
                else:
                    os._exit(0)
            finally:
                os._exit(0)
        else:
            os.waitpid(pid, 0)<commit_msg>define more env variables in private docker container.<commit_after>#!/usr/bin/env python

###############################################################################
#
#    CoCalc: Collaborative Calculation in the Cloud
#
#    Copyright (C) 2016, Sagemath Inc.
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################


# used in naming streams -- changing this would break all existing data...
TO      = "-to-"

# appended to end of snapshot name to make it persistent (never automatically deleted)
PERSIST = "-persist"

TIMESTAMP_FORMAT = "%Y-%m-%d-%H%M%S"

# This is the quota for the .smc directory; must be
# significantly bigger than that directory, and hold user logs.
SMC_TEMPLATE_QUOTA = '1000m'

USER_SWAP_MB = 1000  # amount of swap users get

import errno, hashlib, json, math, os, platform, re, shutil, signal, socket, stat, sys, tempfile, time, uuid

from subprocess import Popen, PIPE

TIMESTAMP_FORMAT = "%Y-%m-%d-%H%M%S"
USER_SWAP_MB     = 1000  # amount of swap users get in addition to how much RAM they have.
PLATFORM         = platform.system().lower()
PROJECTS         = '/projects'

def quota_to_int(x):
    return int(math.ceil(x))

def log(s, *args):
    if args:
        try:
            s = str(s%args)
        except Exception, mesg:
            s = str(mesg) + str(s)
    sys.stderr.write(s+'\n')
    sys.stderr.flush()

def cmd(s, ignore_errors=False, verbose=2, timeout=None, stdout=True, stderr=True):
    if isinstance(s, list):
        s = [str(x) for x in s]
    if verbose >= 1:
        if isinstance(s, list):
            t = [x if len(x.split()) <=1  else "'%s'"%x for x in s]
            log(' '.join(t))
        else:
            log(s)
    t = time.time()

    mesg = "ERROR"
    if timeout:
        mesg = "TIMEOUT: running '%s' took more than %s seconds, so killed"%(s, timeout)
        def handle(*a):
            if ignore_errors:
                return mesg
            else:
                raise KeyboardInterrupt(mesg)
        signal.signal(signal.SIGALRM, handle)
        signal.alarm(timeout)
    try:
        out = Popen(s, stdin=PIPE, stdout=PIPE, stderr=PIPE, shell=not isinstance(s, list))
        x = out.stdout.read() + out.stderr.read()
        e = out.wait()  # this must be *after* the out.stdout.read(), etc. above or will hang when output large!
        if e:
            if ignore_errors:
                return (x + "ERROR").strip()
            else:
                raise RuntimeError(x)
        if verbose>=2:
            log("(%s seconds): %s", time.time()-t, x[:500])
        elif verbose >= 1:
            log("(%s seconds)", time.time()-t)
        return x.strip()
    except IOError:
        return mesg
    finally:
        if timeout:
            signal.signal(signal.SIGALRM, signal.SIG_IGN)  # cancel the alarm

def check_uuid(u):
    try:
        assert uuid.UUID(u).get_version() == 4
    except (AssertionError, ValueError), mesg:
        raise RuntimeError("invalid uuid (='%s')"%u)

def uid(project_id):
    # We take the sha-512 of the uuid just to make it harder to force a collision.  Thus even if a
    # user could somehow generate an account id of their choosing, this wouldn't help them get the
    # same uid as another user.
    # 2^31-1=max uid which works with FUSE and node (and Linux, which goes up to 2^32-2).
    # 2^29 was the biggest that seemed to work with Docker on my crostini pixelbook, so shrinking to that.
    # This is NOT used in production anymore, so should be fine.
    n = int(hashlib.sha512(project_id).hexdigest()[:8], 16)  # up to 2^32
    n //= 8  # up to 2^29  (floor div so will work with python3 too)
    return n if n>65537 else n+65537   # 65534 used by linux for user sync, etc.


def thread_map(callable, inputs):
    """
    Computing [callable(args) for args in inputs]
    in parallel using len(inputs) separate *threads*.

    If an exception is raised by any thread, a RuntimeError exception
    is instead raised.
    """
    log("Doing the following in parallel:\n%s", '\n'.join([str(x) for x in inputs]))
    from threading import Thread
    class F(Thread):
        def __init__(self, x):
            self._x = x
            Thread.__init__(self)
            self.start()
        def run(self):
            try:
                self.result = callable(self._x)
                self.fail = False
            except Exception, msg:
                self.result = msg
                self.fail = True
    results = [F(x) for x in inputs]
    for f in results: f.join()
    e = [f.result for f in results if f.fail]
    if e: raise RuntimeError(e)
    return [f.result for f in results]

class Project(object):
    def __init__(self,
                 project_id,          # v4 uuid string
                 dev           = False,  # if true, use special devel mode where everything run as same user (no sudo needed); totally insecure!
                 projects      = PROJECTS,
                 single        = False,
                 kucalc        = False
                ):
        self._dev    = dev
        self._single = single
        self._kucalc = kucalc
        if kucalc:
            projects = '/home'
        check_uuid(project_id)
        if not os.path.exists(projects):
            if self._dev:
                os.makedirs(projects)
            else:
                raise RuntimeError("mount point %s doesn't exist"%projects)
        self.project_id    = project_id
        self._projects     = projects
        if self._kucalc:
            self.project_path = os.environ['HOME']
        else:
            self.project_path = os.path.join(self._projects, project_id)
        self.smc_path      = os.path.join(self.project_path, '.smc')
        self.forever_path  = os.path.join(self.project_path, '.forever')
        self.uid           = uid(self.project_id)
        self.username      = self.project_id.replace('-','')
        self.open_fail_file = os.path.join(self.project_path, '.sagemathcloud-open-failed')

    def _log(self, name=""):
        def f(s='', *args):
            log("Project(project_id=%s).%s(...): "%(self.project_id, name) + s, *args)
        return f

    def cmd(self, *args, **kwds):
        log("Project(project_id=%s).cmd(...): ", self.project_id)
        return cmd(*args, **kwds)

    ###
    # Users Management
    ###

    def create_user(self, login_shell='/bin/bash'):
        if not os.path.exists(self.project_path):
            os.makedirs(self.project_path)
        # We used to only chown if just made; it's recursive and can be very expensive in general!
        # However, since we're changing the uid mapping, we really do have to do this every time,
        # or we break existing installs.
        self.chown(self.project_path)
        if self._dev:
            return
        cmd(['/usr/sbin/groupadd', '-g', self.uid, '-o', self.username], ignore_errors=True)
        cmd(['/usr/sbin/useradd',  '-u', self.uid, '-g', self.uid, '-o', self.username,
                  '-d', self.project_path, '-s', login_shell], ignore_errors=True)

    def delete_user(self):
        if self._dev:
            return
        cmd(['/usr/sbin/userdel',  self.username], ignore_errors=True)
        cmd(['/usr/sbin/groupdel', self.username], ignore_errors=True)
        if os.path.exists('/etc/cgrules.conf'):
            c = open("/etc/cgrules.conf").read()
            i = c.find(self.username)
            if i != -1:
                j = c[i:].find('\n')
                if j == -1:
                    j = len(c)
                else:
                    j += i
                open("/etc/cgrules.conf",'w').write(c[:i]+c[j+1:])

    def pids(self):
        return [int(x) for x in self.cmd(['pgrep', '-u', self.uid], ignore_errors=True).replace('ERROR','').split()]

    def num_procs(self):
        return len(self.pids())

    def killall(self, grace_s=0.5, max_tries=15):
        log = self._log('killall')
        if self._dev:
            self.dev_env()
            os.chdir(self.project_path)
            self.cmd("smc-local-hub stop")
            self.cmd("smc-console-server stop")
            self.cmd("smc-sage-server stop")
            return

        log("killing all processes by user with id %s"%self.uid)
        # we use both kill and pkill -- pkill seems better in theory, but I've definitely seen it get ignored.
        for i in range(max_tries):
            n = self.num_procs()
            log("kill attempt left %s procs"%n)
            if n == 0:
                return
            self.cmd(['/usr/bin/killall', '-u', self.username], ignore_errors=True)
            self.cmd(['/usr/bin/pkill', '-u', self.uid], ignore_errors=True)
            time.sleep(grace_s)
            self.cmd(['/usr/bin/killall', '-9', '-u', self.username], ignore_errors=True)
            self.cmd(['/usr/bin/pkill', '-9', '-u', self.uid], ignore_errors=True)
        log("WARNING: failed to kill all procs after %s tries"%max_tries)

    def chown(self, path, recursive=True):
        if self._dev:
            return
        if recursive:
            cmd(["chown", "%s:%s"%(self.uid, self.uid), '-R', path])
        else:
            cmd(["chown", "%s:%s"%(self.uid, self.uid), path])

    def ensure_file_exists(self, src, target):
        target = os.path.abspath(target)
        if not os.path.exists(target):
            self.makedirs(os.path.split(target)[0])
            shutil.copyfile(src, target)
            if USERNAME == "root":
                os.chown(target, self.uid, self.uid)

    def create_smc_path(self):
        if not os.path.exists(self.smc_path):
            os.makedirs(self.smc_path)
        self.chown(self.smc_path)
        self.ensure_conf_files_exist()

    def ensure_conf_files_exist(self):
        for filename in ['bashrc', 'bash_profile']:
            target = os.path.join(self.project_path, '.' + filename)
            if not os.path.exists(target):
                source = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'templates', PLATFORM, filename)
                if os.path.exists(source):
                    shutil.copyfile(source, target)
                    if not self._dev:
                        os.chown(target, self.uid, self.uid)

    def remove_forever_path(self):
        if os.path.exists(self.forever_path):
            shutil.rmtree(self.forever_path, ignore_errors=True)

    def remove_smc_path(self):
        # do our best to remove the smc path
        if os.path.exists(self.smc_path):
            shutil.rmtree(self.smc_path, ignore_errors=True)

    def disk_quota(self, quota=0):  # quota in megabytes
        try:
            quota = quota_to_int(quota)
            # requires quotas to be setup as explained nicely at
            # https://www.digitalocean.com/community/tutorials/how-to-enable-user-and-group-quotas
            # and https://askubuntu.com/questions/109585/quota-format-not-supported-in-kernel/165298#165298
            # This sets the quota on all mounted filesystems:
            cmd(['setquota', '-u', self.username, quota*1000, quota*1200, 1000000, 1100000, '-a'])
        except Exception, mesg:
            log("WARNING -- quota failure %s", mesg)

    def compute_quota(self, cores, memory, cpu_shares):
        """
        cores      - number of cores (float)
        memory     - megabytes of RAM (int)
        cpu_shares - determines relative share of cpu (e.g., 256=most users)
        """
        if self._dev:
            return
        cfs_quota = int(100000*cores)

        group = "memory,cpu:%s"%self.username
        try:
            self.cmd(["cgcreate", "-g", group])
        except:
            if os.system("cgcreate") != 0:
                # cgroups not installed
                return
            else:
                raise
        if memory:
            memory = quota_to_int(memory)
            open("/sys/fs/cgroup/memory/%s/memory.limit_in_bytes"%self.username,'w').write("%sM"%memory)
            open("/sys/fs/cgroup/memory/%s/memory.memsw.limit_in_bytes"%self.username,'w').write("%sM"%(USER_SWAP_MB + memory))
        if cpu_shares:
            cpu_shares = quota_to_int(cpu_shares)
            open("/sys/fs/cgroup/cpu/%s/cpu.shares"%self.username,'w').write(str(cpu_shares))
        if cfs_quota:
            open("/sys/fs/cgroup/cpu/%s/cpu.cfs_quota_us"%self.username,'w').write(str(cfs_quota))

        z = "\n%s  cpu,memory  %s\n"%(self.username, self.username)
        cur = open("/etc/cgrules.conf").read() if os.path.exists("/etc/cgrules.conf") else ''

        if z not in cur:
            open("/etc/cgrules.conf",'a').write(z)
        try:
            pids = self.cmd("ps -o pid -u %s"%self.username, ignore_errors=False).split()[1:]
            self.cmd(["cgclassify", "-g", group] + pids, ignore_errors=True)
            # ignore cgclassify errors, since processes come and go, etc.
        except:
            pass  # ps returns an error code if there are NO processes at all

    def cgclassify(self):
        try:
            pids = self.cmd("ps -o pid -u %s"%self.username, ignore_errors=False).split()[1:]
            self.cmd(["cgclassify"] + pids, ignore_errors=True)
            # ignore cgclassify errors, since processes come and go, etc.":
        except:
            # ps returns an error code if there are NO processes at all (a common condition).
            pids = []

    def create_project_path(self):
        if not os.path.exists(self.project_path):
            os.makedirs(self.project_path)
            if not self._dev:
                os.chown(self.project_path, self.uid, self.uid)

    def remove_snapshots_path(self):
        """
        Remove the ~/.snapshots path
        """
        p = os.path.join(self.project_path, '.snapshots')
        if os.path.exists(p):
            shutil.rmtree(p, ignore_errors=True)

    def ensure_bashrc(self):
        # ensure .bashrc has certain properties
        bashrc = os.path.join(self.project_path, '.bashrc')
        if not os.path.exists(bashrc):
            return
        s = open(bashrc).read()
        changed = False
        if '.sagemathcloud' in s:
            s = '\n'.join([y for y in s.splitlines() if '.sagemathcloud' not in y])
            changed = True
        if 'SAGE_ATLAS_LIB' not in s:
            s += '\nexport SAGE_ATLAS_LIB=/usr/lib/   # do not build ATLAS\n\n'
            changed = True
        if '$HOME/bin:$HOME/.local/bin' not in s:
            s += '\nexport PATH=$HOME/bin:$HOME/.local/bin:$PATH\n\n'
            changed = True
        if changed:
            open(bashrc,'w').write(s)

    def dev_env(self):
        os.environ['PATH'] = "{salvus_root}/smc-project/bin:{salvus_root}/smc_pyutil/smc_pyutil:{path}".format(
                                    salvus_root=os.environ['SALVUS_ROOT'], path=os.environ['PATH'])
        os.environ['PYTHONPATH'] = "{home}/.local/lib/python2.7/site-packages".format(home=os.environ['HOME'])
        os.environ['SMC_LOCAL_HUB_HOME'] = self.project_path
        os.environ['SMC_HOST'] = 'localhost'
        os.environ['SMC'] = self.smc_path

        # for development, the raw server, jupyter, etc., have to listen on localhost since that is where
        # the hub is running
        os.environ['SMC_PROXY_HOST'] = 'localhost'

    def start(self, cores, memory, cpu_shares, base_url):
        self.remove_smc_path()   # start can be prevented by massive logs in ~/.smc; if project not stopped via stop, then they will still be there.
        self.ensure_bashrc()
        self.remove_forever_path()    # probably not needed anymore
        self.remove_snapshots_path()
        self.create_user()
        self.create_smc_path()
        self.chown(self.project_path, False) # Sometimes /projects/[project_id] doesn't have group/owner equal to that of the project.

        os.environ['SMC_BASE_URL'] = base_url

        # When running CoCalc inside of CoCalc, this env variable
        # could cause trouble, e.g., confusing the sagews server.
        if 'COCALC_SECRET_TOKEN' in os.environ:
            del os.environ['COCALC_SECRET_TOKEN']
        if self._dev:
            self.dev_env()
            os.chdir(self.project_path)
            self.cmd("smc-local-hub start")
            def started():
                return os.path.exists("%s/local_hub/local_hub.port"%self.smc_path)
            i=0
            while not started():
                time.sleep(0.1)
                i += 1
                sys.stdout.flush()
                if i >= 100:
                    return
            return

        pid = os.fork()
        if pid == 0:
            try:
                os.nice(-os.nice(0))  # Reset nice-ness to 0
                os.setgroups([])      # Drops other groups, like root or sudoers
                os.setsid()           # Make it a session leader
                os.setgid(self.uid)
                os.setuid(self.uid)

                try:
                    # Fork a second child and exit immediately to prevent zombies.  This
                    # causes the second child process to be orphaned, making the init
                    # process responsible for its cleanup.
                    pid = os.fork()
                except OSError, e:
                    raise Exception, "%s [%d]" % (e.strerror, e.errno)

                if pid == 0:
                    os.environ['HOME'] = self.project_path
                    os.environ['SMC'] = self.smc_path
                    os.environ['COCALC_SECRET_TOKEN'] = os.path.join(self.smc_path, 'secret_token')
                    os.environ['COCALC_PROJECT_ID'] = self.project_id
                    os.environ['USER'] = os.environ['USERNAME'] =  os.environ['LOGNAME'] = os.environ['COCALC_USERNAME'] = self.username
                    os.environ['MAIL'] = '/var/mail/%s'%self.username
                    # Needed to read code from system-wide installed location.
                    os.environ['NODE_PATH'] = '/cocalc/src/node_modules/smc-util:/cocalc/src/node_modules:/cocalc/src:/cocalc/src/smc-project/node_modules::/cocalc/src/smc-webapp/node_modules'
                    if self._single:
                        # In single-machine mode, everything is on localhost.
                        os.environ['SMC_HOST'] = 'localhost'
                    del os.environ['SUDO_COMMAND']; del os.environ['SUDO_UID']; del os.environ['SUDO_GID']; del os.environ['SUDO_USER']
                    os.chdir(self.project_path)
                    self.cmd("smc-start")
                else:
                    os._exit(0)
            finally:
                os._exit(0)
        else:
            os.waitpid(pid, 0)
{'fraction_non_alphanumeric': 0.13041772383667705, 'fraction_numerical': 0.007847035128560591, 'mean_word_length': 1.9931887575354263, 'pattern_counts': {'":': 4, '<': 7, '<?xml version=': 0, '>': 15, 'https://': 4, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2259788', 'n_tokens_mistral': 12039, 'n_tokens_neox': 11593, 'n_words': 3062}