<commit_before>#!/usr/bin/env python3
from __future__ import print_function
import argparse, os, sys, time
from concurrent.futures import ThreadPoolExecutor

WORKERS = 3
SRC = os.path.split(os.path.realpath(__file__))[0]

# Only use sudo if not running as root already (this avoids having to install sudo)
import getpass
if getpass.getuser() != 'root':
    SUDO = "sudo "
else:
    SUDO = ""


def nice():
    try:
        import psutil  # not available by default (e.g., when building with docker)
        os.nice(10)
        psutil.Process(os.getpid()).ionice(ioclass=psutil.IOPRIO_CLASS_IDLE)
    except:
        print("WARNING: psutil not available so not re-nicing build of webapp")


def cmd(s, error=True):
    t0 = time.time()
    os.chdir(SRC)
    s = "umask 022; " + s
    print(s)
    if os.system(s) and error:
        sys.exit(1)
    elapsed = time.time() - t0
    print("TOTAL TIME: %.1f seconds" % elapsed)
    return elapsed


def pull():
    cmd("git submodule update --init")
    cmd("git pull")


def install_pyutil():
    cmd(SUDO + "pip2 install --upgrade ./smc_pyutil")


def install_sagews():
    if os.system('which sage') == 0:
        cmd("sage -pip install --upgrade ./smc_sagews")
    cmd(SUDO + "pip2 install --upgrade ./smc_sagews")  # as a fallback


def install_project():
    # unsafe-perm below is needed so can build C code as root

    # global install, hence no "npm ci" (!)
    for pkg in ['coffeescript', 'forever']:
        c = f"npm --loglevel=warn --unsafe-perm=true --progress=false install --upgrade {pkg} -g"
        cmd(SUDO + c)

    pkgs = [
        './smc-project',
        './smc-webapp',
        './smc-util-node',
        './smc-util',
    ]

    # npm ci for using pkg lock file
    def build_op(pkg):
        c = f"npm --loglevel=warn --unsafe-perm=true --progress=false ci {pkg} -g"
        return cmd(SUDO + c)

    with ThreadPoolExecutor(max_workers=WORKERS) as executor:
        total = sum(_ for _ in executor.map(build_op, pkgs))
        print(f"TOTAL PROJECT PKG BUILD TIME: {total:.1f}s")

    # UGLY; hard codes the path -- TODO: fix at some point.
    cmd("cd /usr/lib/node_modules/smc-project/jupyter && %s npm --loglevel=warn ci --unsafe-perm=true --progress=false --upgrade"
        % SUDO)

    # At least run typescript...
    # TODO: currently this errors somewhere in building something in node_modules in smc-webapp, since I can't get
    # tsconfig to just leave that code alone (since it is used?).  Hmm...
    cmd("cd /cocalc/src/smc-project; /cocalc/src/node_modules/.bin/tsc -p tsconfig.json",
        error=False)

    # Pre-compile everything to Javascript, so that loading is much faster and more efficient.<commit_msg>build docker: npm ci does not work with global. instead, we should probably build it and then symlink from /usr/lib/node_modules<commit_after>#!/usr/bin/env python3
from __future__ import print_function
import argparse, os, sys, time
from concurrent.futures import ThreadPoolExecutor

WORKERS = 3
SRC = os.path.split(os.path.realpath(__file__))[0]

# Only use sudo if not running as root already (this avoids having to install sudo)
import getpass
if getpass.getuser() != 'root':
    SUDO = "sudo "
else:
    SUDO = ""


def nice():
    try:
        import psutil  # not available by default (e.g., when building with docker)
        os.nice(10)
        psutil.Process(os.getpid()).ionice(ioclass=psutil.IOPRIO_CLASS_IDLE)
    except:
        print("WARNING: psutil not available so not re-nicing build of webapp")


def cmd(s, error=True):
    t0 = time.time()
    os.chdir(SRC)
    s = "umask 022; " + s
    print(s)
    if os.system(s) and error:
        sys.exit(1)
    elapsed = time.time() - t0
    print("TOTAL TIME: %.1f seconds" % elapsed)
    return elapsed


def pull():
    cmd("git submodule update --init")
    cmd("git pull")


def install_pyutil():
    cmd(SUDO + "pip2 install --upgrade ./smc_pyutil")


def install_sagews():
    if os.system('which sage') == 0:
        cmd("sage -pip install --upgrade ./smc_sagews")
    cmd(SUDO + "pip2 install --upgrade ./smc_sagews")  # as a fallback


def install_project():
    # unsafe-perm below is needed so can build C code as root

    # global install, hence no "npm ci" (!)
    for pkg in ['coffeescript', 'forever']:
        c = f"npm --loglevel=warn --unsafe-perm=true --progress=false install --upgrade {pkg} -g"
        cmd(SUDO + c)

    pkgs = [
        './smc-project',
        './smc-webapp',
        './smc-util-node',
        './smc-util',
    ]

    # TODO switch to use npm ci to install these (which doesn't exist for global installs, AFAIU)
    def build_op(pkg):
        c = f"npm --loglevel=warn --unsafe-perm=true --progress=false install {pkg} -g"
        return cmd(SUDO + c)

    with ThreadPoolExecutor(max_workers=WORKERS) as executor:
        total = sum(_ for _ in executor.map(build_op, pkgs))
        print(f"TOTAL PROJECT PKG BUILD TIME: {total:.1f}s")

    # UGLY; hard codes the path -- TODO: fix at some point.
    cmd("cd /usr/lib/node_modules/smc-project/jupyter && %s npm --loglevel=warn ci --unsafe-perm=true --progress=false --upgrade"
        % SUDO)

    # At least run typescript...
    # TODO: currently this errors somewhere in building something in node_modules in smc-webapp, since I can't get
    # tsconfig to just leave that code alone (since it is used?).  Hmm...
    cmd("cd /cocalc/src/smc-project; /cocalc/src/node_modules/.bin/tsc -p tsconfig.json",
        error=False)

    # Pre-compile everything to Javascript, so that loading is much faster and more efficient.
{'fraction_non_alphanumeric': 0.12757570327898227, 'fraction_numerical': 0.005733739473212686, 'mean_word_length': 2.753866845998655, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2524202', 'n_tokens_mistral': 1943, 'n_tokens_neox': 1842, 'n_words': 581}