<commit_before># -*- coding: utf-8 -*-
"""Runs the server in uwsgi or http modes.

Also supports starting nginx proxy.

:copyright: Copyright (c) 2015 RadiaSoft LLC.  All Rights Reserved.
:license: http://www.apache.org/licenses/LICENSE-2.0.html
"""
from __future__ import absolute_import, division, print_function
from pykern import pkcli
from pykern import pkcollections
from pykern import pkconfig
from pykern import pkio
from pykern import pkjinja
from pykern import pksubprocess
from pykern.pkdebug import pkdc, pkdexc, pkdp, pkdlog
import distutils
import errno
import inspect
import os
import py
import re
import signal
import sirepo.srdb
import socket
import subprocess
import sys


def celery():
    """Start celery"""
    assert pkconfig.channel_in('dev')
    import celery.bin.celery
    import sirepo.celery_tasks
    run_dir = _run_dir().join('celery').ensure(dir=True)
    with pkio.save_chdir(run_dir):
        celery.bin.celery.main(argv=[
            'celery',
            'worker',
            '--app=sirepo.celery_tasks',
            '--no-color',
            '-Ofair',
            '--queue=' + ','.join(sirepo.celery_tasks.QUEUE_NAMES),
        ])


def flower():
    """Start flower"""
    from flower import command
    assert pkconfig.channel_in('dev')
    run_dir = _run_dir().join('flower').ensure(dir=True)
    with pkio.save_chdir(run_dir):
        command.FlowerCommand().execute_from_commandline([
            'flower',
            '--address=' + cfg.ip,
            '--app=sirepo.celery_tasks',
            '--no-color',
            '--persistent',
        ])


def flask():
    from sirepo import server

    use_reloader = pkconfig.channel_in('dev')
    app = server.init(use_reloader=use_reloader)
    # avoid WARNING: Do not use the development server in a production environment.
    app.env = 'development'
    app.run(
        host=cfg.ip,
        port=cfg.port,
        threaded=True,
        use_reloader=use_reloader,
    )


def http(driver='local', nersc_proxy=None, nersc_user=None, sbatch_host=None):
    """Starts the Flask server and job_supervisor.

    Used for development only.

    Args:
        driver (string): The driver type to enable (one of local, docker, sbatch, nersc)
        nersc_proxy (string): A proxy nersc can use to reach the supervisor
    # TODO(e-carlin): talk with rn
        nersc_user(string): A nersc user ??
        sbatch_host (string): A host to ssh into to start sbatch jobs
    # TODO(e-carlin):
    """
    def _env(py_version):
        e = os.environ.copy()
        e.update(
            PYENV_VERSION=py_version,
            PYKERN_PKDEBUG_WANT_PID_TIME='1',
            SIREPO_AUTH_EMAIL_FROM_EMAIL='<EMAIL>',
            SIREPO_AUTH_EMAIL_FROM_NAME='RadiaSoft Support',
            SIREPO_AUTH_EMAIL_SMTP_PASSWORD='<PASSWORD>',
            # POSIT: same as sirepo.auth.email._DEV_SMTP_SERVER
            SIREPO_AUTH_EMAIL_SMTP_SERVER='dev',
            SIREPO_AUTH_EMAIL_SMTP_USER='n/a',
            SIREPO_AUTH_METHODS='email:guest',
            SIREPO_MPI_CORES='2',
        )
        {
            n: f for n, f in inspect.currentframe().f_back.f_locals.items()
            if callable(f)
        }['_env_' + driver](e)

        if 'sbatch' in e['SIREPO_JOB_DRIVER_MODULES']:
            h = e['SIREPO_JOB_DRIVER_SBATCH_HOST']
            m = re.search(r'^{}.+$'.format(h), pkio.read_text('~/.ssh/known_hosts'), re.MULTILINE)
            assert m, \
                (
                    'you need to get the host key in ~/.ssh/known_hosts'
                    ' run: `ssh {} true`'.format(h)
                )
            x = m.group(0)
            e.update(SIREPO_JOB_DRIVER_SBATCH_HOST_KEY=x)
        return e

    def _env_docker(env):
        # TODO(e-carlin): py3 has shutil.which()
        assert distutils.spawn.find_executable('docker') is not None, \
            'docker not installed:  You need to run `radia_run redhat-docker`'
        i = 'radiasoft/sirepo:dev'
        try:
            subprocess.check_output(('docker', 'image', 'inspect', i))
        except subprocess.CalledProcessError:
            pkdlog('docker image {} not installed. Pulling...')
            assert subprocess.call(('docker', 'image', 'pull', i)) == 0, \
                'docker image pull failed'

        env.update(
            SIREPO_JOB_DRIVER_MODULES='docker',
        )

    def _env_local(env):
        env.update(
            SIREPO_JOB_DRIVER_MODULES='local',
        )
        assert 'SIREPO_JOB_DRIVER_MODULES' in env

    def _env_nersc(env):
        # TODO(e-carlin): Shouldn't nersc_user always be nagler?
        assert nersc_proxy and nersc_user, \
            'You need to supply a nersc_proxy and a nserc_user'
        env.update(
            SIREPO_JOB_DRIVER_MODULES='local:sbatch',
            SIREPO_JOB_DRIVER_SBATCH_HOST='cori.nersc.gov',
            SIREPO_JOB_DRIVER_SBATCH_SHIFTER_IMAGE='radiasoft/sirepo:sbatch',
            # TODO(e-carlin): /global/homes/??? NERSC is down so I can't ssh in and figure it out
            # /global/homes/${3::1}/$3/.pyenv/versions/py3/bin/sirepo
            SIREPO_JOB_DRIVER_SBATCH_SIREPO_CMD='/global/homes/{}/{}/.pyenv/versions/py3/bin/sirepo'.format(1, nersc_user),
            SIREPO_JOB_DRIVER_SBATCH_SRDB_ROOT='/global/cscratch1/sd/{sbatch_user}/sirepo-dev',
            SIREPO_JOB_SUPERVISOR_SBATCH_POLL_SECS='15',
            SIREPO_JOB_DRIVER_SBATCH_SUPERVISOR_URI='http://{}:8001'.format(nersc_proxy),
            SIREPO_PKCLI_JOB_SUPERVISOR_IP='0.0.0.0',
            SIREPO_SIMULATION_DB_SBATCH_DISPLAY='Cori@NERSC',
        )

    def _env_sbatch(env):
        h = socket.gethostname()
        env.update(
            SIREPO_JOB_DRIVER_MODULES='local:sbatch',
            SIREPO_JOB_DRIVER_SBATCH_HOST='{}'.format(
                sbatch_host or h,
            ),
            # TODO(e-carlin):
            SIREPO_JOB_DRIVER_SBATCH_SUPERVISOR_URI='http://{}:8001'.format(h),<commit_msg>cleanup comments<commit_after># -*- coding: utf-8 -*-
"""Runs the server in uwsgi or http modes.

Also supports starting nginx proxy.

:copyright: Copyright (c) 2015 RadiaSoft LLC.  All Rights Reserved.
:license: http://www.apache.org/licenses/LICENSE-2.0.html
"""
from __future__ import absolute_import, division, print_function
from pykern import pkcli
from pykern import pkcollections
from pykern import pkconfig
from pykern import pkio
from pykern import pkjinja
from pykern import pksubprocess
from pykern.pkdebug import pkdc, pkdexc, pkdp, pkdlog
import distutils
import errno
import inspect
import os
import py
import re
import signal
import sirepo.srdb
import socket
import subprocess
import sys


def celery():
    """Start celery"""
    assert pkconfig.channel_in('dev')
    import celery.bin.celery
    import sirepo.celery_tasks
    run_dir = _run_dir().join('celery').ensure(dir=True)
    with pkio.save_chdir(run_dir):
        celery.bin.celery.main(argv=[
            'celery',
            'worker',
            '--app=sirepo.celery_tasks',
            '--no-color',
            '-Ofair',
            '--queue=' + ','.join(sirepo.celery_tasks.QUEUE_NAMES),
        ])


def flower():
    """Start flower"""
    from flower import command
    assert pkconfig.channel_in('dev')
    run_dir = _run_dir().join('flower').ensure(dir=True)
    with pkio.save_chdir(run_dir):
        command.FlowerCommand().execute_from_commandline([
            'flower',
            '--address=' + cfg.ip,
            '--app=sirepo.celery_tasks',
            '--no-color',
            '--persistent',
        ])


def flask():
    from sirepo import server

    use_reloader = pkconfig.channel_in('dev')
    app = server.init(use_reloader=use_reloader)
    # avoid WARNING: Do not use the development server in a production environment.
    app.env = 'development'
    app.run(
        host=cfg.ip,
        port=cfg.port,
        threaded=True,
        use_reloader=use_reloader,
    )


def http(driver='local', nersc_proxy=None, nersc_user=None, sbatch_host=None):
    """Starts the Flask server and job_supervisor.

    Used for development only.

    Args:
        driver (string): The driver type to enable (one of local, docker, sbatch, nersc)
        nersc_proxy (string): A proxy nersc can use to reach the supervisor
        nersc_user(string): A nersc user ??
        sbatch_host (string): A host to ssh into to start sbatch jobs
    """
    def _env(py_version):
        e = os.environ.copy()
        e.update(
            PYENV_VERSION=py_version,
            PYKERN_PKDEBUG_WANT_PID_TIME='1',
            SIREPO_AUTH_EMAIL_FROM_EMAIL='<EMAIL>',
            SIREPO_AUTH_EMAIL_FROM_NAME='RadiaSoft Support',
            SIREPO_AUTH_EMAIL_SMTP_PASSWORD='<PASSWORD>',
            # POSIT: same as sirepo.auth.email._DEV_SMTP_SERVER
            SIREPO_AUTH_EMAIL_SMTP_SERVER='dev',
            SIREPO_AUTH_EMAIL_SMTP_USER='n/a',
            SIREPO_AUTH_METHODS='email:guest',
            SIREPO_MPI_CORES='2',
        )
        {
            n: f for n, f in inspect.currentframe().f_back.f_locals.items()
            if callable(f)
        }['_env_' + driver](e)

        if 'sbatch' in e['SIREPO_JOB_DRIVER_MODULES']:
            h = e['SIREPO_JOB_DRIVER_SBATCH_HOST']
            m = re.search(r'^{}.+$'.format(h), pkio.read_text('~/.ssh/known_hosts'), re.MULTILINE)
            assert m, \
                (
                    'you need to get the host key in ~/.ssh/known_hosts'
                    ' run: `ssh {} true`'.format(h)
                )
            x = m.group(0)
            e.update(SIREPO_JOB_DRIVER_SBATCH_HOST_KEY=x)
        return e

    def _env_docker(env):
        # TODO(e-carlin): py3 has shutil.which()
        assert distutils.spawn.find_executable('docker') is not None, \
            'docker not installed:  You need to run `radia_run redhat-docker`'
        i = 'radiasoft/sirepo:dev'
        try:
            subprocess.check_output(('docker', 'image', 'inspect', i))
        except subprocess.CalledProcessError:
            pkdlog('docker image {} not installed. Pulling...')
            assert subprocess.call(('docker', 'image', 'pull', i)) == 0, \
                'docker image pull failed'

        env.update(
            SIREPO_JOB_DRIVER_MODULES='docker',
        )

    def _env_local(env):
        env.update(
            SIREPO_JOB_DRIVER_MODULES='local',
        )
        assert 'SIREPO_JOB_DRIVER_MODULES' in env

    def _env_nersc(env):
        # TODO(e-carlin): Shouldn't nersc_user always be nagler?
        assert nersc_proxy and nersc_user, \
            'You need to supply a nersc_proxy and a nserc_user'
        env.update(
            SIREPO_JOB_DRIVER_MODULES='local:sbatch',
            SIREPO_JOB_DRIVER_SBATCH_HOST='cori.nersc.gov',
            SIREPO_JOB_DRIVER_SBATCH_SHIFTER_IMAGE='radiasoft/sirepo:sbatch',
            SIREPO_JOB_DRIVER_SBATCH_SIREPO_CMD='/global/homes/{}/{}/.pyenv/versions/py3/bin/sirepo'.format(
                nersc_user[0],
                nersc_user,
            ),
            SIREPO_JOB_DRIVER_SBATCH_SRDB_ROOT='/global/cscratch1/sd/{sbatch_user}/sirepo-dev',
            SIREPO_JOB_SUPERVISOR_SBATCH_POLL_SECS='15',
            SIREPO_JOB_DRIVER_SBATCH_SUPERVISOR_URI='http://{}:8001'.format(nersc_proxy),
            SIREPO_PKCLI_JOB_SUPERVISOR_IP='0.0.0.0',
            SIREPO_SIMULATION_DB_SBATCH_DISPLAY='Cori@NERSC',
        )

    def _env_sbatch(env):
        h = socket.gethostname()
        env.update(
            SIREPO_JOB_DRIVER_MODULES='local:sbatch',
            SIREPO_JOB_DRIVER_SBATCH_HOST='{}'.format(
                sbatch_host or h,
            ),
            SIREPO_JOB_DRIVER_SBATCH_SUPERVISOR_URI='http://{}:8001'.format(h),
{'fraction_non_alphanumeric': 0.11341265086022426, 'fraction_numerical': 0.005306856115723701, 'mean_word_length': 2.4113868613138685, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1038113', 'n_tokens_mistral': 4107, 'n_tokens_neox': 3982, 'n_words': 760}