<commit_before># -*- coding: utf-8 -*-
u"""Flask routes

:copyright: Copyright (c) 2015 RadiaSoft LLC.  All Rights Reserved.
:license: http://www.apache.org/licenses/LICENSE-2.0.html
"""
from __future__ import absolute_import, division, print_function
from pykern import pkcollections
from pykern import pkconfig
from pykern import pkio
from pykern import pkresource
from pykern.pkdebug import pkdc, pkdexc, pkdlog, pkdp
from sirepo import simulation_db
from sirepo.template import template_common
import beaker.middleware
import copy
import datetime
import errno
import flask
import flask.sessions
import glob
import json
import os
import py
import re
import signal
import sirepo.template
import subprocess
import sys
import threading
import time
import traceback
import uuid
import werkzeug
import werkzeug.exceptions


#: Identifies the user in the Beaker session
SESSION_KEY_USER = 'uid'

#: where users live under db_dir
_BEAKER_DATA_DIR = 'beaker'

#: where users live under db_dir
_BEAKER_LOCK_DIR = 'lock'

#: What's the key in environ for the session
_ENVIRON_KEY_BEAKER = 'beaker.session'

#: Cache for _json_response_ok
_JSON_RESPONSE_OK = None

#: Parsing errors from subprocess
_SUBPROCESS_ERROR_RE = re.compile(r'(?:warning|exception|error): ([^\n]+)', flags=re.IGNORECASE)

#: Identifies the user in uWSGI logging (read by uwsgi.yml.jinja)
_UWSGI_LOG_KEY_USER = 'sirepo_user'

#: WSGIApp instance (see `init_by_server`)
_wsgi_app = None

#: Flask app instance, must be bound globally
app = flask.Flask(
    __name__,
    static_folder=str(simulation_db.STATIC_FOLDER),
    template_folder=str(simulation_db.STATIC_FOLDER),
)
app.config.update(
    PROPAGATE_EXCEPTIONS=True,
)


def init(db_dir, uwsgi=None):
    """Initialize globals and populate simulation dir"""
    global _wsgi_app
    _wsgi_app = _WSGIApp(app, uwsgi)
    _BeakerSession().sirepo_init_app(app, py.path.local(db_dir))
    simulation_db.init_by_server(app, sys.modules[__name__])


@app.route(simulation_db.SCHEMA_COMMON['route']['copyNonSessionSimulation'], methods=('GET', 'POST'))
def app_copy_nonsession_simulation():
    req = _json_input()
    sim_type = req['simulationType']
    global_path = simulation_db.find_global_simulation(sim_type, req['simulationId'])
    if global_path:
        data = simulation_db.open_json_file(sim_type, os.path.join(global_path, simulation_db.SIMULATION_DATA_FILE))
        data['models']['simulation']['isExample'] = False
        data['models']['simulation']['outOfSessionSimulationId'] = req['simulationId']
        res = _save_new_and_reply(sim_type, data)
        sirepo.template.import_module(data).copy_related_files(data, global_path, str(simulation_db.simulation_dir(sim_type, simulation_db.parse_sid(data))))
        return res
    werkzeug.exceptions.abort(404)


@app.route(simulation_db.SCHEMA_COMMON['route']['copySimulation'], methods=('GET', 'POST'))
def app_copy_simulation():
    """Takes the specified simulation and returns a newly named copy with the suffix (copy X)"""
    req = _json_input()
    sim_type = req['simulationType']
    name = req['name'] if 'name' in req else None
    data = simulation_db.read_simulation_json(sim_type, sid=req['simulationId'])
    if not name:
        base_name = data['models']['simulation']['name']
        names = simulation_db.iterate_simulation_datafiles(sim_type, _simulation_name)
        count = 0
        while True:
            count += 1
            name = base_name + ' (copy{})'.format(' {}'.format(count) if count > 1 else '')
            if name in names and count < 100:
                continue
            break
    data['models']['simulation']['name'] = name
    data['models']['simulation']['isExample'] = False
    data['models']['simulation']['outOfSessionSimulationId'] = ''
    return _save_new_and_reply(sim_type, data)


@app.route(simulation_db.SCHEMA_COMMON['route']['deleteSimulation'], methods=('GET', 'POST'))
def app_delete_simulation():
    data = _parse_data_input()
    pkio.unchecked_remove(simulation_db.simulation_dir(data['simulationType'], data['simulationId']))
    return _json_response_ok();


@app.route(simulation_db.SCHEMA_COMMON['route']['downloadDataFile'], methods=('GET', 'POST'))
def app_download_data_file(simulation_type, simulation_id, model, frame):
    data = {
        'simulationType': simulation_type,
        'simulationId': simulation_id,
        'modelName': model,
    }
    frame = int(frame)
    template = sirepo.template.import_module(data)
    if frame >= 0:
        data['report'] = template.get_animation_name(data)
    else:
        data['report'] = model
    run_dir = simulation_db.simulation_run_dir(data)
    filename, content, content_type = template.get_data_file(run_dir, model, frame)
    response = flask.make_response(content)
    response.mimetype = content_type
    if content_type != 'text/plain':
        response.headers['Content-Disposition'] = 'attachment; filename="{}"'.format(filename)
    return response


@app.route(simulation_db.SCHEMA_COMMON['route']['downloadFile'], methods=('GET', 'POST'))
def app_download_file(simulation_type, simulation_id, filename):
    lib = simulation_db.simulation_lib_dir(simulation_type)
    p = lib.join(werkzeug.secure_filename(filename))
    return flask.send_file(str(p))


@app.route(simulation_db.SCHEMA_COMMON['route']['errorLogging'], methods=('GET', 'POST'))
def app_error_logging():
    ip = flask.request.remote_addr
    try:
        pkdlog(
            '{}: javascript error: {}',
            ip,
            simulation_db.generate_json(_json_input(), pretty=True),
        )
    except ValueError as e:
        pkdlog(
            '{}: error parsing javascript app_error: {} input={}',
            ip,
            e,
            flask.request.data.decode('unicode-escape'),
        )
    return _json_response_ok();


@app.route('/favicon.ico')
def app_favicon():
    """Routes to favicon.ico file."""
    return flask.send_from_directory(
        str(simulation_db.STATIC_FOLDER.join('img')),
        'favicon.ico', mimetype='image/vnd.microsoft.icon'
    )


@app.route(simulation_db.SCHEMA_COMMON['route']['listFiles'], methods=('GET', 'POST'))
def app_file_list(simulation_type, simulation_id, file_type):
    file_type = werkzeug.secure_filename(file_type)
    res = []
    #TODO(pjm): use file prefixes for srw, currently assumes mirror is *.dat and others are *.zip
    if simulation_type == 'srw':
        search = '*.dat' if file_type == 'mirror' else '*.zip'
    else:
        search = '{}.*'.format(file_type)
    d = simulation_db.simulation_lib_dir(simulation_type)
    for f in glob.glob(str(d.join(search))):
        if os.path.isfile(f):
            filename = os.path.basename(f)
            if not simulation_type == 'srw':
                # strip the file_type prefix
                filename = filename[len(file_type) + 1:]
            res.append(filename)
    res.sort()
    return _json_response(res)


@app.route(simulation_db.SCHEMA_COMMON['route']['findByName'], methods=('GET', 'POST'))
def app_find_by_name(simulation_type, application_mode, simulation_name):
    redirect_uri = None
    # use the existing named simulation, or copy it from the examples
    rows = simulation_db.iterate_simulation_datafiles(simulation_type, simulation_db.process_simulation_list, {
        'simulation.name': simulation_name,
    })
    if len(rows) == 0:
        for s in simulation_db.examples(simulation_type):
            if s['models']['simulation']['name'] == simulation_name:
                simulation_db.save_new_example(simulation_type, s)
                rows = simulation_db.iterate_simulation_datafiles(simulation_type, simulation_db.process_simulation_list, {
                    'simulation.name': simulation_name,
                })
                break
    if len(rows):
        if application_mode == 'default':
            redirect_uri = '/{}#/source/{}'.format(simulation_type, rows[0]['simulationId'])
        elif application_mode == 'lattice':
            redirect_uri = '/{}#/lattice/{}'.format(simulation_type, rows[0]['simulationId'])
        elif application_mode == 'wavefront' or application_mode == 'light-sources':
            redirect_uri = '/{}#/beamline/{}?application_mode={}'.format(
                simulation_type, rows[0]['simulationId'], application_mode)
        else:
            redirect_uri = '/{}#/source/{}?application_mode={}'.format(
                simulation_type, rows[0]['simulationId'], application_mode)
    if redirect_uri:
        # redirect using javascript for safari browser which doesn't support hash redirects
        return flask.render_template(
            'html/javascript-redirect.html',
            redirect_uri=redirect_uri
        )
    werkzeug.exceptions.abort(404)


@app.route(simulation_db.SCHEMA_COMMON['route']['getApplicationData'], methods=('GET', 'POST'))
def app_get_application_data():
    data = _parse_data_input()
    return _json_response(sirepo.template.import_module(data).get_application_data(data))


@app.route(simulation_db.SCHEMA_COMMON['route']['importFile'], methods=('GET', 'POST'))
def app_import_file(simulation_type):
    template = sirepo.template.import_module(simulation_type)
    error, data = template.import_file(flask.request, simulation_db.simulation_lib_dir(simulation_type), simulation_db.tmp_dir())
    if error:
        return _json_response({'error': error})
    data['models']['simulation']['folder'] = flask.request.form['folder']
    return _save_new_and_reply(simulation_type, data)


@app.route(simulation_db.SCHEMA_COMMON['route']['newSimulation'], methods=('GET', 'POST'))
def app_new_simulation():
    new_simulation_data = _parse_data_input()
    sim_type = new_simulation_data['simulationType']
    data = simulation_db.open_json_file(
        sim_type,
        simulation_db.STATIC_FOLDER.join('json', '{}-default{}'.format(sim_type, simulation_db.JSON_SUFFIX)),
    )
    data['models']['simulation']['name'] = new_simulation_data['name']
    data['models']['simulation']['folder'] = new_simulation_data['folder']
    sirepo.template.import_module(data).new_simulation(data, new_simulation_data)
    return _save_new_and_reply(sim_type, data)


@app.route(simulation_db.SCHEMA_COMMON['route']['pythonSource'])
def app_python_source(simulation_type, simulation_id):
    data = simulation_db.read_simulation_json(simulation_type, sid=simulation_id)
    template = sirepo.template.import_module(data)
    # ensure the whole source gets generated, not up to the last watchpoint report
    last_watchpoint = None
    if 'beamline' in data['models']:
        for item in reversed(data['models']['beamline']):
            if item['type'] == 'watch':
                last_watchpoint = 'watchpointReport{}'.format(item['id'])
                break
            if last_watchpoint:
                data['report'] = last_watchpoint
    return flask.Response(
        '{}{}'.format(
            template.generate_parameters_file(data, simulation_db.get_schema(simulation_type), is_parallel=True),
            template.run_all_text(data) if simulation_type == 'srw' else template.run_all_text()),
        mimetype='text/plain',
    )


@app.route('/robots.txt')
def app_robots_txt():
    """Tell robots to go away"""
    return flask.Response(
        'User-agent: *\nDisallow: /\n',
        mimetype='text/plain',
    )


@app.route(simulation_db.SCHEMA_COMMON['route']['root'])
def app_root(simulation_type):
    return flask.render_template(
        'html/index.html',
        version=simulation_db.app_version(),
        app_name=simulation_type,
    )


@app.route(simulation_db.SCHEMA_COMMON['route']['runCancel'], methods=('GET', 'POST'))
def app_run_cancel():
    data = _parse_data_input()
    jid = _job_id(data)
    # TODO(robnagler) need to have a way of listing jobs
    # Don't bother with cache_hit check. We don't have any way of canceling
    # if the parameters don't match so for now, always kill.
    if cfg.job_queue.is_processing(jid):
        run_dir = simulation_db.simulation_run_dir(data)
        # Write first, since results are write once, and we want to
        # indicate the cancel instead of the termination error that
        # will happen as a result of the kill.
        simulation_db.write_result({'state': 'canceled'}, run_dir=run_dir)
        cfg.job_queue.kill(jid)
        # TODO(robnagler) should really be inside the template (t.cancel_simulation()?)
        # the last frame file may not be finished, remove it
        t = sirepo.template.import_module(data)
        t.remove_last_frame(run_dir)
    # Always true from the client's perspective
    return _json_response({'state': 'canceled'})


@app.route(simulation_db.SCHEMA_COMMON['route']['runSimulation'], methods=('GET', 'POST'))
def app_run_simulation():
    data = _parse_data_input(validate=True)
    res = _simulation_run_status(data, quiet=True)
    if (
        (
            res['state'] != ('running', 'pending')
            and (res['state'] != 'completed' or data.get('forceRun', False))
        ) or res.get('parametersChanged', True)
    ):
        _start_simulation(data)
        res = _simulation_run_status(data)
    return _json_response(res)


@app.route(simulation_db.SCHEMA_COMMON['route']['runStatus'], methods=('GET', 'POST'))
def app_run_status():
    data = _parse_data_input()
    return _json_response(_simulation_run_status(data))


@app.route(simulation_db.SCHEMA_COMMON['route']['saveSimulationData'], methods=('GET', 'POST'))
def app_save_simulation_data():
    data = _parse_data_input(validate=True)
    res = _validate_serial(data)
    if res:
        return res
    data = simulation_db.save_simulation_json(data['simulationType'], data)
    return app_simulation_data(
        data['simulationType'],
        data['models']['simulation']['simulationId'],
        pretty=False,
    )


@app.route(simulation_db.SCHEMA_COMMON['route']['simulationData'])
def app_simulation_data(simulation_type, simulation_id, pretty):
    #TODO(robnagler) need real type transforms for inputs
    pretty = bool(int(pretty))
    try:
        data = simulation_db.read_simulation_json(simulation_type, sid=simulation_id)
        response = _json_response(
            sirepo.template.import_module(simulation_type).prepare_for_client(data),
            pretty=pretty,
        )
    except simulation_db.CopyRedirect as e:
        response = _json_response(e.sr_response)
    _no_cache(response)
    return response


@app.route(simulation_db.SCHEMA_COMMON['route']['simulationFrame'])
def app_simulation_frame(frame_id):
    #TODO(robnagler) startTime is reportParametersHash; need version on URL and/or param names in URL
    keys = ['simulationType', 'simulationId', 'modelName', 'animationArgs', 'frameIndex', 'startTime']
    data = dict(zip(keys, frame_id.split('*')))
    template = sirepo.template.import_module(data)
    data['report'] = template.get_animation_name(data)
    run_dir = simulation_db.simulation_run_dir(data)
    model_data = simulation_db.read_json(run_dir.join(template_common.INPUT_BASE_NAME))
    response = _json_response(template.get_simulation_frame(run_dir, data, model_data))

    if template.WANT_BROWSER_FRAME_CACHE:
        now = datetime.datetime.utcnow()
        expires = now + datetime.timedelta(365)
        response.headers['Cache-Control'] = 'public, max-age=31536000'
        response.headers['Expires'] = expires.strftime("%a, %d %b %Y %H:%M:%S GMT")
        response.headers['Last-Modified'] = now.strftime("%a, %d %b %Y %H:%M:%S GMT")
    else:
        _no_cache(response)
    return response


@app.route(simulation_db.SCHEMA_COMMON['route']['listSimulations'], methods=('GET', 'POST'))
def app_simulation_list():
    data = _parse_data_input()
    sim_type = data['simulationType']
    search = data['search'] if 'search' in data else None
    simulation_db.verify_app_directory(sim_type)
    return _json_response(
        sorted(
            simulation_db.iterate_simulation_datafiles(sim_type, simulation_db.process_simulation_list, search),
            key=lambda row: row['name'],
        )
    )


@app.route(simulation_db.SCHEMA_COMMON['route']['simulationSchema'], methods=('GET', 'POST'))
def app_simulation_schema():
    sim_type = flask.request.form['simulationType']
    return _json_response(simulation_db.get_schema(sim_type))


@app.route(simulation_db.SCHEMA_COMMON['route']['updateFolder'], methods=('GET', 'POST'))
def app_update_folder():
    #TODO(robnagler) Folder should have a serial, or should it be on data
    data = _parse_data_input()
    old_name = data['oldName']
    new_name = data['newName']
    for row in simulation_db.iterate_simulation_datafiles(data['simulationType'], _simulation_data):
        folder = row['models']['simulation']['folder']
        if folder.startswith(old_name):
            row['models']['simulation']['folder'] = re.sub(re.escape(old_name), new_name, folder, 1)
            simulation_db.save_simulation_json(data['simulationType'], row)
    return _json_response_ok();


@app.route(simulation_db.SCHEMA_COMMON['route']['uploadFile'], methods=('GET', 'POST'))
def app_upload_file(simulation_type, simulation_id, file_type):
    f = flask.request.files['file']
    lib = simulation_db.simulation_lib_dir(simulation_type)
    template = sirepo.template.import_module(simulation_type)
    filename = werkzeug.secure_filename(f.filename)
    if simulation_type == 'srw':
        p = lib.join(filename)
    else:
        p = lib.join(werkzeug.secure_filename('{}.{}'.format(file_type, filename)))
    err = None
    if p.check():
        err = 'file exists: {}'.format(filename)
    if not err:
        f.save(str(p))
        err = template.validate_file(file_type, str(p))
        if err:
            pkio.unchecked_remove(p)
    if err:
        return _json_response({
            'error': err,
            'filename': filename,
            'fileType': file_type,
            'simulationId': simulation_id,
        })
    return _json_response({
        'filename': filename,
        'fileType': file_type,
        'simulationId': simulation_id,
    })


@app.route('/')
@app.route('/light')
def light_landing_page():
    return flask.render_template(
        'html/sr-landing-page.html',
        version=simulation_db.app_version(),
    )


def session_user(*args, **kwargs):
    """Get/set the user from the Flask session

    With no positional arguments, is a getter. Else a setter.

    Args:
        user (str): if args[0], will set the user; else gets
        checked (bool): if kwargs['checked'], assert the user is truthy
        environ (dict): session environment to use instead of `flask.session`

    Returns:
        str: user id
    """
    environ = kwargs.get('environ', None)
    session = environ.get(_ENVIRON_KEY_BEAKER) if environ else flask.session
    if args:
        session[SESSION_KEY_USER] = args[0]
        _wsgi_app.set_log_user(args[0])
    res = session.get(SESSION_KEY_USER)
    if not res and kwargs.get('checked', True):
        raise KeyError(SESSION_KEY_USER)
    return res


@app.route('/sr')
def sr_landing_page():
    return flask.redirect('/light')


class _BeakerSession(flask.sessions.SessionInterface):
    """Session manager for Flask using Beaker.

    Stores session info in files in sirepo.server.data_dir. Minimal info kept
    in session.
    """
    def __init__(self, app=None):
        if app is None:
            self.app = None
        else:
            self.init_app(app)

    def sirepo_init_app(self, app, db_dir):
        """Initialize cfg with db_dir and register self with Flask

        Args:
            app (flask): Flask application object
            db_dir (py.path.local): db_dir passed on command line
        """
        app.sirepo_db_dir = db_dir
        data_dir = db_dir.join(_BEAKER_DATA_DIR)
        lock_dir = data_dir.join(_BEAKER_LOCK_DIR)
        pkio.mkdir_parent(lock_dir)
        sc = {
            'session.auto': True,
            'session.cookie_expires': False,
            'session.type': 'file',
            'session.data_dir': str(data_dir),
            'session.lock_dir': str(lock_dir),
        }
        #TODO(robnagler) Generalize? seems like we'll be shadowing lots of config
        for k in cfg.beaker_session:
            sc['session.' + k] = cfg.beaker_session[k]
        app.wsgi_app = beaker.middleware.SessionMiddleware(app.wsgi_app, sc)
        app.session_interface = self

    def open_session(self, app, request):
        """Called by flask to create the session"""
        return request.environ[_ENVIRON_KEY_BEAKER]

    def save_session(self, *args, **kwargs):
        """Necessary to complete abstraction, but Beaker autosaves"""
        pass


class _WSGIApp(object):
    """Wraps Flask's wsgi_app for logging

    Args:
        app (Flask.app): Flask application being wrapped
        uwsgi (module): `uwsgi` module passed from ``uwsgi.py.jinja``
    """
    def __init__(self, app, uwsgi):
        self.app = app
        # Is None if called from sirepo.pkcli.service.http or FlaskClient
        self.uwsgi = uwsgi
        self.wsgi_app = app.wsgi_app
        app.wsgi_app = self

    def set_log_user(self, user):
        if self.uwsgi:
            log_user = 'li-' + user if user else '-'
            # Only works for uWSGI (service.uwsgi). For service.http,
            # werkzeug.serving.WSGIRequestHandler.log hardwires '%s - - [%s] %s\n',
            # and no point in overriding, since just for development.
            self.uwsgi.set_logvar(_UWSGI_LOG_KEY_USER, log_user)

    def __call__(self, environ, start_response):
        """An "app" called by uwsgi with requests.
        """
        self.set_log_user(session_user(checked=False, environ=environ))
        return self.wsgi_app(environ, start_response)


def _cached_simulation(data):
    """Read the run_dir and return cached_data.

    Only a hit if the models between data and cache match exactly. Otherwise,
    return cached data if it's there and valid.

    Args:
        data (dict): parameters identifying run_dir and models or reportParametersHash

    Returns:
        bool: cache hit and matches data models?
        dict: cached data
    """
    # Sets data['reportParametersHash']
    req_hash = template_common.report_parameters_hash(data)
    run_dir = simulation_db.simulation_run_dir(data)
    if not run_dir.check():
        return False, None
    #TODO(robnagler) Lock
    cached_data = None
    try:
        cached_data = simulation_db.read_json(_simulation_input(run_dir))
        cached_hash = template_common.report_parameters_hash(cached_data)
        if req_hash == cached_hash:
            return True, cached_data
    except IOError as e:
        pkdlog('{}: ignore IOError: {} errno={}', run_dir, e, e.errno)
    except Exception as e:
        pkdlog('{}: ignore other error: {}', run_dir, e)
        # No idea if cache is valid or not so throw away
        cached_data = None
    return False, cached_data


def _cfg_job_queue(value):
    """Converts string to class"""
    if isinstance(value, (_Celery, _Background)):
        # Already initialized but may call initializer with original object
        return value
    if value == 'Celery':
        from sirepo import celery_tasks
        err = None
        try:
            if not celery_tasks.celery.control.ping():
                err = 'You need to start Celery:\nsirepo server celery'
        except Exception:
            err = 'You need to start Rabbit:\nsirepo server rabbitmq'
        if err:
            #TODO(robnagler) really should be pkconfig.Error() or something else
            # but this prints a nice message. Don't call sys.exit, not nice
            import pykern.pkcli
            pykern.pkcli.command_error(err)
        return _Celery
    elif value == 'Background':
        signal.signal(signal.SIGCHLD, _Background.sigchld_handler)
        return _Background
    else:
        raise AssertionError('{}: unknown job_queue'.format(value))


@pkconfig.parse_none
def _cfg_session_secret(value):
    """Reads file specified as config value"""
    if not value:
        return 'dev dummy secret'<commit_msg>fixed typo in error message<commit_after># -*- coding: utf-8 -*-
u"""Flask routes

:copyright: Copyright (c) 2015 RadiaSoft LLC.  All Rights Reserved.
:license: http://www.apache.org/licenses/LICENSE-2.0.html
"""
from __future__ import absolute_import, division, print_function
from pykern import pkcollections
from pykern import pkconfig
from pykern import pkio
from pykern import pkresource
from pykern.pkdebug import pkdc, pkdexc, pkdlog, pkdp
from sirepo import simulation_db
from sirepo.template import template_common
import beaker.middleware
import copy
import datetime
import errno
import flask
import flask.sessions
import glob
import json
import os
import py
import re
import signal
import sirepo.template
import subprocess
import sys
import threading
import time
import traceback
import uuid
import werkzeug
import werkzeug.exceptions


#: Identifies the user in the Beaker session
SESSION_KEY_USER = 'uid'

#: where users live under db_dir
_BEAKER_DATA_DIR = 'beaker'

#: where users live under db_dir
_BEAKER_LOCK_DIR = 'lock'

#: What's the key in environ for the session
_ENVIRON_KEY_BEAKER = 'beaker.session'

#: Cache for _json_response_ok
_JSON_RESPONSE_OK = None

#: Parsing errors from subprocess
_SUBPROCESS_ERROR_RE = re.compile(r'(?:warning|exception|error): ([^\n]+)', flags=re.IGNORECASE)

#: Identifies the user in uWSGI logging (read by uwsgi.yml.jinja)
_UWSGI_LOG_KEY_USER = 'sirepo_user'

#: WSGIApp instance (see `init_by_server`)
_wsgi_app = None

#: Flask app instance, must be bound globally
app = flask.Flask(
    __name__,
    static_folder=str(simulation_db.STATIC_FOLDER),
    template_folder=str(simulation_db.STATIC_FOLDER),
)
app.config.update(
    PROPAGATE_EXCEPTIONS=True,
)


def init(db_dir, uwsgi=None):
    """Initialize globals and populate simulation dir"""
    global _wsgi_app
    _wsgi_app = _WSGIApp(app, uwsgi)
    _BeakerSession().sirepo_init_app(app, py.path.local(db_dir))
    simulation_db.init_by_server(app, sys.modules[__name__])


@app.route(simulation_db.SCHEMA_COMMON['route']['copyNonSessionSimulation'], methods=('GET', 'POST'))
def app_copy_nonsession_simulation():
    req = _json_input()
    sim_type = req['simulationType']
    global_path = simulation_db.find_global_simulation(sim_type, req['simulationId'])
    if global_path:
        data = simulation_db.open_json_file(sim_type, os.path.join(global_path, simulation_db.SIMULATION_DATA_FILE))
        data['models']['simulation']['isExample'] = False
        data['models']['simulation']['outOfSessionSimulationId'] = req['simulationId']
        res = _save_new_and_reply(sim_type, data)
        sirepo.template.import_module(data).copy_related_files(data, global_path, str(simulation_db.simulation_dir(sim_type, simulation_db.parse_sid(data))))
        return res
    werkzeug.exceptions.abort(404)


@app.route(simulation_db.SCHEMA_COMMON['route']['copySimulation'], methods=('GET', 'POST'))
def app_copy_simulation():
    """Takes the specified simulation and returns a newly named copy with the suffix (copy X)"""
    req = _json_input()
    sim_type = req['simulationType']
    name = req['name'] if 'name' in req else None
    data = simulation_db.read_simulation_json(sim_type, sid=req['simulationId'])
    if not name:
        base_name = data['models']['simulation']['name']
        names = simulation_db.iterate_simulation_datafiles(sim_type, _simulation_name)
        count = 0
        while True:
            count += 1
            name = base_name + ' (copy{})'.format(' {}'.format(count) if count > 1 else '')
            if name in names and count < 100:
                continue
            break
    data['models']['simulation']['name'] = name
    data['models']['simulation']['isExample'] = False
    data['models']['simulation']['outOfSessionSimulationId'] = ''
    return _save_new_and_reply(sim_type, data)


@app.route(simulation_db.SCHEMA_COMMON['route']['deleteSimulation'], methods=('GET', 'POST'))
def app_delete_simulation():
    data = _parse_data_input()
    pkio.unchecked_remove(simulation_db.simulation_dir(data['simulationType'], data['simulationId']))
    return _json_response_ok();


@app.route(simulation_db.SCHEMA_COMMON['route']['downloadDataFile'], methods=('GET', 'POST'))
def app_download_data_file(simulation_type, simulation_id, model, frame):
    data = {
        'simulationType': simulation_type,
        'simulationId': simulation_id,
        'modelName': model,
    }
    frame = int(frame)
    template = sirepo.template.import_module(data)
    if frame >= 0:
        data['report'] = template.get_animation_name(data)
    else:
        data['report'] = model
    run_dir = simulation_db.simulation_run_dir(data)
    filename, content, content_type = template.get_data_file(run_dir, model, frame)
    response = flask.make_response(content)
    response.mimetype = content_type
    if content_type != 'text/plain':
        response.headers['Content-Disposition'] = 'attachment; filename="{}"'.format(filename)
    return response


@app.route(simulation_db.SCHEMA_COMMON['route']['downloadFile'], methods=('GET', 'POST'))
def app_download_file(simulation_type, simulation_id, filename):
    lib = simulation_db.simulation_lib_dir(simulation_type)
    p = lib.join(werkzeug.secure_filename(filename))
    return flask.send_file(str(p))


@app.route(simulation_db.SCHEMA_COMMON['route']['errorLogging'], methods=('GET', 'POST'))
def app_error_logging():
    ip = flask.request.remote_addr
    try:
        pkdlog(
            '{}: javascript error: {}',
            ip,
            simulation_db.generate_json(_json_input(), pretty=True),
        )
    except ValueError as e:
        pkdlog(
            '{}: error parsing javascript app_error: {} input={}',
            ip,
            e,
            flask.request.data.decode('unicode-escape'),
        )
    return _json_response_ok();


@app.route('/favicon.ico')
def app_favicon():
    """Routes to favicon.ico file."""
    return flask.send_from_directory(
        str(simulation_db.STATIC_FOLDER.join('img')),
        'favicon.ico', mimetype='image/vnd.microsoft.icon'
    )


@app.route(simulation_db.SCHEMA_COMMON['route']['listFiles'], methods=('GET', 'POST'))
def app_file_list(simulation_type, simulation_id, file_type):
    file_type = werkzeug.secure_filename(file_type)
    res = []
    #TODO(pjm): use file prefixes for srw, currently assumes mirror is *.dat and others are *.zip
    if simulation_type == 'srw':
        search = '*.dat' if file_type == 'mirror' else '*.zip'
    else:
        search = '{}.*'.format(file_type)
    d = simulation_db.simulation_lib_dir(simulation_type)
    for f in glob.glob(str(d.join(search))):
        if os.path.isfile(f):
            filename = os.path.basename(f)
            if not simulation_type == 'srw':
                # strip the file_type prefix
                filename = filename[len(file_type) + 1:]
            res.append(filename)
    res.sort()
    return _json_response(res)


@app.route(simulation_db.SCHEMA_COMMON['route']['findByName'], methods=('GET', 'POST'))
def app_find_by_name(simulation_type, application_mode, simulation_name):
    redirect_uri = None
    # use the existing named simulation, or copy it from the examples
    rows = simulation_db.iterate_simulation_datafiles(simulation_type, simulation_db.process_simulation_list, {
        'simulation.name': simulation_name,
    })
    if len(rows) == 0:
        for s in simulation_db.examples(simulation_type):
            if s['models']['simulation']['name'] == simulation_name:
                simulation_db.save_new_example(simulation_type, s)
                rows = simulation_db.iterate_simulation_datafiles(simulation_type, simulation_db.process_simulation_list, {
                    'simulation.name': simulation_name,
                })
                break
    if len(rows):
        if application_mode == 'default':
            redirect_uri = '/{}#/source/{}'.format(simulation_type, rows[0]['simulationId'])
        elif application_mode == 'lattice':
            redirect_uri = '/{}#/lattice/{}'.format(simulation_type, rows[0]['simulationId'])
        elif application_mode == 'wavefront' or application_mode == 'light-sources':
            redirect_uri = '/{}#/beamline/{}?application_mode={}'.format(
                simulation_type, rows[0]['simulationId'], application_mode)
        else:
            redirect_uri = '/{}#/source/{}?application_mode={}'.format(
                simulation_type, rows[0]['simulationId'], application_mode)
    if redirect_uri:
        # redirect using javascript for safari browser which doesn't support hash redirects
        return flask.render_template(
            'html/javascript-redirect.html',
            redirect_uri=redirect_uri
        )
    werkzeug.exceptions.abort(404)


@app.route(simulation_db.SCHEMA_COMMON['route']['getApplicationData'], methods=('GET', 'POST'))
def app_get_application_data():
    data = _parse_data_input()
    return _json_response(sirepo.template.import_module(data).get_application_data(data))


@app.route(simulation_db.SCHEMA_COMMON['route']['importFile'], methods=('GET', 'POST'))
def app_import_file(simulation_type):
    template = sirepo.template.import_module(simulation_type)
    error, data = template.import_file(flask.request, simulation_db.simulation_lib_dir(simulation_type), simulation_db.tmp_dir())
    if error:
        return _json_response({'error': error})
    data['models']['simulation']['folder'] = flask.request.form['folder']
    return _save_new_and_reply(simulation_type, data)


@app.route(simulation_db.SCHEMA_COMMON['route']['newSimulation'], methods=('GET', 'POST'))
def app_new_simulation():
    new_simulation_data = _parse_data_input()
    sim_type = new_simulation_data['simulationType']
    data = simulation_db.open_json_file(
        sim_type,
        simulation_db.STATIC_FOLDER.join('json', '{}-default{}'.format(sim_type, simulation_db.JSON_SUFFIX)),
    )
    data['models']['simulation']['name'] = new_simulation_data['name']
    data['models']['simulation']['folder'] = new_simulation_data['folder']
    sirepo.template.import_module(data).new_simulation(data, new_simulation_data)
    return _save_new_and_reply(sim_type, data)


@app.route(simulation_db.SCHEMA_COMMON['route']['pythonSource'])
def app_python_source(simulation_type, simulation_id):
    data = simulation_db.read_simulation_json(simulation_type, sid=simulation_id)
    template = sirepo.template.import_module(data)
    # ensure the whole source gets generated, not up to the last watchpoint report
    last_watchpoint = None
    if 'beamline' in data['models']:
        for item in reversed(data['models']['beamline']):
            if item['type'] == 'watch':
                last_watchpoint = 'watchpointReport{}'.format(item['id'])
                break
            if last_watchpoint:
                data['report'] = last_watchpoint
    return flask.Response(
        '{}{}'.format(
            template.generate_parameters_file(data, simulation_db.get_schema(simulation_type), is_parallel=True),
            template.run_all_text(data) if simulation_type == 'srw' else template.run_all_text()),
        mimetype='text/plain',
    )


@app.route('/robots.txt')
def app_robots_txt():
    """Tell robots to go away"""
    return flask.Response(
        'User-agent: *\nDisallow: /\n',
        mimetype='text/plain',
    )


@app.route(simulation_db.SCHEMA_COMMON['route']['root'])
def app_root(simulation_type):
    return flask.render_template(
        'html/index.html',
        version=simulation_db.app_version(),
        app_name=simulation_type,
    )


@app.route(simulation_db.SCHEMA_COMMON['route']['runCancel'], methods=('GET', 'POST'))
def app_run_cancel():
    data = _parse_data_input()
    jid = _job_id(data)
    # TODO(robnagler) need to have a way of listing jobs
    # Don't bother with cache_hit check. We don't have any way of canceling
    # if the parameters don't match so for now, always kill.
    if cfg.job_queue.is_processing(jid):
        run_dir = simulation_db.simulation_run_dir(data)
        # Write first, since results are write once, and we want to
        # indicate the cancel instead of the termination error that
        # will happen as a result of the kill.
        simulation_db.write_result({'state': 'canceled'}, run_dir=run_dir)
        cfg.job_queue.kill(jid)
        # TODO(robnagler) should really be inside the template (t.cancel_simulation()?)
        # the last frame file may not be finished, remove it
        t = sirepo.template.import_module(data)
        t.remove_last_frame(run_dir)
    # Always true from the client's perspective
    return _json_response({'state': 'canceled'})


@app.route(simulation_db.SCHEMA_COMMON['route']['runSimulation'], methods=('GET', 'POST'))
def app_run_simulation():
    data = _parse_data_input(validate=True)
    res = _simulation_run_status(data, quiet=True)
    if (
        (
            res['state'] != ('running', 'pending')
            and (res['state'] != 'completed' or data.get('forceRun', False))
        ) or res.get('parametersChanged', True)
    ):
        _start_simulation(data)
        res = _simulation_run_status(data)
    return _json_response(res)


@app.route(simulation_db.SCHEMA_COMMON['route']['runStatus'], methods=('GET', 'POST'))
def app_run_status():
    data = _parse_data_input()
    return _json_response(_simulation_run_status(data))


@app.route(simulation_db.SCHEMA_COMMON['route']['saveSimulationData'], methods=('GET', 'POST'))
def app_save_simulation_data():
    data = _parse_data_input(validate=True)
    res = _validate_serial(data)
    if res:
        return res
    data = simulation_db.save_simulation_json(data['simulationType'], data)
    return app_simulation_data(
        data['simulationType'],
        data['models']['simulation']['simulationId'],
        pretty=False,
    )


@app.route(simulation_db.SCHEMA_COMMON['route']['simulationData'])
def app_simulation_data(simulation_type, simulation_id, pretty):
    #TODO(robnagler) need real type transforms for inputs
    pretty = bool(int(pretty))
    try:
        data = simulation_db.read_simulation_json(simulation_type, sid=simulation_id)
        response = _json_response(
            sirepo.template.import_module(simulation_type).prepare_for_client(data),
            pretty=pretty,
        )
    except simulation_db.CopyRedirect as e:
        response = _json_response(e.sr_response)
    _no_cache(response)
    return response


@app.route(simulation_db.SCHEMA_COMMON['route']['simulationFrame'])
def app_simulation_frame(frame_id):
    #TODO(robnagler) startTime is reportParametersHash; need version on URL and/or param names in URL
    keys = ['simulationType', 'simulationId', 'modelName', 'animationArgs', 'frameIndex', 'startTime']
    data = dict(zip(keys, frame_id.split('*')))
    template = sirepo.template.import_module(data)
    data['report'] = template.get_animation_name(data)
    run_dir = simulation_db.simulation_run_dir(data)
    model_data = simulation_db.read_json(run_dir.join(template_common.INPUT_BASE_NAME))
    response = _json_response(template.get_simulation_frame(run_dir, data, model_data))

    if template.WANT_BROWSER_FRAME_CACHE:
        now = datetime.datetime.utcnow()
        expires = now + datetime.timedelta(365)
        response.headers['Cache-Control'] = 'public, max-age=31536000'
        response.headers['Expires'] = expires.strftime("%a, %d %b %Y %H:%M:%S GMT")
        response.headers['Last-Modified'] = now.strftime("%a, %d %b %Y %H:%M:%S GMT")
    else:
        _no_cache(response)
    return response


@app.route(simulation_db.SCHEMA_COMMON['route']['listSimulations'], methods=('GET', 'POST'))
def app_simulation_list():
    data = _parse_data_input()
    sim_type = data['simulationType']
    search = data['search'] if 'search' in data else None
    simulation_db.verify_app_directory(sim_type)
    return _json_response(
        sorted(
            simulation_db.iterate_simulation_datafiles(sim_type, simulation_db.process_simulation_list, search),
            key=lambda row: row['name'],
        )
    )


@app.route(simulation_db.SCHEMA_COMMON['route']['simulationSchema'], methods=('GET', 'POST'))
def app_simulation_schema():
    sim_type = flask.request.form['simulationType']
    return _json_response(simulation_db.get_schema(sim_type))


@app.route(simulation_db.SCHEMA_COMMON['route']['updateFolder'], methods=('GET', 'POST'))
def app_update_folder():
    #TODO(robnagler) Folder should have a serial, or should it be on data
    data = _parse_data_input()
    old_name = data['oldName']
    new_name = data['newName']
    for row in simulation_db.iterate_simulation_datafiles(data['simulationType'], _simulation_data):
        folder = row['models']['simulation']['folder']
        if folder.startswith(old_name):
            row['models']['simulation']['folder'] = re.sub(re.escape(old_name), new_name, folder, 1)
            simulation_db.save_simulation_json(data['simulationType'], row)
    return _json_response_ok();


@app.route(simulation_db.SCHEMA_COMMON['route']['uploadFile'], methods=('GET', 'POST'))
def app_upload_file(simulation_type, simulation_id, file_type):
    f = flask.request.files['file']
    lib = simulation_db.simulation_lib_dir(simulation_type)
    template = sirepo.template.import_module(simulation_type)
    filename = werkzeug.secure_filename(f.filename)
    if simulation_type == 'srw':
        p = lib.join(filename)
    else:
        p = lib.join(werkzeug.secure_filename('{}.{}'.format(file_type, filename)))
    err = None
    if p.check():
        err = 'file exists: {}'.format(filename)
    if not err:
        f.save(str(p))
        err = template.validate_file(file_type, str(p))
        if err:
            pkio.unchecked_remove(p)
    if err:
        return _json_response({
            'error': err,
            'filename': filename,
            'fileType': file_type,
            'simulationId': simulation_id,
        })
    return _json_response({
        'filename': filename,
        'fileType': file_type,
        'simulationId': simulation_id,
    })


@app.route('/')
@app.route('/light')
def light_landing_page():
    return flask.render_template(
        'html/sr-landing-page.html',
        version=simulation_db.app_version(),
    )


def session_user(*args, **kwargs):
    """Get/set the user from the Flask session

    With no positional arguments, is a getter. Else a setter.

    Args:
        user (str): if args[0], will set the user; else gets
        checked (bool): if kwargs['checked'], assert the user is truthy
        environ (dict): session environment to use instead of `flask.session`

    Returns:
        str: user id
    """
    environ = kwargs.get('environ', None)
    session = environ.get(_ENVIRON_KEY_BEAKER) if environ else flask.session
    if args:
        session[SESSION_KEY_USER] = args[0]
        _wsgi_app.set_log_user(args[0])
    res = session.get(SESSION_KEY_USER)
    if not res and kwargs.get('checked', True):
        raise KeyError(SESSION_KEY_USER)
    return res


@app.route('/sr')
def sr_landing_page():
    return flask.redirect('/light')


class _BeakerSession(flask.sessions.SessionInterface):
    """Session manager for Flask using Beaker.

    Stores session info in files in sirepo.server.data_dir. Minimal info kept
    in session.
    """
    def __init__(self, app=None):
        if app is None:
            self.app = None
        else:
            self.init_app(app)

    def sirepo_init_app(self, app, db_dir):
        """Initialize cfg with db_dir and register self with Flask

        Args:
            app (flask): Flask application object
            db_dir (py.path.local): db_dir passed on command line
        """
        app.sirepo_db_dir = db_dir
        data_dir = db_dir.join(_BEAKER_DATA_DIR)
        lock_dir = data_dir.join(_BEAKER_LOCK_DIR)
        pkio.mkdir_parent(lock_dir)
        sc = {
            'session.auto': True,
            'session.cookie_expires': False,
            'session.type': 'file',
            'session.data_dir': str(data_dir),
            'session.lock_dir': str(lock_dir),
        }
        #TODO(robnagler) Generalize? seems like we'll be shadowing lots of config
        for k in cfg.beaker_session:
            sc['session.' + k] = cfg.beaker_session[k]
        app.wsgi_app = beaker.middleware.SessionMiddleware(app.wsgi_app, sc)
        app.session_interface = self

    def open_session(self, app, request):
        """Called by flask to create the session"""
        return request.environ[_ENVIRON_KEY_BEAKER]

    def save_session(self, *args, **kwargs):
        """Necessary to complete abstraction, but Beaker autosaves"""
        pass


class _WSGIApp(object):
    """Wraps Flask's wsgi_app for logging

    Args:
        app (Flask.app): Flask application being wrapped
        uwsgi (module): `uwsgi` module passed from ``uwsgi.py.jinja``
    """
    def __init__(self, app, uwsgi):
        self.app = app
        # Is None if called from sirepo.pkcli.service.http or FlaskClient
        self.uwsgi = uwsgi
        self.wsgi_app = app.wsgi_app
        app.wsgi_app = self

    def set_log_user(self, user):
        if self.uwsgi:
            log_user = 'li-' + user if user else '-'
            # Only works for uWSGI (service.uwsgi). For service.http,
            # werkzeug.serving.WSGIRequestHandler.log hardwires '%s - - [%s] %s\n',
            # and no point in overriding, since just for development.
            self.uwsgi.set_logvar(_UWSGI_LOG_KEY_USER, log_user)

    def __call__(self, environ, start_response):
        """An "app" called by uwsgi with requests.
        """
        self.set_log_user(session_user(checked=False, environ=environ))
        return self.wsgi_app(environ, start_response)


def _cached_simulation(data):
    """Read the run_dir and return cached_data.

    Only a hit if the models between data and cache match exactly. Otherwise,
    return cached data if it's there and valid.

    Args:
        data (dict): parameters identifying run_dir and models or reportParametersHash

    Returns:
        bool: cache hit and matches data models?
        dict: cached data
    """
    # Sets data['reportParametersHash']
    req_hash = template_common.report_parameters_hash(data)
    run_dir = simulation_db.simulation_run_dir(data)
    if not run_dir.check():
        return False, None
    #TODO(robnagler) Lock
    cached_data = None
    try:
        cached_data = simulation_db.read_json(_simulation_input(run_dir))
        cached_hash = template_common.report_parameters_hash(cached_data)
        if req_hash == cached_hash:
            return True, cached_data
    except IOError as e:
        pkdlog('{}: ignore IOError: {} errno={}', run_dir, e, e.errno)
    except Exception as e:
        pkdlog('{}: ignore other error: {}', run_dir, e)
        # No idea if cache is valid or not so throw away
        cached_data = None
    return False, cached_data


def _cfg_job_queue(value):
    """Converts string to class"""
    if isinstance(value, (_Celery, _Background)):
        # Already initialized but may call initializer with original object
        return value
    if value == 'Celery':
        from sirepo import celery_tasks
        err = None
        try:
            if not celery_tasks.celery.control.ping():
                err = 'You need to start Celery:\nsirepo service celery'
        except Exception:
            err = 'You need to start Rabbit:\nsirepo service rabbitmq'
        if err:
            #TODO(robnagler) really should be pkconfig.Error() or something else
            # but this prints a nice message. Don't call sys.exit, not nice
            import pykern.pkcli
            pykern.pkcli.command_error(err)
        return _Celery
    elif value == 'Background':
        signal.signal(signal.SIGCHLD, _Background.sigchld_handler)
        return _Background
    else:
        raise AssertionError('{}: unknown job_queue'.format(value))


@pkconfig.parse_none
def _cfg_session_secret(value):
    """Reads file specified as config value"""
    if not value:
        return 'dev dummy secret'
{'fraction_non_alphanumeric': 0.11947836886772925, 'fraction_numerical': 0.0016973711446905402, 'mean_word_length': 3.507043567496968, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4392792', 'n_tokens_mistral': 15348, 'n_tokens_neox': 15011, 'n_words': 3243}