<commit_before># -*- coding: utf-8 -*-
u"""Radia execution template.

All Radia calls have to be done from here, NOT in jinja files, because otherwise the
Radia "instance" goes away and references no longer have any meaning.

:copyright: Copyright (c) 2017-2018 RadiaSoft LLC.  All Rights Reserved.
:license: http://www.apache.org/licenses/LICENSE-2.0.html
"""
from __future__ import division
from pykern import pkcollections
from pykern.pkcollections import PKDict
from pykern import pkio
from pykern import pkjinja
from pykern.pkdebug import pkdc, pkdp
from scipy.spatial.transform import Rotation
from sirepo import simulation_db
from sirepo.template import template_common
from sirepo.template import radia_tk
from sirepo.template import radia_examples
import h5py
import math
import numpy
import re
import sdds
import sirepo.csv
import sirepo.sim_data
import sirepo.util
import time
import uuid

_BEAM_AXIS_ROTATIONS = PKDict(
    x=Rotation.from_matrix([[0, 0, 1], [0, 1, 0], [-1, 0, 0]]),
    y=Rotation.from_matrix([[1, 0, 0], [0, 0, -1], [0, 1, 0]]),
    z=Rotation.from_matrix([[1, 0, 0], [0, 1, 0], [0, 0, 1]])
)

_BEAM_AXIS_VECTORS = PKDict(
    x=[1, 0, 0],
    y=[0, 1, 0],
    z=[0, 0, 1]
)

_GAP_AXIS_MAP = PKDict(
    x='z',
    y='z',
    z='y'
)

_DMP_FILE = 'geometry.dat'

# Note that these column names and units are required by elegant
_FIELD_MAP_COLS = ['x', 'y', 'z', 'Bx', 'By', 'Bz']
_FIELD_MAP_UNITS = ['m', 'm', 'm', 'T', 'T', 'T']
_KICK_MAP_COLS = ['x', 'y', 'xpFactor', 'ypFactor']
_KICK_MAP_UNITS = ['m', 'm', '(T*m)$a2$n', '(T*m)$a2$n']
_FIELDS_FILE = 'fields.h5'
_GEOM_DIR = 'geometry'
_GEOM_FILE = 'geometry.h5'
_KICK_FILE = 'kickMap.h5'
_KICK_SDDS_FILE = 'kickMap.sdds'
_KICK_TEXT_FILE = 'kickMap.txt'
_METHODS = ['get_field', 'get_field_integrals', 'get_geom', 'get_kick_map', 'save_field']
_SIM_REPORTS = ['geometry', 'reset', 'solver']
_REPORTS = ['geometry', 'kickMap', 'reset', 'solver']
_REPORT_RES_MAP = PKDict(
    reset='geometry',
    solver='geometry',
)
_SIM_DATA, SIM_TYPE, _SCHEMA = sirepo.sim_data.template_globals()
_SDDS_INDEX = 0

_ZERO = [0, 0, 0]

GEOM_PYTHON_FILE = 'geometry.py'
KICK_PYTHON_FILE = 'kickMap.py'
RADIA_EXPORT_FILE = 'radia_export.py'
MPI_SUMMARY_FILE = 'mpi-info.json'
VIEW_TYPES = [_SCHEMA.constants.viewTypeObjects, _SCHEMA.constants.viewTypeFields]

# cfg contains sdds instance
_cfg = PKDict(sdds=None)


def background_percent_complete(report, run_dir, is_running):
    res = PKDict(
        percentComplete=0,
        frameCount=0,
    )
    data = simulation_db.read_json(run_dir.join(template_common.INPUT_BASE_NAME))
    if is_running:
        #data = simulation_db.read_json(run_dir.join(template_common.INPUT_BASE_NAME))
        res.percentComplete = 0.0  #_compute_percent_complete(data, last_element)
        return res
    res.percentComplete = 100
    res.frameCount = 1
    if report == 'solver':
        res.solution = _read_solution(data.simulationId)
    #if report == 'kickMap':
    #    res = _kick_map_plot(data.simulationId, data.models.kickMap)
    return res



def create_archive(sim):
    if sim.filename.endswith('dat'):
        return sirepo.http_reply.gen_file_as_attachment(
            _dmp_file(sim.id),
            content_type='application/octet-stream',
            filename=sim.filename,
        )
    return False


def extract_report_data(run_dir, sim_in):
    assert sim_in.report in _REPORTS, 'unknown report: {}'.format(sim_in.report)<commit_msg>WIP #2347 - cleanup<commit_after># -*- coding: utf-8 -*-
u"""Radia execution template.

All Radia calls have to be done from here, NOT in jinja files, because otherwise the
Radia "instance" goes away and references no longer have any meaning.

:copyright: Copyright (c) 2017-2018 RadiaSoft LLC.  All Rights Reserved.
:license: http://www.apache.org/licenses/LICENSE-2.0.html
"""
from __future__ import division
from pykern import pkcollections
from pykern.pkcollections import PKDict
from pykern import pkio
from pykern import pkjinja
from pykern.pkdebug import pkdc, pkdp
from scipy.spatial.transform import Rotation
from sirepo import simulation_db
from sirepo.template import template_common
from sirepo.template import radia_tk
from sirepo.template import radia_examples
import h5py
import math
import numpy
import re
import sdds
import sirepo.csv
import sirepo.sim_data
import sirepo.util
import time
import uuid

_BEAM_AXIS_ROTATIONS = PKDict(
    x=Rotation.from_matrix([[0, 0, 1], [0, 1, 0], [-1, 0, 0]]),
    y=Rotation.from_matrix([[1, 0, 0], [0, 0, -1], [0, 1, 0]]),
    z=Rotation.from_matrix([[1, 0, 0], [0, 1, 0], [0, 0, 1]])
)

_BEAM_AXIS_VECTORS = PKDict(
    x=[1, 0, 0],
    y=[0, 1, 0],
    z=[0, 0, 1]
)

_GAP_AXIS_MAP = PKDict(
    x='z',
    y='z',
    z='y'
)

_DMP_FILE = 'geometry.dat'

# Note that these column names and units are required by elegant
_FIELD_MAP_COLS = ['x', 'y', 'z', 'Bx', 'By', 'Bz']
_FIELD_MAP_UNITS = ['m', 'm', 'm', 'T', 'T', 'T']
_KICK_MAP_COLS = ['x', 'y', 'xpFactor', 'ypFactor']
_KICK_MAP_UNITS = ['m', 'm', '(T*m)$a2$n', '(T*m)$a2$n']
_FIELDS_FILE = 'fields.h5'
_GEOM_DIR = 'geometry'
_GEOM_FILE = 'geometry.h5'
_KICK_FILE = 'kickMap.h5'
_KICK_SDDS_FILE = 'kickMap.sdds'
_KICK_TEXT_FILE = 'kickMap.txt'
_METHODS = ['get_field', 'get_field_integrals', 'get_geom', 'get_kick_map', 'save_field']
_SIM_REPORTS = ['geometry', 'reset', 'solver']
_REPORTS = ['geometry', 'kickMap', 'reset', 'solver']
_REPORT_RES_MAP = PKDict(
    reset='geometry',
    solver='geometry',
)
_SIM_DATA, SIM_TYPE, _SCHEMA = sirepo.sim_data.template_globals()
_SDDS_INDEX = 0

_ZERO = [0, 0, 0]

GEOM_PYTHON_FILE = 'geometry.py'
KICK_PYTHON_FILE = 'kickMap.py'
RADIA_EXPORT_FILE = 'radia_export.py'
MPI_SUMMARY_FILE = 'mpi-info.json'
VIEW_TYPES = [_SCHEMA.constants.viewTypeObjects, _SCHEMA.constants.viewTypeFields]

# cfg contains sdds instance
_cfg = PKDict(sdds=None)


def background_percent_complete(report, run_dir, is_running):
    res = PKDict(
        percentComplete=0,
        frameCount=0,
    )
    data = simulation_db.read_json(run_dir.join(template_common.INPUT_BASE_NAME))
    if is_running:
        res.percentComplete = 0.0
        return res
    res.percentComplete = 100
    res.frameCount = 1
    if report == 'solver':
        res.solution = _read_solution(data.simulationId)
    #if report == 'kickMap':
    #    res = _kick_map_plot(data.simulationId, data.models.kickMap)
    return res


def create_archive(sim):
    if sim.filename.endswith('dat'):
        return sirepo.http_reply.gen_file_as_attachment(
            _dmp_file(sim.id),
            content_type='application/octet-stream',
            filename=sim.filename,
        )
    return False


def extract_report_data(run_dir, sim_in):
    assert sim_in.report in _REPORTS, 'unknown report: {}'.format(sim_in.report)
{'fraction_non_alphanumeric': 0.1350117785630153, 'fraction_numerical': 0.019729093050647822, 'mean_word_length': 4.278166278166278, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1326105', 'n_tokens_mistral': 2704, 'n_tokens_neox': 2567, 'n_words': 611}