text:
from os import close
from numpy.lib.polynomial import poly
import torch
import vtk
import numpy as np
import scipy.sparse as sp
import argparse
import math
import heapq
import pickle
import os
#Set Torch device
device = "cpu"
if torch.cuda.is_available():
    device = "cuda"

class Mesh:
    def __init__(self, v=None, f=None):
        self.v = v
        self.f = f
        self.polydata = None

        if self.v is not None and self.f is not None:
            self.polydata = vtk.vtkPolyData()

            points = vtk.vtkPoints()
            for vertex in self.v:
                points.InsertNextPoint(vertex)
            self.polydata.SetPoints(points)

            cellArray = vtk.vtkCellArray()
            for face in self.f:
                cell = vtk.vtkTriangle()
                cell.GetPointIds().SetId(0, face[0])
                cell.GetPointIds().SetId(1, face[1])
                cell.GetPointIds().SetId(2, face[2])
                cellArray.InsertNextCell(cell)
            self.polydata.SetPolys(cellArray)

    def SetPolyData(self, polydata):
        # print(polydata.GetNumberOfPoints())

        nPoints = polydata.GetNumberOfPoints()
        nFaces = polydata.GetNumberOfCells()

        vertices = []
        for vid in range(nPoints):
            v = polydata.GetPoint(vid)
            vertices.append(v)
        self.v = np.array(vertices)

        faces = []
        for fid in range(nFaces):
            cell = polydata.GetCell(fid)
            faces.append( [cell.GetPointId(0), cell.GetPointId(1), cell.GetPointId(2)] )
            # print(cell.GetPointId(0))

        self.f = np.array(faces)

    def is_on(self, a, b, c):
        "Return true iff point c intersects the line segment from a to b."
        # (or the degenerate case that all 3 points are coincident)

        def collinear(a, b, c):
            "Return true iff a, b, and c all lie on the same line."
            return (b[0] - a[0]) * (c[1] - a[1]) == (c[0] - a[0]) * (b[1] - a[1])

        def within(p, q, r):
            "Return true iff q is between p and r (inclusive)."
            return p <= q <= r or r <= q <= p

        return (collinear(a, b, c)
                and (within(a[0], c[0], b[0]) if a[0] != b[0] else
                    within(a[1], c[1], b[1])))

    def nearest(self, v_samples):

        nearest_faces = []
        "nearest_part tells you whether the closest point in triangle abc is in the interior (0), on an edge (ab:1,bc:2,ca:3), or a vertex (a:4,b:5,c:6)"

        nearest_parts = []
        nearest_vertices = []

        cellLocator = vtk.vtkCellLocator()
        cellLocator.SetDataSet(self.polydata)
        cellLocator.BuildLocator()

        for vertex in v_samples:

            closestPoint = [0, 0, 0]
            cellId = vtk.reference(0)

            cellLocator.FindClosestPoint(vertex, closestPoint, cellId, vtk.mutable(-1), vtk.mutable(-1.0))

            #Check cloesstpoint In the triangle (0), on the Edge (1), On the Vertex(2)
            part = 0
            triangle = self.polydata.GetCell(cellId)

            #TODO  : Add Part checker!!!
            a = np.array(self.polydata.GetPoint(triangle.GetPointId(0)))
            b = np.array(self.polydata.GetPoint(triangle.GetPointId(1)))
            c = np.array(self.polydata.GetPoint(triangle.GetPointId(2)))
            target = np.array(closestPoint)
            if np.array_equal(  a ,target ):
                part = 4
            elif np.array_equal( b  , target ):
                part = 5
            elif np.array_equal( c  , target ):
                part = 6
            elif self.is_on(a, b, target):
                part = 1
            elif self.is_on(b, c, target):
                part = 2
            elif self.is_on(c, a, target):
                part = 3

            #TODO : Debug This Part Later
            weights = [0, 0, 0]
            inside = triangle.EvaluatePosition(closestPoint, [0, 0, 0], vtk.mutable(0), [0,0,0], vtk.mutable(0.0), weights)
            # if weights[0] == 1.0: part = 4
            # elif weights[1] == 1.0 : part = 5
            # elif weights[2] == 1.0 : part = 6

            if part == 1:
                print(inside, "\t", weights)

            nearest_faces.append(cellId)
            nearest_parts.append(part) #???
            nearest_vertices.append(closestPoint)

        # print(v_samples, v_samples.shape, self.polydata.GetNumberOfPoints())

        nearest_faces = np.array(nearest_faces)
        nearest_parts = np.array(nearest_parts)
        nearest_vertices = np.array(nearest_vertices)
        return nearest_faces, nearest_parts, nearest_vertices

def row(A):
    return A.reshape((1, -1))

def col(A):
    return A.reshape((-1, 1))

def _get_sparse_transform(faces, num_original_verts):
    verts_left = np.unique(faces.flatten())
    IS = np.arange(len(verts_left))
    JS = verts_left
    data = np.ones(len(JS))

    mp = np.arange(0, np.max(faces.flatten()) + 1)
    mp[JS] = IS
    new_faces = mp[faces.copy().flatten()].reshape((-1, 3))

    ij = np.vstack((IS.flatten(), JS.flatten()))
    mtx = sp.csc_matrix((data, ij),
                        shape=(len(verts_left), num_original_verts))

    return (new_faces, mtx)

def setup_deformation_transfer(source, target, use_normals=False):
    rows = np.zeros(3 * target.v.shape[0])
    cols = np.zeros(3 * target.v.shape[0])
    coeffs_v = np.zeros(3 * target.v.shape[0])
    coeffs_n = np.zeros(3 * target.v.shape[0])

    # nearest_faces, nearest_parts, nearest_vertices = source.compute_aabb_tree().nearest(target.v, True)
    nearest_faces, nearest_parts, nearest_vertices = source.nearest(target.v) # Not Sure For this

    nearest_faces = nearest_faces.ravel().astype(np.int64)
    nearest_parts = nearest_parts.ravel().astype(np.int64) #Wjat is "Parts"?
    nearest_vertices = nearest_vertices.ravel()

    for i in range(target.v.shape[0]):
        # Closest triangle index
        f_id = nearest_faces[i]
        # Closest triangle vertex ids
        nearest_f = source.f[f_id]

        # Closest surface point
        nearest_v = nearest_vertices[3 * i:3 * i + 3]
        # Distance vector to the closest surface point
        dist_vec = target.v[i] - nearest_v

        rows[3 * i:3 * i + 3] = i * np.ones(3)
        cols[3 * i:3 * i + 3] = nearest_f

        n_id = nearest_parts[i]
        if n_id == 0:
            # Closest surface point in triangle
            A = np.vstack((source.v[nearest_f])).T
            coeffs_v[3 * i:3 * i + 3] = np.linalg.lstsq(A, nearest_v, rcond=-1)[0]
        elif n_id > 0 and n_id <= 3:
            # Closest surface point on edge
            A = np.vstack((source.v[nearest_f[n_id - 1]], source.v[nearest_f[n_id % 3]])).T
            tmp_coeffs = np.linalg.lstsq(A, target.v[i], rcond=-1)[0]
            coeffs_v[3 * i + n_id - 1] = tmp_coeffs[0]
            coeffs_v[3 * i + n_id % 3] = tmp_coeffs[1]
        else:
            # Closest surface point a vertex
            coeffs_v[3 * i + n_id - 4] = 1.0

    matrix = sp.csc_matrix((coeffs_v, (rows, cols)), shape=(target.v.shape[0], source.v.shape[0]))
    return matrix

def get_vert_connectivity(mesh_v, mesh_f):
    """Returns a sparse matrix (of size #verts x #verts) where each nonzero
    element indicates a neighborhood relation. For example, if there is a
    nonzero element in position (15,12), that means vertex 15 is connected
    by an edge to vertex 12."""

    vpv = sp.csc_matrix((len(mesh_v), len(mesh_v)))

    # for each column in the faces...
    for i in range(3):
        IS = mesh_f[:, i]
        JS = mesh_f[:, (i + 1) % 3]
        data = np.ones(len(IS))
        ij = np.vstack((row(IS.ravel()), row(JS.ravel())))
        mtx = sp.csc_matrix((data, ij), shape=vpv.shape)
        vpv = vpv + mtx + mtx.T

    return vpv

def get_vertices_per_edge(mesh_v, mesh_f):
    """Returns an Ex2 array of adjacencies between vertices, where
    each element in the array is a vertex index. Each edge is included
    only once. If output of get_faces_per_edge is provided, this is used to
    avoid call to get_vert_connectivity()"""

    vc = sp.coo_matrix(get_vert_connectivity(mesh_v, mesh_f))
    result = np.hstack((col(vc.row), col(vc.col)))
    result = result[result[:, 0] < result[:, 1]]  # for uniqueness

    return result

def vertex_quadrics(mesh):
    """Computes a quadric for each vertex in the Mesh.
    Returns:
       v_quadrics: an (N x 4 x 4) array, where N is # vertices.
    """

    # Allocate quadrics
    v_quadrics = np.zeros((len(mesh.v), 4, 4,))

    # For each face...
    for f_idx in range(len(mesh.f)):

        # Compute normalized plane equation for that face
        vert_idxs = mesh.f[f_idx]
        verts = np.hstack((mesh.v[vert_idxs], np.array([1, 1, 1]).reshape(-1, 1)))
        u, s, v = np.linalg.svd(verts)
        eq = v[-1, :].reshape(-1, 1)
        eq = eq / (np.linalg.norm(eq[0:3]))

        # Add the outer product of the plane equation to the
        # quadrics of the vertices for this face
        for k in range(3):
            v_quadrics[mesh.f[f_idx, k], :, :] += np.outer(eq, eq)

    return v_quadrics

def qslim_decimator_transformer(mesh, factor=None, n_verts_desired=None):
    """Return a simplified version of this mesh.
    A Qslim-style approach is used here.
    :param factor: fraction of the original vertices to retain
    :param n_verts_desired: number of the original vertices to retain
    :returns: new_faces: An Fx3 array of faces, mtx: Transformation matrix
    """

    if factor is None and n_verts_desired is None:
        raise Exception('Need either factor or n_verts_desired.')

    if n_verts_desired is None:
        n_verts_desired = math.ceil(len(mesh.v) * factor)

    Qv = vertex_quadrics(mesh)

    # fill out a sparse matrix indicating vertex-vertex adjacency
    # from psbody.mesh.topology.connectivity import get_vertices_per_edge
    vert_adj = get_vertices_per_edge(mesh.v, mesh.f)
    # vert_adj = sp.lil_matrix((len(mesh.v), len(mesh.v)))
    # for f_idx in range(len(mesh.f)):
    #     vert_adj[mesh.f[f_idx], mesh.f[f_idx]] = 1

    vert_adj = sp.csc_matrix((vert_adj[:, 0] * 0 + 1, (vert_adj[:, 0], vert_adj[:, 1])), shape=(len(mesh.v), len(mesh.v)))
    vert_adj = vert_adj + vert_adj.T
    vert_adj = vert_adj.tocoo()

    def collapse_cost(Qv, r, c, v):
        Qsum = Qv[r, :, :] + Qv[c, :, :]
        p1 = np.vstack((v[r].reshape(-1, 1), np.array([1]).reshape(-1, 1)))
        p2 = np.vstack((v[c].reshape(-1, 1), np.array([1]).reshape(-1, 1)))

        destroy_c_cost = p1.T.dot(Qsum).dot(p1)
        destroy_r_cost = p2.T.dot(Qsum).dot(p2)
        result = {
            'destroy_c_cost': destroy_c_cost,
            'destroy_r_cost': destroy_r_cost,
            'collapse_cost': min([destroy_c_cost, destroy_r_cost]),
            'Qsum': Qsum}
        return result

    # construct a queue of edges with costs
    queue = []
    for k in range(vert_adj.nnz):
        r = vert_adj.row[k]
        c = vert_adj.col[k]

        if r > c:
            continue

        cost = collapse_cost(Qv, r, c, mesh.v)['collapse_cost']
        heapq.heappush(queue, (cost, (r, c)))

    # decimate
    collapse_list = []
    nverts_total = len(mesh.v)
    faces = mesh.f.copy()
    while nverts_total > n_verts_desired:
        e = heapq.heappop(queue)
        r = e[1][0]
        c = e[1][1]
        if r == c:
            continue

        cost = collapse_cost(Qv, r, c, mesh.v)
        if cost['collapse_cost'] > e[0]:
            heapq.heappush(queue, (cost['collapse_cost'], e[1]))
            # print 'found outdated cost, %.2f < %.2f' % (e[0], cost['collapse_cost'])
            continue
        else:

            # update old vert idxs to new one,
            # in queue and in face list
            if cost['destroy_c_cost'] < cost['destroy_r_cost']:
                to_destroy = c
                to_keep = r
            else:
                to_destroy = r
                to_keep = c

            collapse_list.append([to_keep, to_destroy])

            # in our face array, replace "to_destroy" vertidx with "to_keep" vertidx
            np.place(faces, faces == to_destroy, to_keep)

            # same for queue
            which1 = [idx for idx in range(len(queue)) if queue[idx][1][0] == to_destroy]
            which2 = [idx for idx in range(len(queue)) if queue[idx][1][1] == to_destroy]
            for k in which1:
                queue[k] = (queue[k][0], (to_keep, queue[k][1][1]))
            for k in which2:
                queue[k] = (queue[k][0], (queue[k][1][0], to_keep))

            Qv[r, :, :] = cost['Qsum']
            Qv[c, :, :] = cost['Qsum']

            a = faces[:, 0] == faces[:, 1]
            b = faces[:, 1] == faces[:, 2]
            c = faces[:, 2] == faces[:, 0]

            # remove degenerate faces
            def logical_or3(x, y, z):
                return np.logical_or(x, np.logical_or(y, z))

            faces_to_keep = np.logical_not(logical_or3(a, b, c))
            faces = faces[faces_to_keep, :].copy()

        nverts_total = (len(np.unique(faces.flatten())))

    new_faces, mtx = _get_sparse_transform(faces, len(mesh.v))
    return new_faces, mtx

def generate_transform_matrices(mesh, factors):
    """Generates len(factors) meshes, each of them is scaled by factors[i] and
       computes the transformations between them.

    Returns:
       M: a set of meshes downsampled from mesh by a factor specified in factors.
       A: Adjacency matrix for each of the meshes
       D: csc_matrix Downsampling transforms between each of the meshes
       U: Upsampling transforms between each of the meshes
       F: a list of faces
    """

    factors = map(lambda x: 1.0 / x, factors)
    M, A, D, U, F, V = [], [], [], [], [], []
    F.append(mesh.f)  # F[0]
    V.append(mesh.v)
    A.append(get_vert_connectivity(mesh.v, mesh.f).astype('float32'))  # A[0]
    M.append(mesh)  # M[0]

    for factor in factors:
        print(factor)
        ds_f, ds_D = qslim_decimator_transformer(M[-1], factor=factor)
        D.append(ds_D.astype('float32'))
        new_mesh_v = ds_D.dot(M[-1].v)
        new_mesh = Mesh(v=new_mesh_v, f=ds_f)
        F.append(new_mesh.f)
        V.append(new_mesh.v)
        M.append(new_mesh)
        A.append(get_vert_connectivity(new_mesh.v, new_mesh.f).astype('float32'))
        U.append(setup_deformation_transfer(M[-1], M[-2]).astype('float32'))

    return M, A, D, U, F, V

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description = 'Generate Transform Matrix')
    parser.add_argument('--template', type = str,  default="template/toothTemplate.vtp")
    parser.add_argument("--output", type=str, default = "toothTransform")
    # parser.add_argument('--output', type=str, default="//192.168.0.113/Imagoworks/AITeam/EJSHIM/dcmtosinglevtiforcau")
    args = parser.parse_args()

    output = os.path.join( os.path.dirname(args.template) , args.output+".pkl")
    print("Save Output To ", output)
    os.makedirs(os.path.dirname(output), exist_ok=True)

    #Read Template
    reader = vtk.vtkXMLPolyDataReader()
    reader.SetFileName(args.template)
    reader.Update()

    templatePoly = reader.GetOutput()

    mesh = Mesh()
    mesh.SetPolyData(templatePoly)

    print("Generating Transform Matrices...")
    ds_factors = [4, 4, 4, 4, 4, 4]
    _, A, D, U, F = generate_transform_matrices(mesh, ds_factors)

    tmp = {'face': F, 'adj': A, 'down_transform': D, 'up_transform': U}
    with open(output, 'wb') as fp:
        pickle.dump(tmp, fp)
    print('Done!')

reason: import numpy,from numpy,import scipy


text:
import yaml
import argparse
import importlib
import psutil
import os
from os import path
import argparse
import joblib
from time import sleep

import numpy as np
from gym.spaces import Dict

from rlkit.envs import get_env
import rlkit.torch.pytorch_util as ptu
from rlkit.launchers.launcher_util import setup_logger, set_seed
from rlkit.torch.sac.policies import ReparamTanhMultivariateGaussianPolicy
from rlkit.torch.irl.policy_optimizers.sac import NewSoftActorCritic
from rlkit.torch.networks import FlattenMlp, Mlp
from rlkit.torch.irl.disc_models.airl_disc import StandardAIRLDisc, ResNetAIRLDisc, ThreeWayResNetAIRLDisc
from rlkit.envs.wrappers import ScaledEnv
from rlkit.state_matching_algs.fixed_dist_disc_train_alg import FixedDistDiscTrainAlg
from rlkit.state_matching_algs.three_way_fixed_dist_disc_train_alg import ThreeWayFixedDistDiscTrainAlg

def experiment(variant):
    expert_buffer = joblib.load(variant['exp_xy_data_path'])['xy_data']
    policy_buffer = joblib.load(variant['pol_xy_data_path'])['xy_data']

    # set up the discriminator models
    if variant['threeway']:
        disc_model_class = ThreeWayResNetAIRLDisc
    else:
        if variant['use_resnet_disc']:
            disc_model_class = ResNetAIRLDisc
        else:
            disc_model_class = StandardAIRLDisc
    disc_model = disc_model_class(
        2, # obs is just x-y pos
        num_layer_blocks=variant['disc_num_blocks'],
        hid_dim=variant['disc_hid_dim'],
        hid_act=variant['disc_hid_act'],
        use_bn=variant['disc_use_bn'],
        clamp_magnitude=variant['disc_clamp_magnitude']
    )
    print(disc_model)
    print(disc_model.clamp_magnitude)

    # set up the AIRL algorithm
    alg_class = ThreeWayFixedDistDiscTrainAlg if variant['threeway'] else FixedDistDiscTrainAlg
    algorithm = alg_class(
        disc_model,
        expert_buffer,
        policy_buffer,
        **variant['algo_params']
    )
    print(algorithm.disc_optimizer.defaults['lr'])

    # train
    if ptu.gpu_enabled():
        algorithm.cuda()
    algorithm.train()

    return 1

if __name__ == '__main__':
    # Arguments
    parser = argparse.ArgumentParser()
    parser.add_argument('-e', '--experiment', help='experiment specification file')
    args = parser.parse_args()
    with open(args.experiment, 'r') as spec_file:
        spec_string = spec_file.read()
        exp_specs = yaml.load(spec_string)

    if exp_specs['use_gpu']:
        print('\n\nUSING GPU\n\n')
        ptu.set_gpu_mode(True)
    exp_id = exp_specs['exp_id']
    exp_prefix = exp_specs['exp_name']
    seed = exp_specs['seed']
    set_seed(seed)
    setup_logger(exp_prefix=exp_prefix, exp_id=exp_id, variant=exp_specs)

    experiment(exp_specs)

ext: py

reason: import numpy


text:
# SPDX-FileCopyrightText: 2021 Division of Intelligent Medical Systems, DKFZ
# SPDX-FileCopyrightText: 2021 Janek Groehl
# SPDX-License-Identifier: MIT

from simpa import Tags
import simpa as sp
import numpy as np
from skimage.data import shepp_logan_phantom
from scipy.ndimage import zoom

# FIXME temporary workaround for newest Intel architectures
import os
os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"

# If VISUALIZE is set to True, the simulation result will be plotted
VISUALIZE = True

# TODO: Please make sure that a valid path_config.env file is located in your home directory, or that you
#  point to the correct file in the PathManager().
path_manager = sp.PathManager()

target_spacing = 1.0

label_mask = shepp_logan_phantom()

label_mask = np.digitize(label_mask, bins=np.linspace(0.0, 1.0, 11), right=True)

label_mask = np.reshape(label_mask, (400, 1, 400))

input_spacing = 0.2
segmentation_volume_tiled = np.tile(label_mask, (1, 128, 1))
segmentation_volume_mask = np.round(zoom(segmentation_volume_tiled, input_spacing/target_spacing,
                                         order=0)).astype(int)

def segmentation_class_mapping():
    ret_dict = dict()
    ret_dict[0] = sp.TISSUE_LIBRARY.heavy_water()
    ret_dict[1] = sp.TISSUE_LIBRARY.blood()
    ret_dict[2] = sp.TISSUE_LIBRARY.epidermis()
    ret_dict[3] = sp.TISSUE_LIBRARY.muscle()
    ret_dict[4] = sp.TISSUE_LIBRARY.mediprene()
    ret_dict[5] = sp.TISSUE_LIBRARY.ultrasound_gel()
    ret_dict[6] = sp.TISSUE_LIBRARY.heavy_water()
    ret_dict[7] = (sp.MolecularCompositionGenerator()
                   .append(sp.MOLECULE_LIBRARY.oxyhemoglobin(0.01))
                   .append(sp.MOLECULE_LIBRARY.deoxyhemoglobin(0.01))
                   .append(sp.MOLECULE_LIBRARY.water(0.98))
                   .get_molecular_composition(sp.SegmentationClasses.COUPLING_ARTIFACT))
    ret_dict[8] = sp.TISSUE_LIBRARY.heavy_water()
    ret_dict[9] = sp.TISSUE_LIBRARY.heavy_water()
    ret_dict[10] = sp.TISSUE_LIBRARY.heavy_water()
    return ret_dict

settings = sp.Settings()
settings[Tags.SIMULATION_PATH] = path_manager.get_hdf5_file_save_path()
settings[Tags.VOLUME_NAME] = "SegmentationTest"
settings[Tags.RANDOM_SEED] = 1234
settings[Tags.WAVELENGTHS] = [700]
settings[Tags.SPACING_MM] = target_spacing
settings[Tags.DIM_VOLUME_X_MM] = 400 / (target_spacing / input_spacing)
settings[Tags.DIM_VOLUME_Y_MM] = 128 / (target_spacing / input_spacing)
settings[Tags.DIM_VOLUME_Z_MM] = 400 / (target_spacing / input_spacing)

settings.set_volume_creation_settings({
    Tags.INPUT_SEGMENTATION_VOLUME: segmentation_volume_mask,
    Tags.SEGMENTATION_CLASS_MAPPING: segmentation_class_mapping(),

})

settings.set_optical_settings({
    Tags.OPTICAL_MODEL_NUMBER_PHOTONS: 1e8,
    Tags.OPTICAL_MODEL_BINARY_PATH: path_manager.get_mcx_binary_path(),
    Tags.ILLUMINATION_TYPE: Tags.ILLUMINATION_TYPE_MSOT_ACUITY_ECHO,
    Tags.LASER_PULSE_ENERGY_IN_MILLIJOULE: 50,
})

pipeline = [
    sp.SegmentationBasedVolumeCreationAdapter(settings),
    sp.MCXAdapter(settings)
]

sp.simulate(pipeline, settings, sp.RSOMExplorerP50(element_spacing_mm=1.0))

if Tags.WAVELENGTH in settings:
    WAVELENGTH = settings[Tags.WAVELENGTH]
else:
    WAVELENGTH = 700

if VISUALIZE:
    sp.visualise_data(path_to_hdf5_file=path_manager.get_hdf5_file_save_path() + "/" + "SegmentationTest" + ".hdf5",
                      wavelength=WAVELENGTH,
                      show_initial_pressure=True,
                      show_segmentation_map=True)

max_stars_repo_name: IMSY-DKFZ/simpa

max_issues_repo_name: jgroehl/simpa

max_forks_repo_name: IMSY-DKFZ/simpa

reason: import numpy,from scipy


text:
import unittest

from gym.spaces import Box
from gym.spaces import Discrete
import numpy as np

from garage.envs.wrappers import Resize
from garage.misc.overrides import overrides
from garage.tf.envs import TfEnv
from tests.fixtures.envs.dummy import DummyDiscrete2DEnv

class TestResize(unittest.TestCase):
    @overrides
    def setUp(self):
        self.width = 16
        self.height = 16
        self.env = TfEnv(DummyDiscrete2DEnv())
        self.env_r = TfEnv(
            Resize(DummyDiscrete2DEnv(), width=self.width, height=self.height))

    def test_resize_invalid_environment_type(self):
        with self.assertRaises(ValueError):
            self.env.observation_space = Discrete(64)
            Resize(self.env, width=self.width, height=self.height)

    def test_resize_invalid_environment_shape(self):
        with self.assertRaises(ValueError):
            self.env.observation_space = Box(
                low=0, high=255, shape=(4, ), dtype=np.uint8)
            Resize(self.env, width=self.width, height=self.height)

    def test_resize_output_observation_space(self):
        assert self.env_r.observation_space.shape == (self.width, self.height)

    def test_resize_output_reset(self):
        assert self.env_r.reset().shape == (self.width, self.height)

    def test_resize_output_step(self):
        self.env_r.reset()
        obs_r, _, _, _ = self.env_r.step(1)
        assert obs_r.shape == (self.width, self.height)


text:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

from pathlib import Path
from edibles import PYTHONDIR

from edibles.utils.ISLineFitter import ISLineFitter, ISLineModel

# Load the benchmark data and run it through ISLineFitter. Then compare results.

this_datadir = Path(PYTHONDIR) / "data" / "voigt_benchmarkdata"
filename = this_datadir / "omiper.m95.7698.txt"
v_resolution = 0.56 # km/s
#print(filename)
arrays = np.genfromtxt(filename,skip_header=1)

wave = arrays[:,0]
flux = arrays[:,1]
#print(wave, flux)

# fitting
fitter = ISLineFitter(wave, flux, v_resolution=0.56, normalized=True)
known_n_components = 5
fit_result = fitter.fit(species='KI', windowsize=0.5, n_anchors=4,
                        known_n_components=known_n_components, WaveMax=7700, WaveMin=7695)

# results
print(fit_result.fit_report())

# plot best fit model
plt.plot(fitter.wave2fit, fitter.flux2fit)
plt.plot(fitter.wave2fit, fit_result.best_fit)
plt.xlabel("Overall")
plt.show()

# plot best fit model of each components
comps = fit_result.eval_components(x=wave)
continuum = comps["cont"]
plt.plot(fitter.wave2fit, fitter.flux2fit)
singe_component = ISLineModel(1, lam_0=fitter.air_wavelength,
                              fjj=fitter.oscillator_strength,
                              gamma=fitter.gamma,
                              v_res=fitter.v_res)
for i in range(known_n_components):
    pars = singe_component.make_params(b_Cloud0=fit_result.params["b_Cloud%i" % (i)].value,
                                       N_Cloud0=fit_result.params["N_Cloud%i" % (i)].value,
                                       V_off_Cloud0=fit_result.params["V_off_Cloud%i" % (i)].value)
    plt.plot(fitter.wave2fit, singe_component.eval(params=pars, x=fitter.wave2fit), label="Comp %i" % (i))
plt.xlabel("Individual Components")
plt.legend()
plt.show()

max_stars_repo_name: jancami/edibles

max_issues_repo_name: jancami/edibles

max_forks_repo_name: jancami/edibles


text:
import sys
import numpy as np

def compute_t(lines1, lines2):
    recalls1 = [[0, 0], [0, 0], [0, 0]]
    precisions1 = [[0, 0], [0, 0], [0, 0]]
    for s in range(3):
        start = s
        if start == 2:
            start += 1
        for i in range(start*len(lines1)/4, (start+1)*len(lines1)/4):
            line = lines1[i]
            recall_num = float(line.split('(')[1].split('/')[0])
            recall_den = float(line.split(')')[0].split('/')[1])
            recalls1[s][0] += recall_num
            recalls1[s][1] += recall_den
            precision_num = float(line.split('(')[2].split('/')[0])
            precision_den = float(line.split(')')[1].split('/')[1])
            precisions1[s][0] += precision_num
            precisions1[s][1] += precision_den
    f1s1 = [2*(p[0]/float(p[1]))*(r[0]/float(r[1]))/(p[0]/float(p[1])+r[0]/float(r[1])) for p, r in zip(precisions1, recalls1)]
    recalls2 = [[0, 0], [0, 0], [0, 0]]
    precisions2 = [[0, 0], [0, 0], [0, 0]]
    for s in range(3):
        start = s
        if start == 2:
            start += 1
        for i in range(start*len(lines2)/4, (start+1)*len(lines2)/4):
            line = lines2[i]
            recall_num = float(line.split('(')[1].split('/')[0])
            recall_den = float(line.split(')')[0].split('/')[1])
            recalls2[s][0] += recall_num
            recalls2[s][1] += recall_den
            precision_num = float(line.split('(')[2].split('/')[0])
            precision_den = float(line.split(')')[1].split('/')[1])
            precisions2[s][0] += precision_num
            precisions2[s][1] += precision_den
    f1s2 = [2*(p[0]/float(p[1]))*(r[0]/float(r[1]))/(p[0]/float(p[1])+r[0]/float(r[1])) for p, r in zip(precisions2, recalls2)]
    conllf1_1 = np.mean(f1s1)
    conllf1_2 = np.mean(f1s2)
    tval = conllf1_1-conllf1_2
    return tval

if __name__ == '__main__':
    f1 = open(sys.argv[1])
    f2 = open(sys.argv[2])
    lines1 = f1.readlines()
    lines2 = f2.readlines()
    original_t = compute_t(lines1, lines2)
    print('T: ' + str(original_t))
    greater = 0
    total = 10000
    for _ in range(total):
        lines1new = []
        lines2new = []
        for line1, line2 in zip(lines1, lines2):
            if np.random.rand() >= 0.5:
                lines1new.append(line1)
                lines2new.append(line2)
            else:
                lines1new.append(line2)
                lines2new.append(line1)
        tval = compute_t(lines1new, lines2new)
        if tval >= original_t:
            greater += 1
    print('p-value: ' + str(float(greater)/total))


text:
import bob.ip.base
import numpy
import logging

from .Base import Base
from sklearn.base import TransformerMixin, BaseEstimator

logger = logging.getLogger("bob.bio.face")
from bob.bio.base import load_resource
from .Scale import scale

class FaceCrop(Base):
    """Crops the face according to the given annotations.

    This class is designed to perform a geometric normalization of the face based
    on the eye locations, using :py:class:`bob.ip.base.FaceEyesNorm`. Usually,
    when executing the :py:meth:`crop_face` function, the image and the eye
    locations have to be specified. There, the given image will be transformed
    such that the eye locations will be placed at specific locations in the
    resulting image. These locations, as well as the size of the cropped image,
    need to be specified in the constructor of this class, as
    ``cropped_positions`` and ``cropped_image_size``.

    Some image databases do not provide eye locations, but rather bounding boxes.
    This is not a problem at all.
    Simply define the coordinates, where you want your ``cropped_positions`` to
    be in the cropped image, by specifying the same keys in the dictionary that
    will be given as ``annotations`` to the :py:meth:`crop_face` function.

    .. note::

      These locations can even be outside of the cropped image boundary, i.e.,
      when the crop should be smaller than the annotated bounding boxes.

    Sometimes, databases provide pre-cropped faces, where the eyes are located at
    (almost) the same position in all images. Usually, the cropping does not
    conform with the cropping that you like (i.e., image resolution is wrong, or
    too much background information). However, the database does not provide eye
    locations (since they are almost identical for all images). In that case, you
    can specify the ``fixed_positions`` in the constructor, which will be taken
    instead of the ``annotations`` inside the :py:meth:`crop_face` function (in
    which case the ``annotations`` are ignored).

    Sometimes, the crop of the face is outside of the original image boundaries.
    Usually, these pixels will simply be left black, resulting in sharp edges in
    the image. However, some feature extractors do not like these sharp edges. In
    this case, you can set the ``mask_sigma`` to copy pixels from the valid
    border of the image and add random noise (see
    :py:func:`bob.ip.base.extrapolate_mask`).

    Parameters
    ----------

    cropped_image_size : (int, int)
      The resolution of the cropped image, in order (HEIGHT,WIDTH); if not given,
      no face cropping will be performed

    cropped_positions : dict
      The coordinates in the cropped image, where the annotated points should be
      put to. This parameter is a dictionary with usually two elements, e.g.,
      ``{'reye':(RIGHT_EYE_Y, RIGHT_EYE_X) , 'leye':(LEFT_EYE_Y, LEFT_EYE_X)}``.
      However, also other parameters, such as ``{'topleft' : ..., 'bottomright' :
      ...}`` are supported, as long as the ``annotations`` in the `__call__`
      function are present.

    fixed_positions : dict or None
      If specified, ignore the annotations from the database and use these fixed
      positions throughout.

    mask_sigma : float or None
      Fill the area outside of image boundaries with random pixels from the
      border, by adding noise to the pixel values. To disable extrapolation, set
      this value to ``None``. To disable adding random noise, set it to a
      negative value or 0.

    mask_neighbors : int
      The number of neighbors used during mask extrapolation. See
      :py:func:`bob.ip.base.extrapolate_mask` for details.

    mask_seed : int or None
      The random seed to apply for mask extrapolation.

      .. warning::

         When run in parallel, the same random seed will be applied to all
         parallel processes. Hence, results of parallel execution will differ
         from the results in serial execution.

    allow_upside_down_normalized_faces: bool, optional
      If ``False`` (default), a ValueError is raised when normalized faces are going to be
      upside down compared to input image. This allows you to catch wrong annotations in
      your database easily. If you are sure about your input, you can set this flag to
      ``True``.

    annotator : :any:`bob.bio.base.annotator.Annotator`
      If provided, the annotator will be used if the required annotations are
      missing.

    kwargs
      Remaining keyword parameters passed to the :py:class:`Base` constructor,
      such as ``color_channel`` or ``dtype``.
    """

    def __init__(
        self,
        cropped_image_size,
        cropped_positions,
        fixed_positions=None,
        mask_sigma=None,
        mask_neighbors=5,
        mask_seed=None,
        annotator=None,
        allow_upside_down_normalized_faces=False,
        **kwargs,
    ):
        # call base class constructor
        Base.__init__(self, **kwargs)

        if isinstance(cropped_image_size, int):
            cropped_image_size = (cropped_image_size, cropped_image_size)

        # check parameters
        assert len(cropped_positions) == 2
        if fixed_positions:
            assert len(fixed_positions) == 2

        # copy parameters
        self.cropped_image_size = cropped_image_size
        self.cropped_positions = cropped_positions
        self.cropped_keys = sorted(cropped_positions.keys())
        self.fixed_positions = fixed_positions
        self.mask_sigma = mask_sigma
        self.mask_neighbors = mask_neighbors
        self.mask_seed = mask_seed
        if isinstance(annotator, str):
            annotator = load_resource(annotator, "annotator")
        self.annotator = annotator
        self.allow_upside_down_normalized_faces = allow_upside_down_normalized_faces

        # create objects required for face cropping
        self.cropper = bob.ip.base.FaceEyesNorm(
            crop_size=cropped_image_size,
            right_eye=cropped_positions[self.cropped_keys[0]],
            left_eye=cropped_positions[self.cropped_keys[1]],
        )
        self.cropped_mask = numpy.ndarray(cropped_image_size, bool)

        self._init_non_pickables()

    def _init_non_pickables(self):
        self.mask_rng = (
            bob.core.random.mt19937(self.mask_seed)
            if self.mask_seed is not None
            else bob.core.random.mt19937()
        )
        self.cropper = bob.ip.base.FaceEyesNorm(
            crop_size=self.cropped_image_size,
            right_eye=self.cropped_positions[self.cropped_keys[0]],
            left_eye=self.cropped_positions[self.cropped_keys[1]],
        )

    def crop_face(self, image, annotations=None):
        """Crops the face.
        Executes the face cropping on the given image and returns the cropped
        version of it.

        Parameters
        ----------
        image : 2D :py:class:`numpy.ndarray`
            The face image to be processed.

        annotations : dict or ``None``
            The annotations that fit to the given image. ``None`` is only accepted,
            when ``fixed_positions`` were specified in the constructor.

        Returns
        -------
        face : 2D :py:class:`numpy.ndarray` (float)
            The cropped face.

        Raises
        ------
        ValueError
            If the annotations is None.
        """
        if self.fixed_positions is not None:
            annotations = self.fixed_positions
        if annotations is None:
            raise ValueError(
                "Cannot perform image cropping since annotations are not given, and "
                "no fixed annotations are specified."
            )

        assert isinstance(annotations, dict)
        if not all(k in annotations for k in self.cropped_keys):
            raise ValueError(
                "At least one of the expected annotations '%s' are not given "
                "in '%s'." % (self.cropped_keys, annotations.keys())
            )

        reye = self.cropped_keys[0]
        leye = self.cropped_keys[1]
        reye_desired_width = self.cropped_positions[reye][1]
        leye_desired_width = self.cropped_positions[leye][1]
        right_eye = annotations[reye]
        left_eye = annotations[leye]
        if not self.allow_upside_down_normalized_faces:
            if (
                reye_desired_width > leye_desired_width and right_eye[1] < left_eye[1]
            ) or (
                reye_desired_width < leye_desired_width and right_eye[1] > left_eye[1]
            ):
                raise ValueError(
                    "Looks like {leye} and {reye} in annotations: {annot} are swapped. "
                    "This will make the normalized face upside down (compared to the original "
                    "image). Most probably your annotations are wrong. Otherwise, you can set "
                    "the ``allow_upside_down_normalized_faces`` parameter to "
                    "True.".format(leye=leye, reye=reye, annot=annotations)
                )

        # create output
        mask = numpy.ones(image.shape[-2:], dtype=bool)
        shape = (
            self.cropped_image_size
            if image.ndim == 2
            else [image.shape[0]] + list(self.cropped_image_size)
        )
        cropped_image = numpy.zeros(shape)
        self.cropped_mask[:] = False

        # perform the cropping
        self.cropper(
            image,  # input image
            mask,  # full input mask
            cropped_image,  # cropped image
            self.cropped_mask,  # cropped mask
            # position of first annotation, usually right eye
            right_eye=right_eye,
            # position of second annotation, usually left eye
            left_eye=left_eye,
        )

        if self.mask_sigma is not None:
            # extrapolate the mask so that pixels outside of the image original image
            # region are filled with border pixels
            if cropped_image.ndim == 2:
                bob.ip.base.extrapolate_mask(
                    self.cropped_mask,
                    cropped_image,
                    self.mask_sigma,
                    self.mask_neighbors,
                    self.mask_rng,
                )
            else:
                [
                    bob.ip.base.extrapolate_mask(
                        self.cropped_mask,
                        cropped_image_channel,
                        self.mask_sigma,
                        self.mask_neighbors,
                        self.mask_rng,
                    )
                    for cropped_image_channel in cropped_image
                ]

        return cropped_image

    def is_annotations_valid(self, annotations):
        if not annotations:
            return False
        # check if the required keys are available
        return all(key in annotations for key in self.cropped_keys)

    def transform(self, X, annotations=None):
        """Aligns the given image according to the given annotations.

        First, the desired color channel is extracted from the given image.
        Afterward, the face is cropped, according to the given ``annotations`` (or
        to ``fixed_positions``, see :py:meth:`crop_face`). Finally, the resulting
        face is converted to the desired data type.

        Parameters
        ----------
        image : 2D or 3D :py:class:`numpy.ndarray`
            The face image to be processed.
        annotations : dict or ``None``
            The annotations that fit to the given image.

        Returns
        -------
        face : 2D :py:class:`numpy.ndarray`
            The cropped face.
        """

        def _crop(image, annot):
            # if annotations are missing and cannot do anything else return None.
            if (
                not self.is_annotations_valid(annot)
                and not self.fixed_positions
                and self.annotator is None
            ):
                logger.warn(
                    "Cannot crop face without valid annotations or "
                    "fixed_positions or an annotator. Returning None. "
                    "The annotations were: {}".format(annot)
                )
                return None

            # convert to the desired color channel
            image = self.change_color_channel(image)

            # annotate the image if annotations are missing
            if (
                not self.is_annotations_valid(annot)
                and not self.fixed_positions
                and self.annotator is not None
            ):
                annot = self.annotator([image], annotations=[annot])[0]
                if not self.is_annotations_valid(annot):
                    logger.warn(
                        "The annotator failed and the annot are missing too"
                        ". Returning None."
                    )
                    return None

            # crop face
            return self.data_type(self.crop_face(image, annot))

        if annotations is None:
            return [_crop(data, None) for data in X]
        else:
            return [_crop(data, annot) for data, annot in zip(X, annotations)]

    def __getstate__(self):
        d = self.__dict__.copy()
        d.pop("mask_rng")
        d.pop("cropper")
        return d

    def __setstate__(self, d):
        self.__dict__ = d
        self._init_non_pickables()

class MultiFaceCrop(Base):
    """Wraps around FaceCrop to enable a dynamical cropper that can handle several annotation types.
    Initialization and usage is similar to the FaceCrop, but the main difference here is that one specifies
    a *list* of cropped_positions, and optionally a *list* of associated fixed positions.

    For each set of cropped_positions in the list, a new FaceCrop will be instanciated that handles this
    exact set of annotations.
    When calling the *transform* method, the MultiFaceCrop matches each sample to its associated cropper
    based on the received annotation, then performs the cropping of each subset, and finally gathers the results.

    In case of ambiguity (when no cropper is a match for the received annotations, or when several croppers
    match the received annotations), raises a ValueError.

    """

    def __init__(
        self,
        cropped_image_size,
        cropped_positions_list,
        fixed_positions_list=None,
        mask_sigma=None,
        mask_neighbors=5,
        mask_seed=None,
        annotator=None,
        allow_upside_down_normalized_faces=False,
        **kwargs,
    ):
        # Check parameters
        assert isinstance(cropped_positions_list, list)
        if fixed_positions_list is None:
            fixed_positions_list = [None] * len(cropped_positions_list)
        assert isinstance(fixed_positions_list, list)

        # copy parameters (sklearn convention : each explicit __init__ argument *has* to become an attribute of the estimator)
        self.cropped_image_size = cropped_image_size
        self.cropped_positions_list = cropped_positions_list
        self.fixed_positions_list = fixed_positions_list
        self.mask_sigma = mask_sigma
        self.mask_neighbors = mask_neighbors
        self.mask_seed = mask_seed
        if isinstance(annotator, str):
            annotator = load_resource(annotator, "annotator")
        self.annotator = annotator
        self.allow_upside_down_normalized_faces = allow_upside_down_normalized_faces

        # Instantiate individual croppers
        self.croppers = {}
        for cropped_positions, fixed_positions in zip(
            self.cropped_positions_list, self.fixed_positions_list
        ):
            assert len(cropped_positions) == 2
            self.croppers[tuple(cropped_positions)] = FaceCrop(
                self.cropped_image_size,
                cropped_positions,
                fixed_positions,
                self.mask_sigma,
                self.mask_neighbors,
                self.mask_seed,
                self.annotator,
                self.allow_upside_down_normalized_faces,
                **kwargs,
            )

    def transform(self, X, annotations=None):
        subsets = {k: {"X": [], "annotations": []} for k in self.croppers.keys()}

        def assign(X_elem, annotations_elem):
            # Assign a single sample to its matching cropper

            # Compare the received annotations keys to the cropped_positions keys of each cropper
            valid_keys = [
                k
                for k in self.croppers.keys()
                if set(k).issubset(set(annotations_elem.keys()))
            ]

            # Ensure exactly one cropper is a match
            if len(valid_keys) != 1:
                raise ValueError(
                    "Cropper selection from the annotations is ambiguous ({} valid croppers)".format(
                        len(valid_keys)
                    )
                )
            else:
                # Assign the sample to this particuler cropper
                cropper_key = valid_keys[0]
                subsets[cropper_key]["X"].append(X_elem)
                subsets[cropper_key]["annotations"].append(annotations_elem)

        # Assign each sample to its matching cropper
        for X_elem, annotations_elem in zip(X, annotations):
            assign(X_elem, annotations_elem)

        # Call each FaceCrop on its sample subset
        transformed_subsets = {
            k: self.croppers[k].transform(**subsets[k]) for k in subsets.keys()
        }

        # Gather the results
        return [item for sublist in transformed_subsets.values() for item in sublist]

    def fit(self, X, y=None):
        return self

class BoundingBoxAnnotatorCrop(Base):
    """
    This face cropper uses a 2 stage strategy to crop and align faces in case `annotation_type` has a bounding-box.
    In the first stage, it crops the face using the {`topleft`, `bottomright`} parameters and expands them using a `margin` factor.
    In the second stage, it uses the `annotator` to estimate {`leye` and `reye`} to make the crop using :py:class:`bob.ip.base.FaceEyesNorm`.
    In case the annotator doesn't work, it returnds the cropped face using the `bounding-box` coordinates.

    .. warning::
            `cropped_positions` must be set with `leye`, `reye`, `topleft` and `bottomright` positions

    Parameters
    ----------

    cropped_image_size : (int, int)
      The resolution of the cropped image, in order (HEIGHT,WIDTH); if not given,
      no face cropping will be performed

    cropped_positions : dict
      The coordinates in the cropped image, where the annotated points should be
      put to. This parameter is a dictionary with usually two elements, e.g.,
      ``{'reye':(RIGHT_EYE_Y, RIGHT_EYE_X) , 'leye':(LEFT_EYE_Y, LEFT_EYE_X)}``.
      However, also other parameters, such as ``{'topleft' : ..., 'bottomright' :
      ...}`` are supported, as long as the ``annotations`` in the `__call__`
      function are present.

    fixed_positions : dict or None
      If specified, ignore the annotations from the database and use these fixed
      positions throughout.

    mask_sigma : float or None
      Fill the area outside of image boundaries with random pixels from the
      border, by adding noise to the pixel values. To disable extrapolation, set
      this value to ``None``. To disable adding random noise, set it to a
      negative value or 0.

    mask_neighbors : int
      The number of neighbors used during mask extrapolation. See
      :py:func:`bob.ip.base.extrapolate_mask` for details.

    mask_seed : int or None
      The random seed to apply for mask extrapolation.

      .. warning::

         When run in parallel, the same random seed will be applied to all
         parallel processes. Hence, results of parallel execution will differ
         from the results in serial execution.

    allow_upside_down_normalized_faces: bool, optional
      If ``False`` (default), a ValueError is raised when normalized faces are going to be
      upside down compared to input image. This allows you to catch wrong annotations in
      your database easily. If you are sure about your input, you can set this flag to
      ``True``.

    annotator : :any:`bob.bio.base.annotator.Annotator`
      If provided, the annotator will be used if the required annotations are
      missing.

    margin: float
       The cropped face will be scaled to this factor (proportionally to the bouding-box width and height). Default to `0.5`.

    """

    def __init__(
        self,
        cropped_image_size,
        cropped_positions,
        annotator,
        mask_sigma=None,
        mask_neighbors=5,
        mask_seed=None,
        allow_upside_down_normalized_faces=False,
        color_channel="rgb",
        margin=0.5,
        **kwargs,
    ):

        if annotator is None:
            raise ValueError(f"A valid annotator needs to be set.")

        if isinstance(annotator, str):
            annotator = load_resource(annotator, "annotator")
        self.annotator = annotator

        # We need to have the four coordinates
        assert "leye" in cropped_positions
        assert "reye" in cropped_positions
        assert "topleft" in cropped_positions
        assert "bottomright" in cropped_positions

        # copy parameters (sklearn convention : each explicit __init__ argument *has* to become an attribute of the estimator)
        self.cropped_image_size = cropped_image_size
        self.cropped_positions = cropped_positions
        self.mask_sigma = mask_sigma
        self.mask_neighbors = mask_neighbors
        self.mask_seed = mask_seed
        self.allow_upside_down_normalized_faces = allow_upside_down_normalized_faces
        self.color_channel = color_channel

        ## Eyes cropper
        eyes_position = dict()
        eyes_position["leye"] = cropped_positions["leye"]
        eyes_position["reye"] = cropped_positions["reye"]
        self.eyes_cropper = FaceCrop(
            cropped_image_size,
            eyes_position,
            fixed_positions=None,
            mask_sigma=mask_sigma,
            mask_neighbors=mask_neighbors,
            mask_seed=mask_seed,
            allow_upside_down_normalized_faces=allow_upside_down_normalized_faces,
            color_channel=color_channel,
        )
        self.margin = margin

    def transform(self, X, annotations=None):
        faces = []

        for x, annot in zip(X, annotations):

            # If it's grayscaled, expand dims
            if x.ndim == 2:
                logger.warning(
                    "Gray-scaled image. Expanding the channels before detection"
                )
                x = numpy.repeat(numpy.expand_dims(x, 0), 3, axis=0)

            top = annot["topleft"][0]
            left = annot["topleft"][1]

            bottom = annot["bottomright"][0]
            right = annot["bottomright"][1]

            width = right - left
            height = bottom - top

            # Expanding the borders
            top_expanded = int(numpy.maximum(top - self.margin * height, 0))
            left_expanded = int(numpy.maximum(left - self.margin * width, 0))

            bottom_expanded = int(
                numpy.minimum(bottom + self.margin * height, x.shape[1])
            )
            right_expanded = int(numpy.minimum(right + self.margin * width, x.shape[2]))

            face_crop = x[
                :, top_expanded:bottom_expanded, left_expanded:right_expanded,
            ]

            # get the coordinates with the annotator
            annotator_annotations = self.annotator([face_crop])[0]

            # If nothing was detected OR if the annotations are swaped, return the cropped face
            if (
                annotator_annotations is None
                or annotator_annotations["reye"][1] > annotator_annotations["leye"][1]
            ):
                logger.warning(
                    f"Unable to detect face in bounding box. Got : {annotator_annotations}. Cropping will be only based on bounding-box."
                )

                face_crop = scale(face_crop, self.cropped_image_size)
                faces.append(face_crop)
            else:

                faces.append(
                    self.eyes_cropper.transform([face_crop], [annotator_annotations])[0]
                )

        return faces

    def fit(self, X, y=None):
        return self


text:
"""
Plot full psychometric functions as a function of choice history,
and separately for 20/80 and 80/20 blocks
"""

import pandas as pd
import numpy as np
import sys, os, time
import matplotlib.pyplot as plt
import seaborn as sns
import datajoint as dj
from IPython import embed as shell # for debugging
from scipy.special import erf # for psychometric functions
import datetime

## INITIALIZE A FEW THINGS
sns.set(style="darkgrid", context="paper", font='Arial')
sns.set(style="darkgrid", context="paper")
sns.set(style="darkgrid", context="paper", font_scale=1.3)

# import wrappers etc
from ibl_pipeline import reference, subject, action, acquisition, data, behavior
from ibl_pipeline.utils import psychofit as psy
from ibl_pipeline.analyses import behavior as behavioral_analyses
from dj_tools import *
import training_criteria_schemas as criteria_urai

figpath  = os.path.join(os.path.expanduser('~'), 'Data/Figures_IBL')

# ================================= #
# GRAB ALL DATA FROM DATAJOINT
# ================================= #

use_subjects = subject.Subject() & 'subject_birth_date between "2018-09-01" and "2019-02-01"' & 'subject_line IS NULL OR subject_line="C57BL/6J"'
criterion = criteria_urai.SessionTrainingStatus_v0()
criterion = behavioral_analyses.SessionTrainingStatus()
sess = ((acquisition.Session) * \
 (criterion & 'training_status="trained"')) \
 * subject.SubjectLab * use_subjects

s = pd.DataFrame.from_dict(sess.fetch(as_dict=True))
labs = list(s['lab_name'].unique())
print(labs)
# labs = list(filter(None, labs)) # remove None lab

# hack to get around SQL limit
for lidx, lab in enumerate(labs):

	print(lab)
	subjects = s[s['lab_name'].str.contains(lab)].reset_index()

	for midx, mousename in enumerate(subjects['subject_nickname'].unique()):

        # ============================================= #
        # check whether the subject is trained based the the lastest session
        # ============================================= #

		subj = subject.Subject & 'subject_nickname="{}"'.format(mousename)
		last_session = subj.aggr(
		behavior.TrialSet, session_start_time='max(session_start_time)')
		training_status = \
		(criterion & last_session).fetch1(
		'training_status')

		if training_status in ['trained', 'ready for ephys']:
			first_trained_session = subj.aggr(
			criterion &
			'training_status="trained"',
			first_trained='min(session_start_time)')
			first_trained_session_time = first_trained_session.fetch1(
			'first_trained')
			# convert to timestamp
			trained_date = pd.DatetimeIndex([first_trained_session_time])[0]
		else:
			print('WARNING: THIS MOUSE WAS NOT TRAINED!')
			continue

		# now get the sessions that went into this
		# https://github.com/shenshan/IBL-pipeline/blob/master/ibl_pipeline/analyses/behavior.py#L390
		sessions = (behavior.TrialSet & subj &
			(acquisition.Session) &
			'session_start_time <= "{}"'.format(
			trained_date.strftime(
			'%Y-%m-%d %H:%M:%S')
			)).fetch('KEY')

		# if not more than 3 biased sessions, keep status trained
		sessions_rel = sessions[-3:]

		b = (behavior.TrialSet.Trial & sessions_rel) \
			* (subject.SubjectLab & 'lab_name="%s"'%lab) \
			* (subject.Subject & 'subject_nickname="%s"'%mousename)

		bdat = pd.DataFrame(b.fetch(order_by='subject_nickname, session_start_time, trial_id'))
		print(bdat['subject_nickname'].unique())
		print(trained_date)
		print(bdat['session_start_time'].unique())

		# APPEND
		if not 'behav' in locals():
			behav = bdat.copy()
		else:
			behav = behav.append(bdat.copy(), sort=False, ignore_index=True)

# ================================================================== #
# for now, manually add the cortexlab matlab animals
# ================================================================== #

# ucl_mice = ['KS001', 'MW003', 'MW001', 'MW002', 'LEW008', 'LEW009', 'LEW010']
# ucl_trained_dates = ['2019-02-25', '2018-12-10', '2019-02-11', '2019-01-14', '2018-10-04', '2018-10-04', 'LEW010']

# for midx, mouse in enumerate(ucl_mice):

# 	print(mouse)
# 	sess = (acquisition.Session.proj('session_start_time', 'task_protocol', 'session_uuid', session_date='DATE(session_start_time)') & \
# 		'session_date > "%s"'%ucl_trained_dates[midx]) * behavioral_analyses.SessionTrainingStatus()
# 	b = ((behavior.TrialSet.Trial & (subject.SubjectLab() & 'lab_name="cortexlab"')) \
# 		* subject.SubjectLab.proj('lab_name') \
# 		* subject.Subject() & 'subject_nickname="%s"'%mouse) \
# 		* sess

# 	bdat  = pd.DataFrame(b.fetch(order_by='subject_nickname, session_start_time, trial_id'))
# 	behav = behav.append(bdat.copy(), sort=False, ignore_index=True)

# ================================= #
# convert
# ================================= #

behav = dj2pandas(behav)
behav['lab_name'] = behav['lab_name'].str.replace('zadorlab', 'churchlandlab')
print(behav.describe())

# ================================= #
# ONE PANEL PER MOUSE
# ================================= #

fig = sns.FacetGrid(behav,
	col="subject_nickname", col_wrap=6,
	palette="gist_gray", sharex=True, sharey=True)
fig.map(plot_psychometric, "signed_contrast", "choice_right", "subject_nickname", color='k').add_legend()
fig.set_axis_labels('Signed contrast (%)', 'Rightward choice (%)')
fig.set_titles("{col_name}")
fig.despine(trim=True)
fig.savefig(os.path.join(figpath, "psychfuncs_permouse_dj.pdf"))
fig.savefig(os.path.join(figpath, "psychfuncs_permouse_dj.png"), dpi=100)
plt.close('all')

# ALSO CHRONOMETRIC FUNCTIONS
sns.set_style("darkgrid", {'xtick.bottom': True,'ytick.left': True, 'lines.markeredgewidth':0})
fig = sns.FacetGrid(behav,
	col="subject_nickname", col_wrap=6,
	palette="gist_gray", sharex=True, sharey=True)
fig.map(plot_chronometric, "signed_contrast", "rt", "subject_nickname").add_legend()
fig.set_axis_labels('Signed contrast (%)', 'RT (s)')
fig.set_titles("{col_name}")
fig.despine(trim=True)
fig.savefig(os.path.join(figpath, "chrono_permouse_dj.pdf"))
fig.savefig(os.path.join(figpath, "chrono_permouse_dj.png"), dpi=100)

shell()

# ================================= #
# RT ACROSS ALL CONTRASTS, PER LAB
# ================================= #

median_rt = behav.groupby(['subject_nickname', 'lab_name'])['rt'].median().reset_index()
fig = sns.swarmplot(x="lab_name", y="rt", data=median_rt)
fig.set_title('RT median')
plt.savefig(os.path.join(figpath, "rt_median.pdf"))
plt.close('all')

mean_rt = behav.groupby(['subject_nickname', 'lab_name'])['rt'].mean().reset_index()
fig = sns.swarmplot(x="lab_name", y="rt", data=mean_rt)
fig.set_title('RT mean')
plt.savefig(os.path.join(figpath, "rt_mean.pdf"))
plt.close('all')

# RT DISTRIBUTIONS - ONE PANEL PER MOUSE
fig = sns.FacetGrid(behav[behav.init_unbiased == True],
	col="subject_nickname", col_wrap=7,
	palette="gist_gray", sharex=False, sharey=False, xlim=[-1, 5])
fig.map(sns.distplot, "rt", bins=500).add_legend()
# fig.set_axis_labels('Signed contrast (%)', 'Rightward choice (%)')
fig.set_titles("{col_name}")
#fig.set_xlim([-0.1, 3])
fig.despine(trim=True)
fig.savefig(os.path.join(figpath, "rt_dist_permouse.pdf"))
plt.close('all')

# ================================= #
# ONLY BLACK/neutral CURVES!
# ================================= #

sns.set_palette("gist_gray")  # palette for water types
fig = sns.FacetGrid(behav,
	col="lab_name", col_order=['cortexlab', 'churchlandlab', 'mainenlab', 'angelakilab', 'danlab', 'wittenlab'], col_wrap=3,
	palette="gist_gray", sharex=True, sharey=True, aspect=1)
fig.map(plot_psychometric, "signed_contrast", "choice_right", "subject_nickname")

#fig.map(plot_chronometric, "signed_contrast", "rt")
fig.set_axis_labels('Signed contrast (%)', 'Rightward choice (%)')

# FOR PARIS MEETING, ADD N IN TITLE
# titles = behav.groupby('lab_name').agg({'subject_nickname':'nunique', 'choice':
	# 'count'}).reset_index()
# titles2 = ['%s | %d mice, %d trials'%(df['lab_name'], df['subject_nickname'], df['choice']) for i, df in titles.iterrows()]
# titles2 = ['%s lab'%(str.title(df['lab_name'][:-3])) for i, df in titles.iterrows()]
# # titles2 = [t.replace('Churchland lab', 'Churchland & Zador labs') for t in titles2]
# titles2 = [t.replace('Cortex lab', 'Carandini-Harris lab') for t in titles2]
titles2 = ['Carandini-Harris lab', 'Churchland & Zador labs', 'Mainen lab', 'Angelaki lab', 'Dan lab', 'Witten lab']

for ax, title in zip(fig.axes.flat, titles2):
    ax.set_title(title)
fig.despine(trim=True)
fig.savefig(os.path.join(figpath, "psychfuncs_summary.pdf"))
fig.savefig(os.path.join(figpath, "psychfuncs_summary.png"), dpi=600)
plt.close('all')

# ================================= #
## ALSO ADD WITH individual subjects
# ================================= #

fig = sns.FacetGrid(behav, hue='subject_nickname',
	col="lab_name", col_order=sorted(behav.lab_name.unique()), col_wrap=3,
	palette="colorblind", sharex=True, sharey=True, aspect=.8)
fig.map(plot_psychometric, "signed_contrast", "choice_right", "subject_nickname")
fig.set_axis_labels('Signed contrast (%)', 'Rightward choice (%)')
fig.add_legend()
fig.despine(trim=True)
fig.savefig(os.path.join(figpath, "psychfuncs_summary_permouse.pdf"))
fig.savefig(os.path.join(figpath, "psychfuncs_summary_permouse.png"), dpi=600)

# ================================= #
# ALSO CHRONOMETRIC FUNCTIONS
# ================================= #

behav['abs_contrast'] = np.abs(behav.signed_contrast)
fig = sns.FacetGrid(behav[behav.init_unbiased == True],
	col="subject_nickname", col_wrap=7,
	palette="gist_gray", sharex=True, sharey=True)
fig.map(plot_chronometric, "abs_contrast", "rt", "subject_nickname").add_legend()
fig.set_axis_labels('Contrast (%)', 'RT (s)')
fig.set_titles("{col_name}")
fig.despine(trim=True)
fig.savefig(os.path.join(figpath, "chrono_abs_permouse_black.pdf"))

fig = sns.FacetGrid(behav, hue='subject_nickname',
	col="lab_name", col_order=sorted(behav.lab_name.unique()), col_wrap=2,
	palette="colorblind", sharex=True, sharey=True, aspect=.8)
fig.map(plot_chronometric, "signed_contrast", "rt", "subject_nickname")
fig.set_axis_labels('Signed contrast (%)', 'RT (s)')
fig.add_legend()
fig.despine(trim=True)
fig.savefig(os.path.join(figpath, "chrono_summary_permouse.pdf"))

# ABSOLUTE CHRONO, PER LAB
fig = sns.FacetGrid(behav,
	col="lab_name", col_order=['cortexlab', 'churchlandlab', 'mainenlab', 'angelakilab', 'danlab', 'wittenlab'], col_wrap=3,
	palette="gist_gray", sharex=True, sharey=True, aspect=1)
fig.map(plot_chronometric, "abs_contrast", "rt", "subject_nickname").add_legend()
fig.set_axis_labels('Contrast (%)', 'RT (s)')
for ax, title in zip(fig.axes.flat, titles2):
    ax.set_title(title)
fig.despine(trim=True)
fig.savefig(os.path.join(figpath, "chrono_abs.pdf"))

max_stars_repo_name: ineslaranjeira/analysis

max_issues_repo_name: ineslaranjeira/analysis

max_forks_repo_name: ineslaranjeira/analysis


text:
import pandas
from bs4 import BeautifulSoup
import re
import nltk
import numpy
#only do next line once
#nltk.download() #download everything except panlex_lite
from nltk.corpus import stopwords
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn import model_selection, preprocessing
from sklearn.metrics import accuracy_score
from sklearn.svm import SVC
from gensim.models import word2vec
from sklearn.metrics import accuracy_score
from sklearn.naive_bayes import GaussianNB, BernoulliNB
import sklearn.model_selection as cv
from sklearn.decomposition import PCA
from sklearn.model_selection import train_test_split, GridSearchCV
from datetime import datetime

"""
author: Mahesh Gaya
project 5
description: Twitter Sentiment Analysis
"""
"""
Steps:
1. data clean up
	- some of the texts were combined when it should be. I arranged them
		so that they are on different rows (excel)
	- Added "Result" and "Message" to know what columns we are using
"""

def clean_sentence( raw ):
	bs = BeautifulSoup(raw, "html.parser")
	letters_only = re.sub("[^a-zA-Z]"," ",bs.get_text())
	lower_case = letters_only.lower()
	words = lower_case.split()
	return words

def review_to_sentences( message_data, tokenizer):
	#didn’t seem to work without it, thanks StackOverflow
	#review = review.decode("utf-8")
	#strip out whitespace at beginning and end
	sentences_to_return = []
	for row in message_data:
		message = row.strip()
		raw_sentences = tokenizer.tokenize(message)
		sentences_list = []

		for sentence in raw_sentences:
			if len(sentence) > 0: #skip it if the sentence is empty
				cl_sent = clean_sentence(sentence)
				sentences_list += cl_sent
		sentences_to_return.append(sentences_list)
	return sentences_to_return

def make_attribute_vec(words, model, num_attributes):
	# Pre-initialize an empty numpy array (for speed)
	attribute_vec = numpy.zeros((num_attributes,),dtype="float32")
	nwords = 0.0
	# Loop over each word in the message and, if it is in the model’s
	# vocabulary, add its attribute vector to the total
	for word in words:
		if word in model.vocab:
			nwords = nwords + 1.0
			attribute_vec = numpy.add(attribute_vec,model[word])

	# Divide the result by the number of words to get the average
	attribute_vec = numpy.divide(attribute_vec,nwords)
	return attribute_vec

def calc_avg_accuarcy(list):
	list_sum = sum(list)
	return list_sum/len(list)

def main():
	tokenizer = nltk.data.load("tokenizers/punkt/english.pickle")
	test_data = pandas.read_csv("testdata.csv", delimiter=",", encoding = "ISO-8859-1")
	train_data = pandas.read_csv("training.csv", delimiter=",", encoding = "ISO-8859-1")
	test_sentences_for_all_messages = review_to_sentences(test_data["Message"],tokenizer)
	#print(test_sentences_for_all_messages)
	train_sentences_for_all_messages = review_to_sentences(train_data["Message"],tokenizer)
	#print(train_sentences_for_all_messages)
	num_attributes = 300 # Word vector dimensionality
	min_word_count = 40 # Minimum word frequency
	num_workers = 4 # Number of threads to run in parallel
	context = 10 # Context window size
	downsampling = 1e-3 # Downsample setting for frequent words
	# Initialize and train the model (this will take some time)

	test_model = word2vec.Word2Vec(test_sentences_for_all_messages, \
			workers=num_workers, size=num_attributes, \
			min_count = min_word_count, \
			window = context, sample = downsampling)
	#saves memory if you’re done training it
	test_model.init_sims(replace=True)
	#print(test_model.vocab)
	#print("christmas" in test_model.vocab)

	train_model = word2vec.Word2Vec(train_sentences_for_all_messages, \
			workers=num_workers, size=num_attributes, \
			min_count = min_word_count, \
			window = context, sample = downsampling)
	#saves memory if you’re done training it
	train_model.init_sims(replace=True)
	#print(train_model.vocab)
	#print("christmas" in train_model.vocab)

	#make the two lists into vector
	test_message_vector = []
	for i in range(len(test_sentences_for_all_messages)):
		vector = make_attribute_vec(test_sentences_for_all_messages[i], test_model, \
		num_attributes)
		test_message_vector.append(vector.tolist())

	#print("vector:", str(test_message_vector))

	train_message_vector = []
	for i in range(len(train_sentences_for_all_messages)):
		vector = make_attribute_vec(train_sentences_for_all_messages[i], test_model, \
		num_attributes)
		train_message_vector.append(vector.tolist())
	#print("vector:", str(train_message_vector))
	#print(len(train_message_vector))
	#print(len(train_data))

	train_message_vector = numpy.array(train_message_vector)

	#check_nan = numpy.isnan(train_message_vector)
	#if (True in check_nan): print("Yes")

	#convert any nan to zero
	train_message_vector = numpy.nan_to_num(train_message_vector)

	# Split the training data into testing and training data to check for accuracy
	def gaussianNB():
		print("\nGaussianNB")
		accuracy_list = []
		for i in range(0,20):
			(training_data, testing_data, training_target, testing_target) = \
				cv.train_test_split(train_message_vector, train_data["Result"],\
						test_size = 0.2)
			gaussian_nb = GaussianNB()
			gaussian_nb.fit(training_data, training_target)
			prediction = gaussian_nb.predict(testing_data)
			accuracy = accuracy_score(prediction, testing_target)
			accuracy_list.append(accuracy)
			if (i < 10):
				print("0" + str(i) + " :--: " + str(accuracy))
			else:
				print(str(i) + " :--: " + str(accuracy))
		print(str("avg accuracy = ") + str(calc_avg_accuarcy(accuracy_list)))

	def bernoulliNB():
		print("\nBernoulliNB")
		accuracy_list = []
		for i in range(0,20):
			(training_data, testing_data, training_target, testing_target) = \
				cv.train_test_split(train_message_vector, train_data["Result"],\
						test_size = 0.2)
			bernoulli_nb = BernoulliNB()
			bernoulli_nb.fit(training_data, training_target)
			prediction = bernoulli_nb.predict(testing_data)
			accuracy = accuracy_score(prediction, testing_target)
			accuracy_list.append(accuracy)
			if (i < 10):
				print("0" + str(i) + " :--: " + str(accuracy))
			else:
				print(str(i) + " :--: " + str(accuracy))
		print(str("avg accuracy = ") + str(calc_avg_accuarcy(accuracy_list)))

	def svm(kernel_type="rbf", n_degree=3):
		print("\nSVC: " + kernel_type + ", degree=" + str(n_degree))
		accuracy_list = []
		for i in range(0,20):
			(training_data, testing_data, training_target, testing_target) = \
				cv.train_test_split(train_message_vector, train_data["Result"],\
						test_size = 0.2)
			svc = SVC(kernel=kernel_type, degree=n_degree)
			svc.fit(training_data, training_target)
			prediction = svc.predict(testing_data)
			accuracy = accuracy_score(prediction, testing_target)
			accuracy_list.append(accuracy)
			if (i < 10):
				print("0" + str(i) + " :--: " + str(accuracy))
			else:
				print(str(i) + " :--: " + str(accuracy))
		print(str("avg accuracy = ") + str(calc_avg_accuarcy(accuracy_list)))

	print(datetime.now().time())
	gaussianNB()
	print(datetime.now().time())
	bernoulliNB()
	print(datetime.now().time())
	svm("linear")
	print(datetime.now().time())
	svm(kernel_type="poly", n_degree=1)
	print(datetime.now().time())
	svm(kernel_type="poly")
	print(datetime.now().time())
	svm()
	print(datetime.now().time())
	svm(n_degree=1)
	print(datetime.now().time())

main()

max_stars_repo_name: maheshgaya/ml-text-classification

max_issues_repo_name: maheshgaya/ml-text-classification

max_forks_repo_name: maheshgaya/ml-text-classification


text:
import dash
import dash_table
import dash_core_components as dcc
import dash_html_components as html
import dash_bootstrap_components as dbc
import plotly.graph_objs as go
import pandas as pd
import numpy as np
from decimal import *
from operator import add
from functools import reduce
import os.path

BS = 'https://stackpath.bootstrapcdn.com/bootstrap/4.4.1/css/bootstrap.min.css'
app = dash.Dash(external_stylesheets=[BS])

def reformat_dollar_amount(dollars):
    data_chars = [l for l in str(dollars).split('.')[0]]
    cents = str(dollars).split('.')[1]
    for i in range(len(data_chars) - 4, -1, -3):
        data_chars[i] = str(data_chars[i]) + ','
    return ''.join(data_chars) + "." + cents

def pre_process():
    # Data frames
    global inventory_df, transactions_df

    here = os.path.dirname(os.path.abspath(__file__))

    inventory_filename = os.path.join(here, 'Inventory.xlsx')
    transactions_filename = os.path.join(here, 'Transactions.xlsx')

    if os.path.isfile(inventory_filename):
        inventory_df = pd.read_excel(inventory_filename, ignore_index=True)
    else:
        print("No Inventory File Was Found!")
        inventory_df = pd.DataFrame([])

    if os.path.isfile(transactions_filename):
        transactions_df = pd.read_excel(transactions_filename, ignore_index=True)
    else:
        print("No Transactions File Was Found!")
        transactions_df = pd.DataFrame([])

def do_calculations():
    global total_inventory_amount, total_revenue_amount, total_tax_amount
    # Calculations
    inventory_df['Quantity'] = inventory_df['Quantity'].astype(dtype=float)
    inventory_df['S.Price'] = np.array(inventory_df['S.Price'], dtype=float)

    sp_times_quant = [s * q for s, q in zip(list(inventory_df['S.Price']), list(inventory_df['Quantity']))]
    total_inventory_amount = Decimal(reduce(add, sp_times_quant)).quantize(Decimal('.01'))
    total_inventory_amount = reformat_dollar_amount(total_inventory_amount)

    transactions_df['Total'] = transactions_df['Total'].fillna(0.0)
    total_revenue_amount = Decimal(reduce(add, list(transactions_df['Total']))).quantize(Decimal('.01'))
    total_revenue_amount = reformat_dollar_amount(total_revenue_amount)

    transactions_df['Tax'] = transactions_df['Tax'].fillna(0.0)
    total_tax_amount = Decimal(reduce(add, list(transactions_df['Tax']))).quantize(Decimal('.01'))
    total_tax_amount = reformat_dollar_amount(total_tax_amount)

def daily_graph():
    global date_fig, today_total

    pre_process()
    do_calculations()

    # Daily Graph
    dates_trans_df = transactions_df.groupby('Date').sum()
    date_graph_x = dates_trans_df.index
    date_graph_y = dates_trans_df['Total']
    # Today's Total
    today_total = Decimal(date_graph_y[-1]).quantize(Decimal('.01'))
    today_total = reformat_dollar_amount(today_total)

    date_fig = go.Figure(data=[go.Bar(x=date_graph_x, y=date_graph_y,
                                      hovertemplate="Sale: $%{y}<extra></extra>")])
    # Customize aspect
    date_fig.update_traces(marker_color='rgb(235, 52, 91)', marker_line_color='rgb(8,48,107)',
                           marker_line_width=1.5, opacity=0.6)
    date_fig.layout.plot_bgcolor = 'rgb(255, 255, 255)'
    date_fig.update_layout(
        xaxis= dict(
            tickvals=date_graph_x,
        ),
        hoverlabel=dict(
            bgcolor="rgb(255, 81, 69)",
            font_size=20,
            font_family="Rockwell"
        )
    )

    return date_fig

def monthly_graph():
    global t2, months_fig

    pre_process()
    do_calculations()

    # Monthly Graph
    t2 = transactions_df.copy()
    t2['Date'] = t2['Date'].astype(str)
    t2['Date'] = [d[:7] for d in t2['Date']]
    months_trans_df = t2.groupby('Date').sum()

    months_graph_x = months_trans_df.index
    months_graph_y = months_trans_df['Total']

    months_fig = go.Figure(data=[go.Bar(x=months_graph_x, y=months_graph_y,
                                        hovertemplate="Sale: $%{y}<extra></extra>")])
    # Customize aspect
    months_fig.update_traces(marker_color='rgb(235, 52, 91)', marker_line_color='rgb(8,48,107)',
                             marker_line_width=1.5, opacity=0.6)
    months_fig.layout.plot_bgcolor = 'rgb(255, 255, 255)'
    months_dict = {
        '01': 'January',
        '02': 'February',
        '03': 'March',
        '04': 'April',
        '05': 'May',
        '06': 'June',
        '07': 'July',
        '08': 'August',
        '09': 'September',
        '10': 'October',
        '11': 'November',
        '12': 'December'
    }
    months_labels = [months_dict[m[5:7]] + ', ' + m[0:4] for m in months_graph_x]
    months_fig.update_layout(
        xaxis=dict(
            tickmode='array',
            tickvals=months_graph_x,
            ticktext=months_labels
        ),
        hoverlabel=dict(
            bgcolor="rgb(255, 81, 69)",
            font_size=20,
            font_family="Rockwell"
        )
    )

    return months_fig

def yearly_graph():
    global years_fig

    pre_process()
    do_calculations()

    # Yearly Graph
    t2['Date'] = transactions_df['Date']
    t2['Date'] = t2['Date'].astype(str)
    t2['Date'] = [d[:4] for d in t2['Date']]
    years_trans_df = t2.groupby('Date').sum()

    years_graph_x = years_trans_df.index
    years_graph_y = years_trans_df['Total']

    years_fig = go.Figure(data=[go.Bar(x=years_graph_x, y=years_graph_y,
                                       hovertemplate="Sale: $%{y}<extra></extra>")])
    # Customize aspect
    years_fig.update_traces(marker_color='rgb(235, 52, 91)', marker_line_color='rgb(8,48,107)',
                            marker_line_width=1.5, opacity=0.6)
    years_fig.layout.plot_bgcolor = 'rgb(255, 255, 255)'

    years_fig.update_layout(
        xaxis=dict(
            tickvals=years_graph_x,
        ),
        hoverlabel=dict(
            bgcolor="rgb(255, 81, 69)",
            font_size=20,
            font_family="Rockwell"
        )
    )

    return years_fig

def pie_chart():
    global pie_fig

    pre_process()
    do_calculations()

    # Pie Chart - Cash/Credit
    cash_count = transactions_df.where(transactions_df['P.Type'] == 'CASH').count()
    cash_val = cash_count['P.Type']
    credit_count = transactions_df.where(transactions_df['P.Type'] == 'CREDIT').count()
    credit_val = credit_count['P.Type']
    labels = ['CASH', 'CREDIT']
    values = [cash_val, credit_val]
    pie_fig = go.Figure(data=[go.Pie(labels=labels, values=values, hole=.4, text=values,
                                     hovertemplate="# of Transactions: %{text}")])
    pie_fig.layout.plot_bgcolor = 'rgb(255, 255, 255)'
    pie_fig.update_layout(
        title_text="CASH/CREDIT Transactions",
        hoverlabel=dict(
            bgcolor="rgb(25, 212, 128)",
            font_size=18,
            font_family="Rockwell"
        ))

    return pie_fig

def top_ten_table():
    global table_df
    # Top Ten Selling Items Table
    item_names_and_frequency = []
    for name in transactions_df['Name']:
        if name != 'Misc.':  # don't want to count these
            count = transactions_df['Name'].where(transactions_df['Name'] == name).count()
            if [name, count] not in item_names_and_frequency:
                item_names_and_frequency.append([name, count])

    # sorting the list by greatest to least sale frequency
    item_names_and_frequency.sort(key=lambda x: x[1], reverse=True)
    # only want top 10
    item_names_and_frequency = item_names_and_frequency[:10]
    # creating table data frame
    table_df = pd.DataFrame(
        {
            '🔥 Top 10 Selling Items': [item[0] for item in item_names_and_frequency],
            'Count': [item[1] for item in item_names_and_frequency]
        }
    )

pre_process()
do_calculations()
daily_graph()
monthly_graph()
yearly_graph()
top_ten_table()

def serve_layout():
    return html.Div(style={'margin-left': '0px', 'margin-right': '0px',
                           'margin-top': '0px', 'margin-bottom': '0px', 'backgroundColor': 'rgb(255, 255, 255)'},
                    children=[
                        dbc.Navbar(
                            [
                                dbc.Row(
                                    [
                                        dbc.Col(dbc.NavbarBrand("| BizTracker", className="ml-2",
                                                                style={'font-family': 'verdana',
                                                                       'font-size': '150%',
                                                                       'font-weight': 'bold'}),
                                                width=3),

                                        dbc.Col(
                                            html.Div(
                                                children=[
                                                    html.H5(children=["Today's Sales: $" + str(today_total)],
                                                            style={'color': 'orange',
                                                                   'font-family': 'impact',
                                                                   'font-size': '150%'}, id='today_sale'),
                                                ],
                                                style={'padding-left': '80px', 'padding-top': '10px'}
                                            ))
                                    ],
                                    no_gutters=True,
                                ),
                            ],
                            color="#191a1a",
                            dark=True,
                            fixed='top',
                            style={'opacity': '0.9'}
                        ),

                        html.Div(
                            children=[
                                dbc.Row(
                                    [
                                        dbc.Col(
                                            dbc.Card([
                                                dbc.CardHeader("TOTAL INVENTORY"),
                                                dbc.CardBody(
                                                    [
                                                        html.H4("$ " + str(total_inventory_amount),
                                                                className="card-title", id='total_inv_amount'),

                                                    ]
                                                ),
                                            ], color="primary",
                                                inverse=True)
                                        ),

                                        dbc.Col(
                                            dbc.Card([
                                                dbc.CardHeader("TOTAL SALE"),
                                                dbc.CardBody(
                                                    [
                                                        html.H4("$ " + str(total_revenue_amount),
                                                                className="card-title", id='total_rev_amount'),

                                                    ]
                                                ),
                                            ], color="success",
                                                inverse=True)
                                        ),

                                        dbc.Col(
                                            dbc.Card([
                                                dbc.CardHeader("TOTAL TAX"),
                                                dbc.CardBody(
                                                    [
                                                        html.H4("$ " + str(total_tax_amount), className="card-title",
                                                                id='total_tax_amount'),

                                                    ]
                                                ),
                                            ], color="danger",
                                                inverse=True)
                                        )
                                    ],
                                    style={'padding-right': '0px', 'padding-left': '0px', 'margin-right': '0px',
                                           'margin-left': '0px', 'padding-top': '22px', 'padding-bottom': '24px',
                                           'borderBottom': 'thick lightgrey solid'},
                                ),

                                dbc.Row(
                                    [
                                        dbc.Col(
                                            dcc.Dropdown(
                                                id='sale_dropdown',
                                                options=[
                                                    {'label': 'Daily Sales', 'value': 'Day'},
                                                    {'label': 'Monthly Sales', 'value': 'Mon'},
                                                    {'label': 'Yearly Sales', 'value': 'Yea'},
                                                ],
                                                style={'width': '300px', 'padding-left': '30px'},
                                                value='Day',
                                                clearable=False,
                                                searchable=False,
                                            ), style={'backgroundColor': 'rgb(255, 255, 255)',
                                                      'padding-top': '10px'},
                                        ),
                                    ]
                                ),

                                dbc.Row(
                                    [
                                        dbc.Col(
                                            dcc.Graph(id='sale_graph')
                                        )
                                    ]
                                ),

                                dbc.Row(
                                    [
                                        dbc.Col(
                                            dash_table.DataTable(
                                                id='table',
                                                columns=[{"name": i, "id": i} for i in table_df.columns],
                                                data=table_df.to_dict('records'),
                                                style_as_list_view=True,
                                                style_header={'backgroundColor': 'rgb(255, 255, 255)',
                                                              'color': 'rgb(50, 187, 191)',
                                                              'height': '50px',
                                                              'font_family': 'impact',
                                                              'textAlign': 'center',
                                                              'font_size': '24px'},
                                                style_cell={
                                                    'backgroundColor': 'rgb(255, 255, 255)',
                                                    'font_family': 'impact',
                                                    'font_size': '20px',
                                                    'color': 'rgb(201, 52, 172)',
                                                    'textAlign': 'center',
                                                    'whiteSpace': 'normal',
                                                    'height': 'auto',
                                                },
                                                style_table={'height': '450px', 'width': '785px', 'overflowY': 'auto'},
                                            ),
                                            style={'padding-right': '0px', 'padding-left': '0px', 'margin-left': '0px',
                                                   'margin-right': '0px'}
                                        ),

                                        dbc.Col(
                                            dcc.Graph(
                                                id='pie',
                                                figure=pie_chart()
                                            ),
                                            style={'padding-left': '0px', 'margin-left': '0px',
                                                   'borderTop': 'thin lightgrey solid'}
                                        )
                                    ],
                                    style={'margin-bottom': '0px !important', 'padding-top': '0px',
                                           'padding-bottom': '0px', 'padding-right': '0px', 'padding-left': '0px'}
                                )
                            ],
                            style={'margin-bottom': '0px !important', 'padding-top': '60px',
                                   'padding-bottom': '0px',
                                   'padding-right': '0px', 'padding-left': '0px'},
                        ),
                    ])

app.title = 'Live Dashboard'
app.layout = serve_layout

@app.callback(
    [dash.dependencies.Output('sale_graph', 'figure'),
     dash.dependencies.Output('total_inv_amount', 'children'),
     dash.dependencies.Output('total_rev_amount', 'children'),
     dash.dependencies.Output('total_tax_amount', 'children'),
     dash.dependencies.Output('today_sale', 'children'),
     dash.dependencies.Output('table', 'columns'),
     dash.dependencies.Output('table', 'data')
     ],
    [dash.dependencies.Input('sale_dropdown', 'value')]
)
def update_graph(type_value):
    top_ten_table()
    columns = [{"name": i, "id": i} for i in table_df.columns]
    data = table_df.to_dict('records')
    pie_chart()

    if str(type_value) == 'Day':
        return [daily_graph(), "$ " + str(total_inventory_amount), "$ " + str(total_revenue_amount), "$ " +
                str(total_tax_amount), "Today's Sales: $" + str(today_total), columns, data]
    elif str(type_value) == 'Mon':
        return [monthly_graph(), "$ " + str(total_inventory_amount), "$ " +
                str(total_revenue_amount), "$ " + str(total_tax_amount), "Today's Sales: $" + str(today_total),
                columns, data]
    else:
        return [yearly_graph(), "$ " + str(total_inventory_amount), "$ " + str(total_revenue_amount), "$ " +
                str(total_tax_amount), "Today's Sales: $" + str(today_total),  columns, data]

app.run_server(debug=True)


text:
import random
import unittest

import numpy

import chainer
from chainer import cuda
from chainer import functions
from chainer import gradient_check
from chainer import testing
from chainer.testing import attr

@testing.parameterize(*testing.product({
    'shape': [(3, 2), ()],
    'dtype': [numpy.float16, numpy.float32, numpy.float64],
}))
@testing.fix_random()
class TestELU(unittest.TestCase):

    def setUp(self):
        # Avoid unstability of numeraical grad
        self.x = numpy.random.uniform(-1, 1, self.shape).astype(self.dtype)
        self.x[(-0.01 < self.x) & (self.x < 0.01)] = 0.5
        self.gy = numpy.random.uniform(-1, 1, self.shape).astype(self.dtype)
        self.ggx = numpy.random.uniform(-1, 1, self.shape).astype(self.dtype)
        self.alpha = random.random()
        self.check_forward_options = {}
        self.check_backward_options = {}
        if self.dtype == numpy.float16:
            self.check_forward_options = {'atol': 5e-4, 'rtol': 5e-3}
            self.check_backward_options = {'atol': 5e-4, 'rtol': 5e-3}

    def check_forward(self, x_data):
        x = chainer.Variable(x_data)
        y = functions.elu(x, alpha=self.alpha)
        self.assertEqual(y.data.dtype, self.dtype)

        expected = self.x.copy()
        for i in numpy.ndindex(self.x.shape):
            if self.x[i] < 0:
                expected[i] = self.alpha * (numpy.exp(expected[i]) - 1)

        testing.assert_allclose(
            expected, y.data, **self.check_forward_options)

    def test_forward_cpu(self):
        self.check_forward(self.x)

    @attr.gpu
    def test_forward_gpu(self):
        self.check_forward(cuda.to_gpu(self.x))

    def check_backward(self, x_data, y_grad):
        def f(x):
            return functions.elu(x, alpha=self.alpha)

        gradient_check.check_backward(
            f, x_data, y_grad, dtype=numpy.float64,
            **self.check_backward_options)

    def test_backward_cpu(self):
        self.check_backward(self.x, self.gy)

    @attr.gpu
    def test_backward_gpu(self):
        self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(self.gy))

    def check_double_backward(self, x_data, y_grad, x_grad_grad):
        def f(x):
            y = functions.elu(x, alpha=self.alpha)
            return y * y

        gradient_check.check_double_backward(
            f, x_data, y_grad, x_grad_grad, dtype=numpy.float64,
            **self.check_backward_options)

    def test_double_backward_cpu(self):
        self.check_double_backward(self.x, self.gy, self.ggx)

    @attr.gpu
    def test_double_backward_gpu(self):
        self.check_double_backward(cuda.to_gpu(self.x), cuda.to_gpu(self.gy),
                                   cuda.to_gpu(self.ggx))

testing.run_module(__name__, __file__)

max_stars_repo_name: LuoYuanke/PrivChainer

max_issues_repo_name: LuoYuanke/PrivChainer

max_forks_repo_name: LuoYuanke/PrivChainer


text:
import matplotlib.pyplot as plt
import numpy as np
import re

plt.rcParams['axes.xmargin'] = 0
plt.rcParams['axes.ymargin'] = 0.01

episode = []
average_score = []
current_average = 0

fo = open("cms_2_alt.txt", "r")

for line in fo.readlines():
    search = re.match(r'(.*)episode [0-9]+(.*)', line, re.M|re.I)
    if search:
        #episode number
        e = float(re.findall(r'[0-9.]+', search.group())[9])
        if e not in episode:
            episode.append(e)
            average_score.append(current_average/7)
            current_average = 0
        avg = float(re.findall(r'[0-9.]+', search.group())[12])
        current_average += avg

line_up, = plt.plot(episode,average_score,label="Alternating Marines")

plt.xlabel('Episodes')
plt.ylabel('Average score')

episode_ca = []
average_score_ca = []
current_average_ca = 0

f1 = open("cms_2_alt_control_another.txt", "r")

for line in f1.readlines():
    search = re.match(r'(.*)episode [0-9]+(.*)', line, re.M|re.I)
    if search:
        #episode number
        e = float(re.findall(r'[0-9.]+', search.group())[9])
        if e not in episode_ca and e <= 4176:
            episode_ca.append(e)
            average_score_ca.append(current_average_ca/7)
            current_average_ca = 0
        avg = float(re.findall(r'[0-9.]+', search.group())[12])
        current_average_ca += avg

line_down, = plt.plot(episode_ca,average_score_ca, label="Grouped Marines")

episode_mc = []
average_score_mc = []
current_average_mc = 0

f2 = open("sc2g_a2c_multimovement_cms_1.txt", "r")

for line in f2.readlines():
    search = re.match(r'(.*)episode [0-9]+(.*)', line, re.M|re.I)
    if search:
        #episode number
        e = float(re.findall(r'[0-9.]+', search.group())[9])
        if e not in episode_mc:
            episode_mc.append(e)
            average_score_mc.append(current_average_mc/7)
            current_average_mc = 0
        avg = float(re.findall(r'[0-9.]+', search.group())[12])
        current_average_mc += avg
for x in range(2146, 4176):
    episode_mc.append(x)
    average_score_mc.append(10)

line_r, = plt.plot(episode_mc,average_score_mc, label="Two Pairs of Co-ordinates")
lgd = plt.legend(bbox_to_anchor=(1.05, 1), loc=2, handles=[line_up, line_down, line_r])

plt.savefig("raymond.png", bbox_extra_artists=(lgd,), bbox_inches='tight')

fo.close()