text:
from torch.utils.data import Dataset
import numpy as np
import torch
from sklearn.preprocessing import PowerTransformer

class LightDataset(Dataset):
    def __init__(self, row_id_to_idx, col_id_to_idx, propagation_scores, directed_pairs_list,
                 sources, terminals, normalization_method, samples_normalization_constants,
                 degree_feature_normalization_constants=None,
                 pairs_source_type=None, id_to_degree=None, train=True):
        self.row_id_to_idx = row_id_to_idx
        self.col_id_to_idx = col_id_to_idx
        self.col_idx_to_id = {xx: x for x, xx in self.col_id_to_idx.items()}
        self.propagation_scores = propagation_scores
        self.source_indexes = self.get_experiment_indexes(sources)
        self.terminal_indexes = self.get_experiment_indexes(terminals)
        self.pairs_indexes = [(self.col_id_to_idx[pair[0]], self.col_id_to_idx[pair[1]]) for pair in directed_pairs_list]
        self.longest_source = np.max([len(source) for source in sources.values()])
        self.longest_terminal = np.max([len(terminal) for terminal in terminals.values()])
        normalizer = self.get_normalization_method(normalization_method)
        self.normalizer = normalizer(samples_normalization_constants)
        self.pairs_source_type = pairs_source_type
        self.idx_to_degree = {self.col_id_to_idx[id]: id_to_degree[id] for id in self.col_id_to_idx.keys()}
        self.degree_normalizer = self.get_degree_normalizar(degree_feature_normalization_constants)
        self.idx_to_degree = {self.col_id_to_idx[id]: self.degree_normalizer(id_to_degree[id]) for id in self.col_id_to_idx.keys()}
        self.propagation_scores = self.normalizer(self.propagation_scores)
        self.train = train

    def __len__(self):
        return len(self.pairs_indexes) * 2

    def __getitem__(self, idx):
        if type(idx) == torch.Tensor:
            idx = idx.item()

        neg_flag = idx >= len(self.pairs_indexes)
        idx = np.mod(idx, len(self.pairs_indexes))
        label = 0 if neg_flag else 1
        pair = self.pairs_indexes[idx]
        if neg_flag:
            pair = (pair[1], pair[0])

        from_degree = self.idx_to_degree[pair[0]]
        to_degree = self.idx_to_degree[pair[1]]

        pair_source_type = self.pairs_source_type[idx] if self.pairs_source_type is not None else None
        source_sample = np.zeros((len(self.source_indexes), self.longest_source, 2))
        terminal_sample = np.zeros((len(self.source_indexes), self.longest_terminal, 2))

        for exp_idx in range(len(self.source_indexes)):
            source_sample[exp_idx, :len(self.source_indexes[exp_idx]), :] =\
                self.propagation_scores[:, pair][self.source_indexes[exp_idx], :]
            terminal_sample[exp_idx, :len(self.terminal_indexes[exp_idx]), :] =\
                self.propagation_scores[:, pair][self.terminal_indexes[exp_idx], :]

        return source_sample, terminal_sample, label, pair, pair_source_type, np.array([from_degree, to_degree])

    def get_experiment_indexes(self, experiment_id_sets):
        experiment_indexes = []
        for set in experiment_id_sets.values():
            experiment_indexes.append([self.row_id_to_idx[id] for id in set])

        return experiment_indexes

    @staticmethod
    def get_normalization_method(normalization_method):
        if normalization_method == 'standard':
            return StandardNormalizer
        elif normalization_method == 'power':
            return PowerNormalizer
        else:
            assert 0, '{} is not a valid normalization method name'.format(normalization_method)

    def get_degree_normalizar(self, normalization_constants):
        if normalization_constants is None:
            all_degrees = []
            for pair in self.pairs_indexes:
                all_degrees.append(self.idx_to_degree[pair[0]])
                all_degrees.append(self.idx_to_degree[pair[1]])
            all_degrees = np.array(all_degrees)[:, np.newaxis]
            pt = PowerTransformer(method='box-cox', standardize=False)
            transformed = pt.fit_transform(all_degrees)
            mean = np.mean(transformed)
            std = np.std(transformed)
            lmbda = pt.lambdas_[0]
        else:
            mean = normalization_constants['mean']
            std = normalization_constants['std']
            lmbda = normalization_constants['lmbda']
        return PowerNormalizer({'lmbda': lmbda, 'mean': mean, 'std': std})

class StandardNormalizer():
    def __init__(self, normalization_constants):
        self.dataset_mean = normalization_constants['mean']
        self.dataset_std = normalization_constants['std']

    def __call__(self, x, *args, **kwargs):
        return (x-self.dataset_mean) / self.dataset_std

class PowerNormalizer:
    """
    yeo_johnson transform
    """
    def __init__(self, normalization_constants):
        from scipy.stats import yeojohnson
        from functools import partial
        self.transformer = partial(yeojohnson, lmbda=normalization_constants['lmbda'])
        self.lmbda = normalization_constants['lmbda']
        self.mean = normalization_constants['mean']
        self.std = normalization_constants['std']

    def __call__(self, x, *args, **kwargs):
        return (self.transformer(x) - self.mean) / self.std

max_stars_repo_name: pirakd/DeepProp

max_issues_repo_name: pirakd/DeepProp

max_forks_repo_name: pirakd/DeepProp


text:
import pickle
import cv2
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.image as mpimg

# Read in the saved camera matrix and distortion coefficients
dist_pickle = pickle.load(open("dist_pickle.p", "rb"))
mtx = dist_pickle["mtx"]
dist = dist_pickle["dist"]

# Read in an image
img = cv2.imread('./test_images/test3.jpg')
#img = cv2.imread('./output_images/binary.jpg')

img_size = (img.shape[1], img.shape[0])

src2 = np.float32(
    [[(img_size[0] / 2) - 55, img_size[1] / 2 + 100],
    [((img_size[0] / 6) - 10), img_size[1]],
    [(img_size[0] * 5 / 6) + 60, img_size[1]],
    [(img_size[0] / 2 + 55), img_size[1] / 2 + 100]])
dst2 = np.float32(
    [[(img_size[0] / 4), 0],
    [(img_size[0] / 4), img_size[1]],
    [(img_size[0] * 3 / 4), img_size[1]],
    [(img_size[0] * 3 / 4), 0]])

p1 = [600,460]
p2 = [228,720]
p3 = [1142,720]
p4 = [720,460]

srcPts = np.float32([p1, p2, p3, p4])
dstPts = np.float32(
    [[(img_size[0] / 4), 0],
    [(img_size[0] / 4), img_size[1]],
    [(img_size[0] * 3 / 4), img_size[1]],
    [(img_size[0] * 3 / 4), 0]])

def draw_quadrangle(img, p1, p2, p3, p4, color=(0,0,255), pen_size=2):

    line_img = cv2.line(img, tuple(p1), tuple(p2), color, pen_size)
    line_img = cv2.line(line_img, tuple(p2), tuple(p3), color, pen_size)
    line_img = cv2.line(line_img, tuple(p3), tuple(p4), color, pen_size)
    line_img = cv2.line(line_img, tuple(p4), tuple(p1), color, pen_size)

    return line_img

def persp_trans(img, src, dst):

    M = cv2.getPerspectiveTransform(src, dst)
    Minv = cv2.getPerspectiveTransform(dst, src)

    persp_pickle = {}
    persp_pickle["PerspM"] = M
    persp_pickle["PerspMinv"] = Minv
    pickle.dump(persp_pickle, open("persp_pickle.p", "wb"))

    warped = cv2.warpPerspective(img, M, (img.shape[1], img.shape[0]), flags=cv2.INTER_LINEAR)
    unwarped = cv2.warpPerspective(warped, Minv, (img.shape[1], img.shape[0]), flags=cv2.INTER_LINEAR)

    return warped, unwarped, M

def undist_persp_trans(img, mtx, dist, src, dst):

    persp_pickle = {}

    undist = cv2.undistort(img, mtx, dist, None, mtx)
    line_img = cv2.line(undist, tuple(p1), tuple(p2), (0,0,255), 2)
    line_img = cv2.line(line_img, tuple(p2), tuple(p3), (0,0,255), 2)
    line_img = cv2.line(line_img, tuple(p3), tuple(p4), (0,0,255), 2)
    line_img = cv2.line(line_img, tuple(p4), tuple(p1), (0,0,255), 2)

    M = cv2.getPerspectiveTransform(src, dst)
    Minv = cv2.getPerspectiveTransform(dst, src)

    persp_pickle["PerspM"] = M
    persp_pickle["PerspMinv"] = Minv
    pickle.dump(persp_pickle, open("persp_pickle.p", "wb"))

    warped = cv2.warpPerspective(undist, M, (img.shape[1], img.shape[0]), flags=cv2.INTER_LINEAR)
    unwarped = cv2.warpPerspective(warped, Minv, (img.shape[1], img.shape[0]), flags=cv2.INTER_LINEAR)

    return warped, unwarped, line_img, undist, M

#warp, warp_inv, lines, undistorted, perspective_M = undist_persp_trans(img, mtx, dist, srcPts, dstPts)

undist = cv2.undistort(img, mtx, dist, None, mtx)
line_img = draw_quadrangle(undist, p1, p2, p3, p4)
warp, warp_inv, M = persp_trans(line_img, srcPts, dstPts)

cv2.imwrite('./output_images/lines.jpg', line_img)
cv2.imwrite('./output_images/warped.jpg', warp)
cv2.imwrite('./output_images/unwarped.jpg', warp_inv)

ext: py

reason: import numpy


text:
#
# Tests for the Scipy Solver class
#
import pybamm
import unittest
import numpy as np
from tests import get_mesh_for_testing
import warnings
import sys
from platform import system, version

class TestScipySolver(unittest.TestCase):
    def test_model_solver_python_and_jax(self):

        if not (
            system() == "Windows" or (system() == "Darwin" and "ARM64" in version())
        ):
            formats = ["python", "jax"]
        else:
            formats = ["python"]

        for convert_to_format in formats:
            # Create model
            model = pybamm.BaseModel()
            model.convert_to_format = convert_to_format
            domain = ["negative electrode", "separator", "positive electrode"]
            var = pybamm.Variable("var", domain=domain)
            model.rhs = {var: 0.1 * var}
            model.initial_conditions = {var: 1}
            # No need to set parameters;
            # can use base discretisation (no spatial operators)

            # create discretisation
            mesh = get_mesh_for_testing()
            spatial_methods = {"macroscale": pybamm.FiniteVolume()}
            disc = pybamm.Discretisation(mesh, spatial_methods)
            disc.process_model(model)
            # Solve
            # Make sure that passing in extra options works
            solver = pybamm.ScipySolver(
                rtol=1e-8, atol=1e-8, method="RK45", extra_options={"first_step": 1e-4}
            )
            t_eval = np.linspace(0, 1, 80)
            solution = solver.solve(model, t_eval)
            np.testing.assert_array_equal(solution.t, t_eval)
            np.testing.assert_allclose(solution.y[0], np.exp(0.1 * solution.t))

            # Test time
            self.assertEqual(
                solution.total_time, solution.solve_time + solution.set_up_time
            )
            self.assertEqual(solution.termination, "final time")

    def test_model_solver_failure(self):
        # Turn off warnings to ignore sqrt error
        warnings.simplefilter("ignore")
        model = pybamm.BaseModel()
        model.convert_to_format = "python"
        var = pybamm.Variable("var")
        model.rhs = {var: -pybamm.sqrt(var)}
        model.initial_conditions = {var: 1}
        # No need to set parameters; can use base discretisation (no spatial operators)

        # create discretisation
        disc = pybamm.Discretisation()
        disc.process_model(model)

        t_eval = np.linspace(0, 3, 100)
        solver = pybamm.ScipySolver(rtol=1e-8, atol=1e-8, method="RK45")
        # Expect solver to fail when y goes negative
        with self.assertRaises(pybamm.SolverError):
            solver.solve(model, t_eval)

        # Turn warnings back on
        warnings.simplefilter("default")

    def test_model_solver_with_event_python(self):
        # Create model
        model = pybamm.BaseModel()
        model.convert_to_format = "python"
        domain = ["negative electrode", "separator", "positive electrode"]
        var = pybamm.Variable("var", domain=domain)
        model.rhs = {var: -0.1 * var}
        model.initial_conditions = {var: 1}
        # needs to work with multiple events (to avoid bug where only last event is
        # used)
        model.events = [
            pybamm.Event("var=0.5", pybamm.min(var - 0.5)),
            pybamm.Event("var=-0.5", pybamm.min(var + 0.5)),
        ]
        # No need to set parameters; can use base discretisation (no spatial operators)

        # create discretisation
        mesh = get_mesh_for_testing()
        spatial_methods = {"macroscale": pybamm.FiniteVolume()}
        disc = pybamm.Discretisation(mesh, spatial_methods)
        disc.process_model(model)
        # Solve
        solver = pybamm.ScipySolver(rtol=1e-8, atol=1e-8, method="RK45")
        t_eval = np.linspace(0, 10, 100)
        solution = solver.solve(model, t_eval)
        self.assertLess(len(solution.t), len(t_eval))
        np.testing.assert_array_equal(solution.t[:-1], t_eval[: len(solution.t) - 1])
        np.testing.assert_allclose(solution.y[0], np.exp(-0.1 * solution.t))
        np.testing.assert_equal(solution.t_event[0], solution.t[-1])
        np.testing.assert_array_equal(solution.y_event[:, 0], solution.y[:, -1])

    def test_model_solver_ode_with_jacobian_python(self):
        # Create model
        model = pybamm.BaseModel()
        model.convert_to_format = "python"
        whole_cell = ["negative electrode", "separator", "positive electrode"]
        var1 = pybamm.Variable("var1", domain=whole_cell)
        var2 = pybamm.Variable("var2", domain=whole_cell)
        model.rhs = {var1: var1, var2: 1 - var1}
        model.initial_conditions = {var1: 1.0, var2: -1.0}
        model.variables = {"var1": var1, "var2": var2}

        # create discretisation
        mesh = get_mesh_for_testing()
        spatial_methods = {"macroscale": pybamm.FiniteVolume()}
        disc = pybamm.Discretisation(mesh, spatial_methods)
        disc.process_model(model)

        # Add user-supplied Jacobian to model
        combined_submesh = mesh.combine_submeshes(
            "negative electrode", "separator", "positive electrode"
        )
        N = combined_submesh.npts

        # construct jacobian in order of model.rhs
        J = []
        for var in model.rhs.keys():
            if var.id == var1.id:
                J.append([np.eye(N), np.zeros((N, N))])
            else:
                J.append([-1.0 * np.eye(N), np.zeros((N, N))])

        J = np.block(J)

        def jacobian(t, y):
            return J

        # Solve
        solver = pybamm.ScipySolver(rtol=1e-9, atol=1e-9)
        t_eval = np.linspace(0, 1, 100)
        solution = solver.solve(model, t_eval)
        np.testing.assert_array_equal(solution.t, t_eval)

        T, Y = solution.t, solution.y
        np.testing.assert_array_almost_equal(
            model.variables["var1"].evaluate(T, Y),
            np.ones((N, T.size)) * np.exp(T[np.newaxis, :]),
        )
        np.testing.assert_array_almost_equal(
            model.variables["var2"].evaluate(T, Y),
            np.ones((N, T.size)) * (T[np.newaxis, :] - np.exp(T[np.newaxis, :])),
        )

    def test_model_step_python(self):
        # Create model
        model = pybamm.BaseModel()
        model.convert_to_format = "python"
        domain = ["negative electrode", "separator", "positive electrode"]
        var = pybamm.Variable("var", domain=domain)
        model.rhs = {var: 0.1 * var}
        model.initial_conditions = {var: 1}

        # create discretisation
        mesh = get_mesh_for_testing()
        spatial_methods = {"macroscale": pybamm.FiniteVolume()}
        disc = pybamm.Discretisation(mesh, spatial_methods)
        disc.process_model(model)

        solver = pybamm.ScipySolver(rtol=1e-8, atol=1e-8, method="RK45")

        # Step once
        dt = 1
        step_sol = solver.step(None, model, dt)
        np.testing.assert_array_equal(step_sol.t, [0, dt])
        np.testing.assert_array_almost_equal(step_sol.y[0], np.exp(0.1 * step_sol.t))

        # Step again (return 5 points)
        step_sol_2 = solver.step(step_sol, model, dt, npts=5)
        np.testing.assert_array_equal(
            step_sol_2.t, np.concatenate([np.array([0]), np.linspace(dt, 2 * dt, 5)])
        )
        np.testing.assert_array_almost_equal(
            step_sol_2.y[0], np.exp(0.1 * step_sol_2.t)
        )

        # Check steps give same solution as solve
        t_eval = step_sol.t
        solution = solver.solve(model, t_eval)
        np.testing.assert_array_almost_equal(solution.y[0], step_sol.y[0])

    def test_step_different_model(self):
        disc = pybamm.Discretisation()

        # Create and discretise model1
        model1 = pybamm.BaseModel()
        var = pybamm.Variable("var")
        var2 = pybamm.Variable("var2")
        model1.rhs = {var: 0.1 * var}
        model1.initial_conditions = {var: 1}
        model1.variables = {"var": var, "mul_var": 2 * var, "var2": var}
        disc.process_model(model1)

        # Create and discretise model2, which is slightly different
        model2 = pybamm.BaseModel()
        var = pybamm.Variable("var")
        var2 = pybamm.Variable("var2")
        model2.rhs = {var: 0.2 * var, var2: -0.5 * var2}
        model2.initial_conditions = {var: 1, var2: 1}
        model2.variables = {"var": var, "mul_var": 3 * var, "var2": var2}
        disc.process_model(model2)

        solver = pybamm.ScipySolver(rtol=1e-8, atol=1e-8, method="RK45")

        # Step once
        dt = 1
        step_sol1 = solver.step(None, model1, dt)
        np.testing.assert_array_equal(step_sol1.t, [0, dt])
        np.testing.assert_array_almost_equal(step_sol1.y[0], np.exp(0.1 * step_sol1.t))

        # Step again, the model has changed
        step_sol2 = solver.step(step_sol1, model2, dt)
        np.testing.assert_array_equal(step_sol2.t, [0, dt, 2 * dt])
        np.testing.assert_array_almost_equal(
            step_sol2.all_ys[0][0], np.exp(0.1 * step_sol1.t)
        )

    def test_model_solver_with_inputs(self):
        # Create model
        model = pybamm.BaseModel()
        model.convert_to_format = "python"
        domain = ["negative electrode", "separator", "positive electrode"]
        var = pybamm.Variable("var", domain=domain)
        model.rhs = {var: -pybamm.InputParameter("rate") * var}
        model.initial_conditions = {var: 1}
        # No need to set parameters; can use base discretisation (no spatial
        # operators)
        model.events = [pybamm.Event("var=0.5", pybamm.min(var - 0.5))]
        # create discretisation
        mesh = get_mesh_for_testing()
        spatial_methods = {"macroscale": pybamm.FiniteVolume()}
        disc = pybamm.Discretisation(mesh, spatial_methods)
        disc.process_model(model)
        # Solve
        solver = pybamm.ScipySolver(rtol=1e-8, atol=1e-8, method="RK45")
        t_eval = np.linspace(0, 10, 100)
        solution = solver.solve(model, t_eval, inputs={"rate": 0.1})
        self.assertLess(len(solution.t), len(t_eval))
        np.testing.assert_array_equal(solution.t[:-1], t_eval[: len(solution.t) - 1])
        np.testing.assert_allclose(solution.y[0], np.exp(-0.1 * solution.t))

    def test_model_solver_multiple_inputs_happy_path(self):
        for convert_to_format in ["python", "casadi"]:
            # Create model
            model = pybamm.BaseModel()
            model.convert_to_format = convert_to_format
            domain = ["negative electrode", "separator", "positive electrode"]
            var = pybamm.Variable("var", domain=domain)
            model.rhs = {var: -pybamm.InputParameter("rate") * var}
            model.initial_conditions = {var: 1}
            # create discretisation
            mesh = get_mesh_for_testing()
            spatial_methods = {"macroscale": pybamm.FiniteVolume()}
            disc = pybamm.Discretisation(mesh, spatial_methods)
            disc.process_model(model)

            solver = pybamm.ScipySolver(rtol=1e-8, atol=1e-8, method="RK45")
            t_eval = np.linspace(0, 10, 100)
            ninputs = 8
            inputs_list = [{"rate": 0.01 * (i + 1)} for i in range(ninputs)]

            solutions = solver.solve(model, t_eval, inputs=inputs_list, nproc=2)
            for i in range(ninputs):
                with self.subTest(i=i):
                    solution = solutions[i]
                    np.testing.assert_array_equal(solution.t, t_eval)
                    np.testing.assert_allclose(
                        solution.y[0], np.exp(-0.01 * (i + 1) * solution.t)
                    )

    def test_model_solver_multiple_inputs_discontinuity_error(self):
        # Create model
        model = pybamm.BaseModel()
        model.convert_to_format = "casadi"
        domain = ["negative electrode", "separator", "positive electrode"]
        var = pybamm.Variable("var", domain=domain)
        model.rhs = {var: -pybamm.InputParameter("rate") * var}
        model.initial_conditions = {var: 1}
        # create discretisation
        mesh = get_mesh_for_testing()
        spatial_methods = {"macroscale": pybamm.FiniteVolume()}
        disc = pybamm.Discretisation(mesh, spatial_methods)
        disc.process_model(model)

        solver = pybamm.ScipySolver(rtol=1e-8, atol=1e-8, method="RK45")
        t_eval = np.linspace(0, 10, 100)
        ninputs = 8
        inputs_list = [{"rate": 0.01 * (i + 1)} for i in range(ninputs)]

        model.events = [
            pybamm.Event(
                "discontinuity",
                pybamm.Scalar(t_eval[-1] / 2),
                event_type=pybamm.EventType.DISCONTINUITY,
            )
        ]
        with self.assertRaisesRegex(
            pybamm.SolverError,
            (
                "Cannot solve for a list of input parameters"
                " sets with discontinuities"
            ),
        ):
            solver.solve(model, t_eval, inputs=inputs_list, nproc=2)

    def test_model_solver_multiple_inputs_initial_conditions_error(self):
        # Create model
        model = pybamm.BaseModel()
        model.convert_to_format = "casadi"
        domain = ["negative electrode", "separator", "positive electrode"]
        var = pybamm.Variable("var", domain=domain)
        model.rhs = {var: -pybamm.InputParameter("rate") * var}
        model.initial_conditions = {var: 2 * pybamm.InputParameter("rate")}
        # create discretisation
        mesh = get_mesh_for_testing()
        spatial_methods = {"macroscale": pybamm.FiniteVolume()}
        disc = pybamm.Discretisation(mesh, spatial_methods)
        disc.process_model(model)

        solver = pybamm.ScipySolver(rtol=1e-8, atol=1e-8, method="RK45")
        t_eval = np.linspace(0, 10, 100)
        ninputs = 8
        inputs_list = [{"rate": 0.01 * (i + 1)} for i in range(ninputs)]

        with self.assertRaisesRegex(
            pybamm.SolverError,
            ("Input parameters cannot appear in expression " "for initial conditions."),
        ):
            solver.solve(model, t_eval, inputs=inputs_list, nproc=2)

    def test_model_solver_multiple_inputs_jax_format_error(self):
        # Create model
        model = pybamm.BaseModel()
        model.convert_to_format = "jax"
        domain = ["negative electrode", "separator", "positive electrode"]
        var = pybamm.Variable("var", domain=domain)
        model.rhs = {var: -pybamm.InputParameter("rate") * var}
        model.initial_conditions = {var: 2 * pybamm.InputParameter("rate")}
        # No need to set parameters; can use base discretisation (no spatial
        # operators)
        # create discretisation
        mesh = get_mesh_for_testing()
        spatial_methods = {"macroscale": pybamm.FiniteVolume()}
        disc = pybamm.Discretisation(mesh, spatial_methods)
        disc.process_model(model)

        solver = pybamm.ScipySolver(rtol=1e-8, atol=1e-8, method="RK45")
        t_eval = np.linspace(0, 10, 100)
        ninputs = 8
        inputs_list = [{"rate": 0.01 * (i + 1)} for i in range(ninputs)]

        with self.assertRaisesRegex(
            pybamm.SolverError,
            (
                "Cannot solve list of inputs with multiprocessing "
                'when model in format "jax".'
            ),
        ):
            solver.solve(model, t_eval, inputs=inputs_list, nproc=2)

    def test_model_solver_with_external(self):
        # Create model
        model = pybamm.BaseModel()
        domain = ["negative electrode", "separator", "positive electrode"]
        var1 = pybamm.Variable("var1", domain=domain)
        var2 = pybamm.Variable("var2", domain=domain)
        model.rhs = {var1: -var2}
        model.initial_conditions = {var1: 1}
        model.external_variables = [var2]
        model.variables = {"var2": var2}
        # No need to set parameters; can use base discretisation (no spatial
        # operators)

        # create discretisation
        mesh = get_mesh_for_testing()
        spatial_methods = {"macroscale": pybamm.FiniteVolume()}
        disc = pybamm.Discretisation(mesh, spatial_methods)
        disc.process_model(model)
        # Solve
        solver = pybamm.ScipySolver(rtol=1e-8, atol=1e-8)
        t_eval = np.linspace(0, 10, 100)
        solution = solver.solve(
            model, t_eval, external_variables={"var2": 0.5 * np.ones(100)}
        )
        np.testing.assert_allclose(solution.y[0], 1 - 0.5 * solution.t, rtol=1e-06)

    def test_model_solver_with_event_with_casadi(self):
        # Create model
        model = pybamm.BaseModel()
        for use_jacobian in [True, False]:
            model.use_jacobian = use_jacobian
            model.convert_to_format = "casadi"
            domain = ["negative electrode", "separator", "positive electrode"]
            var = pybamm.Variable("var", domain=domain)
            model.rhs = {var: -0.1 * var}
            model.initial_conditions = {var: 1}
            # needs to work with multiple events (to avoid bug where only last event is
            # used)
            model.events = [
                pybamm.Event("var=0.5", pybamm.min(var - 0.5)),
                pybamm.Event("var=-0.5", pybamm.min(var + 0.5)),
            ]
            # No need to set parameters; can use base discretisation (no spatial
            # operators)

            # create discretisation
            mesh = get_mesh_for_testing()
            spatial_methods = {"macroscale": pybamm.FiniteVolume()}
            disc = pybamm.Discretisation(mesh, spatial_methods)
            model_disc = disc.process_model(model, inplace=False)
            # Solve
            solver = pybamm.ScipySolver(rtol=1e-8, atol=1e-8, method="RK45")
            t_eval = np.linspace(0, 10, 100)
            solution = solver.solve(model_disc, t_eval)
            self.assertLess(len(solution.t), len(t_eval))
            np.testing.assert_array_equal(
                solution.t[:-1], t_eval[: len(solution.t) - 1]
            )
            np.testing.assert_allclose(solution.y[0], np.exp(-0.1 * solution.t))

    def test_model_solver_with_inputs_with_casadi(self):
        # Create model
        model = pybamm.BaseModel()
        model.convert_to_format = "casadi"
        domain = ["negative electrode", "separator", "positive electrode"]
        var = pybamm.Variable("var", domain=domain)
        model.rhs = {var: -pybamm.InputParameter("rate") * var}
        model.initial_conditions = {var: 1}
        model.events = [pybamm.Event("var=0.5", pybamm.min(var - 0.5))]
        # No need to set parameters; can use base discretisation (no spatial
        # operators)

        # create discretisation
        mesh = get_mesh_for_testing()
        spatial_methods = {"macroscale": pybamm.FiniteVolume()}
        disc = pybamm.Discretisation(mesh, spatial_methods)
        disc.process_model(model)
        # Solve
        solver = pybamm.ScipySolver(rtol=1e-8, atol=1e-8, method="RK45")
        t_eval = np.linspace(0, 10, 100)
        solution = solver.solve(model, t_eval, inputs={"rate": 0.1})
        self.assertLess(len(solution.t), len(t_eval))
        np.testing.assert_array_equal(solution.t[:-1], t_eval[: len(solution.t) - 1])
        np.testing.assert_allclose(solution.y[0], np.exp(-0.1 * solution.t))

    def test_model_solver_inputs_in_initial_conditions(self):
        # Create model
        model = pybamm.BaseModel()
        var1 = pybamm.Variable("var1")
        model.rhs = {var1: pybamm.InputParameter("rate") * var1}
        model.initial_conditions = {var1: pybamm.InputParameter("ic 1")}

        # Solve
        solver = pybamm.ScipySolver(rtol=1e-8, atol=1e-8)
        t_eval = np.linspace(0, 5, 100)
        solution = solver.solve(model, t_eval, inputs={"rate": -1, "ic 1": 0.1})
        np.testing.assert_array_almost_equal(
            solution.y[0], 0.1 * np.exp(-solution.t), decimal=5
        )

        # Solve again with different initial conditions
        solution = solver.solve(model, t_eval, inputs={"rate": -0.1, "ic 1": 1})
        np.testing.assert_array_almost_equal(
            solution.y[0], 1 * np.exp(-0.1 * solution.t), decimal=5
        )

    def test_model_solver_manually_update_initial_conditions(self):
        # Create model
        model = pybamm.BaseModel()
        var1 = pybamm.Variable("var1")
        model.rhs = {var1: -var1}
        model.initial_conditions = {var1: 1}

        # Solve
        solver = pybamm.ScipySolver(rtol=1e-8, atol=1e-8)
        t_eval = np.linspace(0, 5, 100)
        solution = solver.solve(model, t_eval)
        np.testing.assert_array_almost_equal(
            solution.y[0], 1 * np.exp(-solution.t), decimal=5
        )

        # Change initial conditions and solve again
        model.concatenated_initial_conditions = pybamm.NumpyConcatenation(
            pybamm.Vector([[2]])
        )
        solution = solver.solve(model, t_eval)
        np.testing.assert_array_almost_equal(
            solution.y[0], 2 * np.exp(-solution.t), decimal=5
        )

if __name__ == "__main__":
    print("Add -v for more debug output")

    if "-v" in sys.argv:
        debug = True
    pybamm.settings.debug_mode = True
    unittest.main()

max_stars_repo_name: tobykirk/PyBaMM

max_issues_repo_name: tobykirk/PyBaMM

max_forks_repo_name: tobykirk/PyBaMM


text:
import unittest
import numpy as np
import numpy.testing as npt
from wilson.run.smeft import beta
from wilson.util import smeftutil
from wilson.util.common import EFTutil
from wilson.run.smeft.tests import test_beta
from wilson.test_wilson import get_random_wc
from wilson import wcxf
from numbers import Number

C = test_beta.C.copy()
for i in C:
    if i in smeftutil.WC_keys_2f + smeftutil.WC_keys_4f:
        # make Wilson coefficients involving fermions complex!
        C[i] = C[i] + 1j*C[i]

class TestSymm(unittest.TestCase):
    def test_keys(self):
        # check no parameter or WC was forgotten in the C_symm_keys lists
        self.assertEqual(
          set(smeftutil.C_keys),
          {c for cs in smeftutil.C_symm_keys.values() for c in cs}
        )

    def test_symmetrize_symmetric(self):
        a = np.array([[1, 2, 3], [2, 4, 5], [3, 5, 6]])
        npt.assert_array_equal(EFTutil.symmetrize_2(a), a)
        b = np.array([[1, 2, 3], [0, 4, 5], [0, 0, 6]])
        npt.assert_array_equal(EFTutil.symmetrize_2(b), a)

    def test_symmetrize_hermitian(self):
        a = np.array([[1, 2j, 3j], [-2j, 4, 5j], [-3j, -5j, 6]])
        npt.assert_array_equal(EFTutil.symmetrize_2(a), a)
        b = np.array([[1, 2j, 3j], [0, 4, 5j], [0, 0, 6]])
        npt.assert_array_equal(EFTutil.symmetrize_2(b), a)

    def test_symmetrize_C(self):
        C_symm = smeftutil.symmetrize(C)
        # check all keys are present
        self.assertSetEqual(set(C.keys()), set(C_symm.keys()))
        for i, v in C_symm.items():
            # check trivial cases are the same
            if i in smeftutil.C_symm_keys[0] + smeftutil.C_symm_keys[1] + smeftutil.C_symm_keys[3]:
                if smeftutil.C_keys_shape[i] == 1:
                    self.assertEqual(v, C[i])
                else:
                    npt.assert_array_equal(v, C[i])
            # check symmetric
            elif i in smeftutil.C_symm_keys[9]:
                npt.assert_array_equal(v, v.T)
            # check hermitian
            elif i in smeftutil.C_symm_keys[2]:
                npt.assert_array_equal(v, v.T.conj())
            # check 2 identical FFbar
            elif i in smeftutil.C_symm_keys[4]:
                npt.assert_array_equal(v, v.transpose((2, 3, 0, 1)))
                npt.assert_array_equal(v, v.transpose((1, 0, 3, 2)).conj())
            # check 2 independent FFbar
            elif i in smeftutil.C_symm_keys[5]:
                npt.assert_array_equal(v, v.transpose((1, 0, 3, 2)).conj())
            # check special case ee
            elif i in smeftutil.C_symm_keys[6]:
                npt.assert_array_equal(v, v.transpose((2, 3, 0, 1)))
                npt.assert_array_equal(v, v.transpose((0, 3, 2, 1)))
                npt.assert_array_equal(v, v.transpose((2, 1, 0, 3)))
            # check special case qque
            elif i in smeftutil.C_symm_keys[7]:
                npt.assert_array_equal(v, v.transpose((1, 0, 2, 3)))
            # check special case qqql
            elif i in smeftutil.C_symm_keys[8]:
                # see eq. (10) of arXiv:1405.0486
                npt.assert_array_almost_equal(v + v.transpose((1, 0, 2, 3)), v.transpose((1, 2, 0, 3)) + v.transpose((2, 1, 0, 3)), decimal=15)

    def test_wcxf2array(self):
        wc = get_random_wc('SMEFT', 'Warsaw', 160)
        C = smeftutil.wcxf2arrays_symmetrized(wc.dict)
        d = smeftutil.arrays2wcxf_nonred(C)
        self.assertEqual(wc.dict.keys(), d.keys())
        for k, v in wc.dict.items():
            self.assertAlmostEqual(v, d[k], msg=f"Failed for {k}")

    def test_wcxf2array_incomplete(self):
        wc = wcxf.WC('SMEFT', 'Warsaw', 160, {'G': 1e-10})
        C = smeftutil.wcxf2arrays_symmetrized(wc.dict)
        d = smeftutil.arrays2wcxf_nonred(C)
        self.assertEqual(wc.dict.keys(), d.keys())
        for k, v in d.items():
            self.assertEqual(v, wc[k], msg=f"Failed for {k}")
            self.assertIsInstance(v, Number)

class TestKeysShapesSymm(unittest.TestCase):

    # names of SM parameters
    SM_keys = ['g', 'gp', 'gs', 'Lambda', 'm2', 'Gu', 'Gd', 'Ge',]

    # names of WCs with 0, 2, or 4 fermions (i.e. scalars, 3x3 matrices,
    # and 3x3x3x3 tensors)
    WC_keys_0f = ["G", "Gtilde", "W", "Wtilde", "phi", "phiBox", "phiD", "phiG",
                  "phiB", "phiW", "phiWB", "phiGtilde", "phiBtilde", "phiWtilde",
                  "phiWtildeB"]
    WC_keys_2f = ["uphi", "dphi", "ephi", "eW", "eB", "uG", "uW", "uB", "dG", "dW",
                  "dB", "phil1", "phil3", "phie", "phiq1", "phiq3", "phiu", "phid",
                  "phiud", "llphiphi"]
    WC_keys_4f = ["ll", "qq1", "qq3", "lq1", "lq3", "ee", "uu", "dd", "eu", "ed",
                  "ud1", "ud8", "le", "lu", "ld",  "qe", "qu1", "qd1", "qu8",
                  "qd8", "ledq", "quqd1", "quqd8", "lequ1", "lequ3", "duql",
                  "qque", "qqql", "duue"]

    C_keys = SM_keys + WC_keys_0f + WC_keys_2f + WC_keys_4f
    WC_keys = WC_keys_0f + WC_keys_2f + WC_keys_4f

    C_keys_shape = {
       'g': 1,
       'gp': 1,
       'gs': 1,
       'Lambda': 1,
       'm2': 1,
       'Gu': (3, 3),
       'Gd': (3, 3),
       'Ge': (3, 3),
       'G': 1,
       'Gtilde': 1,
       'W': 1,
       'Wtilde': 1,
       'phi': 1,
       'phiBox': 1,
       'phiD': 1,
       'phiG': 1,
       'phiB': 1,
       'phiW': 1,
       'phiWB': 1,
       'phiGtilde': 1,
       'phiBtilde': 1,
       'phiWtilde': 1,
       'phiWtildeB': 1,
       'uphi': (3, 3),
       'dphi': (3, 3),
       'ephi': (3, 3),
       'eW': (3, 3),
       'eB': (3, 3),
       'uG': (3, 3),
       'uW': (3, 3),
       'uB': (3, 3),
       'dG': (3, 3),
       'dW': (3, 3),
       'dB': (3, 3),
       'phil1': (3, 3),
       'phil3': (3, 3),
       'phie': (3, 3),
       'phiq1': (3, 3),
       'phiq3': (3, 3),
       'phiu': (3, 3),
       'phid': (3, 3),
       'phiud': (3, 3),
       'llphiphi': (3, 3),
       'll': (3, 3, 3, 3),
       'qq1': (3, 3, 3, 3),
       'qq3': (3, 3, 3, 3),
       'lq1': (3, 3, 3, 3),
       'lq3': (3, 3, 3, 3),
       'ee': (3, 3, 3, 3),
       'uu': (3, 3, 3, 3),
       'dd': (3, 3, 3, 3),
       'eu': (3, 3, 3, 3),
       'ed': (3, 3, 3, 3),
       'ud1': (3, 3, 3, 3),
       'ud8': (3, 3, 3, 3),
       'le': (3, 3, 3, 3),
       'lu': (3, 3, 3, 3),
       'ld': (3, 3, 3, 3),
       'qe': (3, 3, 3, 3),
       'qu1': (3, 3, 3, 3),
       'qd1': (3, 3, 3, 3),
       'qu8': (3, 3, 3, 3),
       'qd8': (3, 3, 3, 3),
       'ledq': (3, 3, 3, 3),
       'quqd1': (3, 3, 3, 3),
       'quqd8': (3, 3, 3, 3),
       'lequ1': (3, 3, 3, 3),
       'lequ3': (3, 3, 3, 3),
       'duql': (3, 3, 3, 3),
       'qque': (3, 3, 3, 3),
       'qqql': (3, 3, 3, 3),
       'duue': (3, 3, 3, 3),
    }

    # names of Wilson coefficients with the same fermionic symmetry properties
    C_symm_keys = {}
    # 0 0F scalar object
    C_symm_keys[0] = WC_keys_0f + ['g', 'gp', 'gs', 'Lambda', 'm2',]
    # 1 2F general 3x3 matrix
    C_symm_keys[1] = ["uphi", "dphi", "ephi", "eW", "eB", "uG", "uW", "uB", "dG", "dW", "dB", "phiud"] + ['Gu', 'Gd', 'Ge']
    # 2 2F Hermitian matrix
    C_symm_keys[2] = ["phil1", "phil3", "phie", "phiq1", "phiq3", "phiu", "phid",]
    # 3 4F general 3x3x3x3 object
    C_symm_keys[3] = ["ledq", "quqd1", "quqd8", "lequ1", "lequ3", "duql", "duue"]
    # 4 4F two identical ffbar currents
    C_symm_keys[4] = ["ll", "qq1", "qq3", "uu", "dd",]
    # 5 4F two independent ffbar currents
    C_symm_keys[5] = ["lq1", "lq3", "eu", "ed", "ud1", "ud8", "le", "lu", "ld", "qe", "qu1", "qd1", "qu8", "qd8",]
    # 6 4F two identical ffbar currents - special case Cee
    C_symm_keys[6] = ["ee",]
    # 7 4F Baryon-number-violating - special case Cqque
    C_symm_keys[7] = ["qque",]
    # 8 4F Baryon-number-violating - special case Cqqql
    C_symm_keys[8] = ["qqql",]
    # 9 2F symmetric matrix
    C_symm_keys[9] = ["llphiphi"]

    def test_keys(self):
        self.assertEqual(self.SM_keys, smeftutil.dim4_keys)
        self.assertEqual(self.WC_keys_0f, smeftutil.WC_keys_0f)
        self.assertEqual(self.WC_keys_2f, smeftutil.WC_keys_2f)
        self.assertEqual(self.WC_keys_4f, smeftutil.WC_keys_4f)
        self.assertEqual(self.C_keys, smeftutil.C_keys)
        self.assertEqual(self.WC_keys, smeftutil.WC_keys)

    def test_shape(self):
        self.assertEqual(self.C_keys_shape, smeftutil.C_keys_shape)

    def test_symm_keys(self):
        self.assertEqual(self.C_symm_keys.keys(), smeftutil.C_symm_keys.keys())
        for k, v in self.C_symm_keys.items():
            self.assertEqual(set(v), set(smeftutil.C_symm_keys[k]))

    def test_needs_padding(self):
        self.assertEqual(smeftutil._needs_padding, False)

max_stars_repo_name: jackypheno/wilson

max_issues_repo_name: jackypheno/wilson

max_forks_repo_name: jackypheno/wilson


text:
# Stencil Computation for PYNQ 2021.04.29 Naoki F., AIT
# ライセンスについては LICENSE.txt を参照してください．

import numpy as np
from scipy.signal import convolve2d
from time import perf_counter

WEIGHT = np.ones((3, 3), dtype=np.uint32)

# from init_buf()
def init_buf(buf):
    buf.fill(0)
    for i in range(4, N, 8):
        for j in range(4, N, 8):
            buf[i, j] = (i << 16) + (j << 4)
    buf[4, 4] = 0x0fffffff

# from stencil_soft()
def stencil_soft(src, dst, coproc):
    for y in range(1, N - 1):
        for x in range(1, N - 1):
            dst[y, x] = (src[y-1, x-1] + src[y-1, x] + src[y-1, x+1] +
                         src[y  , x-1] + src[y  , x] + src[y  , x+1] +
                         src[y+1, x-1] + src[y+1, x] + src[y+1, x+1]) // 9
    dst[4, 4] = 0x0fffffff

def stencil_scipy(src, dst, coproc):
    dst[1:-1, 1:-1] = convolve2d(src[1:-1, 1:-1], WEIGHT, mode="same") // 9
    dst[4, 4] = 0x0fffffff

# from printresult()
def printresult(dst, elapsed):
    for y in range(16):
        for x in range(8):
            print("%08x " % dst[y, x], end="")
        print("")
    print("checksum         : %08x" % np.sum(dst, dtype=np.uint32))
    print("elapsed time [ms]: %.3f" % (elapsed * 1000))
    print("")

def evaluate(func, buf1, buf2, coproc=None):
    start_time = perf_counter()
    for i in range(ITER):
        func(buf1, buf2, coproc)
        buf1, buf2 = buf2, buf1
    end_time = perf_counter()
    printresult(buf1, end_time - start_time)

# for PYNQ
ADDR_GO   = 0x0
ADDR_DONE = 0x0
ADDR_SIZE = 0x4
ADDR_SRC  = 0x8
ADDR_DST  = 0xc

# from stencil_hard()
def stencil_hard(src, dst, coproc):
    coproc.write(ADDR_SIZE, N)
    coproc.write(ADDR_SRC, src.device_address)
    coproc.write(ADDR_DST, dst.device_address)
    coproc.write(ADDR_GO, 1)
    while coproc.read(ADDR_DONE) == 1:
        pass
    coproc.write(ADDR_GO, 0)
    while coproc.read(ADDR_DONE) == 0:
        pass

# main routine
from pynq import Overlay
from pynq import allocate
import numpy as np

N = 512
ITER = 100

print("== STENCIL ==")
print("size = %d, iteration = %d" % (N, ITER))
print("")

pl = Overlay("stencil.bit")
coproc = pl.stencil_top_0
buf1 = np.ndarray((N, N), dtype=np.uint32)
buf2 = np.ndarray((N, N), dtype=np.uint32)
hbuf1 = allocate(shape=(N, N), dtype=np.uint32)
hbuf2 = allocate(shape=(N, N), dtype=np.uint32)

print("-- SOFTWARE (w/o SciPy) --")
init_buf(buf1)
init_buf(buf2)
evaluate(stencil_soft, buf1, buf2)

print("-- SOFTWARE (w/ SciPy) --")
init_buf(buf1)
init_buf(buf2)
evaluate(stencil_scipy, buf1, buf2)

print("-- HARDWARE --")
init_buf(hbuf1)
init_buf(hbuf2)
evaluate(stencil_hard, hbuf1, hbuf2, coproc)

hbuf1.freebuffer()
hbuf2.freebuffer()

reason: import numpy,from scipy


text:
import numpy as np
import torch.utils.data

class BiosBaseDataset(torch.utils.data.Dataset):
    def __init__(self, bios, titles, **kwargs):
        super().__init__()

        self.bios = bios
        self.titles = titles

        if len(kwargs) != 0:
            self.extra_targets = kwargs
        else:
            self.extra_targets = None

        self.nb_classes = len(set(self.titles))

    def __getitem__(self, index):
        bio = self.bios[index]
        title = self.titles[index]

        if self.extra_targets is None:
            targets = title
        else:
            extra_targets = [self.extra_targets[k][index] for k in self.extra_targets.keys()]
            targets = (title, *extra_targets)

        return bio, targets

    def __len__(self):
        return len(self.titles)

class BiosCountsDataset(BiosBaseDataset):
    def __init__(self, feature_names, *args, **kwargs):
        super().__init__(*args, **kwargs)

        self.feature_names = feature_names
        self.nb_features = len(feature_names)

    def __getitem__(self, index):
        bio, targets = super().__getitem__(index)

        bio = bio.astype(np.float32)
        if not isinstance(bio, np.ndarray):
            bio = bio.toarray().squeeze()

        return bio, targets

class BiosSeqDataset(BiosBaseDataset):
    def __init__(self, vocab, *args, **kwargs):
        super().__init__(*args, **kwargs)

        self.vocab = vocab
        self.vocab_size = len(vocab) + 1  # +1 for 0 pad
        self.max_len = self.bios.shape[1]

max_stars_repo_name: MSR-LIT/MultilingualBias

max_issues_repo_name: MSR-LIT/MultilingualBias

max_forks_repo_name: MSR-LIT/MultilingualBias


text:
import argparse
import os
import random

import numpy as np
import torch
from torch.optim import Adam

from buffer import RolloutStorage
from common.wrappers import make_atari, wrap_deepmind, wrap_pytorch
from config import Config
from core.util import get_class_attr_val
from model import CnnDQN, Dueling_DQN
from tester import Tester
from trainer import Trainer

class CnnDDQNAgent:

    def __init__(self, config: Config):
        self.config = config
        self.is_training = True
        self.buffer = RolloutStorage(config)
        if self.config.Dueling_DQN:
            self.model = Dueling_DQN(self.config.state_shape,
                                     self.config.action_dim)
            self.target_model = Dueling_DQN(self.config.state_shape,
                                            self.config.action_dim)
        else:
            self.model = CnnDQN(self.config.state_shape,
                                self.config.action_dim)
            self.target_model = CnnDQN(self.config.state_shape,
                                       self.config.action_dim)
        self.target_model.load_state_dict(self.model.state_dict())
        self.model_optim = torch.optim.Adam(
            self.model.parameters(), lr=self.config.learning_rate)

        if self.config.use_cuda:
            self.cuda()

    def act(self, state, epsilon=None):
        if epsilon is None:
            epsilon = self.config.epsilon_min
        if random.random() > epsilon or not self.is_training:
            state = torch.tensor(state, dtype=torch.float) / 255.0
            if self.config.use_cuda:
                state = state.to(self.config.device)
            q_value = self.model.forward(state)
            action = q_value.max(1)[1].item()
        else:
            action = random.randrange(self.config.action_dim)
        return action

    def learning(self, fr):
        s0, s1, a, r, done = self.buffer.sample(self.config.batch_size)
        if self.config.use_cuda:
            s0 = s0.float().to(self.config.device) / 255.0
            s1 = s1.float().to(self.config.device) / 255.0
            a = a.to(self.config.device)
            r = r.to(self.config.device)
            done = done.to(self.config.device)

        # How to calculate Q(s,a) for all actions
        # q_values is a vector with size (batch_size, action_shape, 1)
        # each dimension i represents Q(s0,a_i)
        q_s0_values = self.model(s0).cuda()

        # How to calculate argmax_a Q(s,a)
        # actions = q_values.max(1)[1]
        q_s0_a = torch.gather(q_s0_values, 1, a)

        # Tips: function torch.gather may be helpful
        # You need to design how to calculate the loss
        if self.config.DQN:
            q_target_s1_values = self.target_model(s1).cuda().detach()
            q_target_s1_a_prime = q_target_s1_values.max(1)[0].unsqueeze(1)
            # if current state is end of episode, then there  is no next Q value
            q_target_s1_a_prime = torch.mul(q_target_s1_a_prime, (1 - done))
            y = r + self.config.gamma * q_target_s1_a_prime
        elif self.config.Double_DQN:
            q_s1_values = self.model(s1).cuda().detach()
            s1_a_prime = q_s1_values.max(1)[1].unsqueeze(1)
            q_target_s1_values = self.target_model(s1).cuda().detach()
            q_target_s1_a_prime = torch.gather(q_target_s1_values, 1, s1_a_prime)
            q_target_s1_a_prime = torch.mul(q_target_s1_a_prime, (1 - done))
            y = r + self.config.gamma * q_target_s1_a_prime
        else:
            pass
        mse_loss = torch.nn.MSELoss()
        loss = mse_loss(q_s0_a, y)
        self.model_optim.zero_grad()
        loss.backward()
        self.model_optim.step()

        if fr % self.config.update_tar_interval == 0:
            self.target_model.load_state_dict(self.model.state_dict())
        return loss.item()

    def cuda(self):
        self.model.to(self.config.device)
        self.target_model.to(self.config.device)

    def load_weights(self, model_path):
        model = torch.load(model_path)
        if 'model' in model:
            self.model.load_state_dict(model['model'])
        else:
            self.model.load_state_dict(model)

    def save_model(self, output, name=''):
        torch.save(self.model.state_dict(), '%s/model_%s.pkl' % (output, name))

    def save_config(self, output):
        with open(output + '/config.txt', 'w') as f:
            attr_val = get_class_attr_val(self.config)
            for k, v in attr_val.items():
                f.write(str(k) + " = " + str(v) + "\n")

    def save_checkpoint(self, fr, output):
        checkpath = output + '/checkpoint_model'
        os.makedirs(checkpath, exist_ok=True)
        torch.save({
            'frames': fr,
            'model': self.model.state_dict()
        }, '%s/checkpoint_fr_%d.tar' % (checkpath, fr))

    def load_checkpoint(self, model_path):
        checkpoint = torch.load(model_path)
        fr = checkpoint['frames']
        self.model.load_state_dict(checkpoint['model'])
        self.target_model.load_state_dict(checkpoint['model'])
        return fr

if __name__ == '__main__':

    parser = argparse.ArgumentParser(description='')
    parser.add_argument(
        '--train', dest='train', action='store_true', help='train model')
    parser.add_argument(
        '--env',
        default='PongNoFrameskip-v4',
        type=str,
        help='gym environment')
    parser.add_argument(
        '--test', dest='test', action='store_true', help='test model')
    parser.add_argument(
        '--retrain', dest='retrain', action='store_true', help='retrain model')
    parser.add_argument(
        '--model_path', type=str, help='if test or retrain, import the model')
    parser.add_argument(
        '--no-cuda',
        action='store_true',
        default=False,
        help='disables CUDA training')
    parser.add_argument(
        '--cuda_id',
        type=str,
        default='0',
        help='if test or retrain, import the model')
    args = parser.parse_args()
    args.cuda = not args.no_cuda and torch.cuda.is_available()

    config = Config()
    config.env = args.env
    config.DQN = True
    config.Double_DQN = False
    config.Dueling_DQN = False
    config.gamma = 0.99
    config.epsilon = 1
    config.epsilon_min = 0.01
    config.eps_decay = 30000
    config.frames = 2000000
    config.use_cuda = args.cuda
    config.learning_rate = 1e-4
    config.init_buff = 10000
    config.max_buff = 100000
    config.learning_interval = 4
    config.update_tar_interval = 1000
    config.batch_size = 32
    config.gif_interval = 20000
    config.print_interval = 5000
    config.log_interval = 5000
    config.checkpoint = True
    config.checkpoint_interval = 500000
    config.win_reward = 18
    config.win_break = True
    config.device = torch.device("cuda:" +
                                 args.cuda_id if args.cuda else "cpu")
    # handle the atari env
    env = make_atari(config.env)
    env = wrap_deepmind(env)
    env = wrap_pytorch(env)

    config.action_dim = env.action_space.n
    config.state_shape = env.observation_space.shape
    print(config.action_dim, config.state_shape)
    agent = CnnDDQNAgent(config)

    if args.train:
        trainer = Trainer(agent, env, config)
        trainer.train()

    elif args.test:
        if args.model_path is None:
            print('please add the model path:', '--model_path xxxx')
            exit(0)
        tester = Tester(agent, env, args.model_path)
        tester.test()

    elif args.retrain:
        if args.model_path is None:
            print('please add the model path:', '--model_path xxxx')
            exit(0)

        fr = agent.load_checkpoint(args.model_path)
        trainer = Trainer(agent, env, config)
        trainer.train(fr)


text:
"""
Generic data algorithms. This module is experimental at the moment and not
intended for public consumption
"""

import numpy as np

import pandas.core.common as com
import pandas.lib as lib
import pandas._algos as _algos

def match(to_match, values, na_sentinel=-1):
    """
    Compute locations of to_match into values

    Parameters
    ----------
    to_match : array-like
        values to find positions of
    values : array-like
        Unique set of values
    na_sentinel : int, default -1
        Value to mark "not found"

    Examples
    --------

    Returns
    -------
    match : ndarray of integers
    """
    values = com._asarray_tuplesafe(values)
    if issubclass(values.dtype.type, basestring):
        values = np.array(values, dtype='O')

    f = lambda htype, caster: _match_generic(to_match, values, htype, caster)
    return _hashtable_algo(f, values.dtype)

def unique(values):
    """
    Compute unique values (not necessarily sorted) efficiently from input array
    of values

    Parameters
    ----------
    values : array-like

    Returns
    -------
    uniques
    """
    values = com._asarray_tuplesafe(values)
    f = lambda htype, caster: _unique_generic(values, htype, caster)
    return _hashtable_algo(f, values.dtype)

# def count(values, uniques=None):
#     f = lambda htype, caster: _count_generic(values, htype, caster)

#     if uniques is not None:
#         raise NotImplementedError
#     else:
#         return _hashtable_algo(f, values.dtype)

def _hashtable_algo(f, dtype):
    """
    f(HashTable, type_caster) -> result
    """
    if com.is_float_dtype(dtype):
        return f(lib.Float64HashTable, com._ensure_float64)
    elif com.is_integer_dtype(dtype):
        return f(lib.Int64HashTable, com._ensure_int64)
    else:
        return f(lib.PyObjectHashTable, com._ensure_object)

def _count_generic(values, table_type, type_caster):
    from pandas.core.series import Series

    values = type_caster(values)
    table = table_type(min(len(values), 1000000))
    uniques, labels = table.factorize(values)

    return Series(counts, index=uniques)

def _match_generic(values, index, table_type, type_caster):
    values = type_caster(values)
    index = type_caster(index)
    table = table_type(min(len(index), 1000000))
    table.map_locations(index)
    return table.lookup(values)

def _unique_generic(values, table_type, type_caster):
    values = type_caster(values)
    table = table_type(min(len(values), 1000000))
    uniques = table.unique(values)
    return type_caster(uniques)

def factorize(values, sort=False, order=None, na_sentinel=-1):
    """
    Encode input values as an enumerated type or categorical variable

    Parameters
    ----------
    values : sequence
    sort :
    order :

    Returns
    -------
    """
    values = np.asarray(values)
    is_datetime = com.is_datetime64_dtype(values)
    (hash_klass, vec_klass), values = _get_data_algo(values, _hashtables)

    table = hash_klass(len(values))
    uniques = vec_klass()
    labels = table.get_labels(values, uniques, 0, na_sentinel)

    labels = com._ensure_platform_int(labels)

    uniques = uniques.to_array()

    if sort and len(uniques) > 0:
        sorter = uniques.argsort()
        reverse_indexer = np.empty(len(sorter), dtype=np.int_)
        reverse_indexer.put(sorter, np.arange(len(sorter)))

        mask = labels < 0
        labels = reverse_indexer.take(labels)
        np.putmask(labels, mask, -1)

        uniques = uniques.take(sorter)

    if is_datetime:
        uniques = uniques.view('M8[ns]')

    return labels, uniques

def value_counts(values, sort=True, ascending=False):
    """
    Compute a histogram of the counts of non-null values

    Parameters
    ----------
    values : ndarray (1-d)
    sort : boolean, default True
        Sort by values
    ascending : boolean, default False
        Sort in ascending order

    Returns
    -------
    value_counts : Series
    """
    from pandas.core.series import Series
    from collections import defaultdict

    values = np.asarray(values)

    if com.is_integer_dtype(values.dtype):
        values = com._ensure_int64(values)
        keys, counts = lib.value_count_int64(values)
        result = Series(counts, index=keys)
    else:
        counter = defaultdict(lambda: 0)
        values = values[com.notnull(values)]
        for value in values:
            counter[value] += 1
        result = Series(counter)

    if sort:
        result.sort()
        if not ascending:
            result = result[::-1]

    return result

def rank(values, axis=0, method='average', na_option='keep',
         ascending=True):
    """

    """
    if values.ndim == 1:
        f, values = _get_data_algo(values, _rank1d_functions)
        ranks = f(values, ties_method=method, ascending=ascending,
                  na_option=na_option)
    elif values.ndim == 2:
        f, values = _get_data_algo(values, _rank2d_functions)
        ranks = f(values, axis=axis, ties_method=method,
                  ascending=ascending, na_option=na_option)
    return ranks

def quantile(x, q, interpolation_method='fraction'):
    """
    Compute sample quantile or quantiles of the input array. For example, q=0.5
    computes the median.

    The `interpolation_method` parameter supports three values, namely
    `fraction` (default), `lower` and `higher`. Interpolation is done only,
    if the desired quantile lies between two data points `i` and `j`. For
    `fraction`, the result is an interpolated value between `i` and `j`;
    for `lower`, the result is `i`, for `higher` the result is `j`.

    Parameters
    ----------
    a : ndarray
        Values from which to extract score.
    q : scalar or array
        Percentile at which to extract score.
    interpolation : {'fraction', 'lower', 'higher'}, optional
        This optional parameter specifies the interpolation method to use,
        when the desired quantile lies between two data points `i` and `j`:

        - fraction: `i + (j - i)*fraction`, where `fraction` is the
                    fractional part of the index surrounded by `i` and `j`.
        -lower: `i`.
        - higher: `j`.

    Returns
    -------
    score : float
        Score at percentile.

    Examples
    --------
    >>> from scipy import stats
    >>> a = np.arange(100)
    >>> stats.scoreatpercentile(a, 50)
    49.5

    """
    x = np.asarray(x)
    mask = com.isnull(x)

    x = x[-mask]

    values = np.sort(x)

    def _get_score(at):
        if len(values) == 0:
            return np.nan

        idx = at * (len(values) - 1)
        if (idx % 1 == 0):
            score = values[idx]
        else:
            if interpolation_method == 'fraction':
                score = _interpolate(values[int(idx)], values[int(idx) + 1],
                                     idx % 1)
            elif interpolation_method == 'lower':
                score = values[np.floor(idx)]
            elif interpolation_method == 'higher':
                score = values[np.ceil(idx)]
            else:
                raise ValueError("interpolation_method can only be 'fraction' "
                                 ", 'lower' or 'higher'")

        return score

    if np.isscalar(q):
        return _get_score(q)
    else:
        q = np.asarray(q, np.float64)
        return _algos.arrmap_float64(q, _get_score)

def _interpolate(a, b, fraction):
    """Returns the point at the given fraction between a and b, where
    'fraction' must be between 0 and 1.
    """
    return a + (b - a) * fraction

def _get_data_algo(values, func_map):
    if com.is_float_dtype(values):
        f = func_map['float64']
        values = com._ensure_float64(values)
    elif com.is_datetime64_dtype(values):
        f = func_map['int64']
        values = values.view('i8')
    elif com.is_integer_dtype(values):
        f = func_map['int64']
        values = com._ensure_int64(values)
    else:
        f = func_map['generic']
        values = com._ensure_object(values)
    return f, values

def group_position(*args):
    """
    Get group position
    """
    from collections import defaultdict
    table = defaultdict(int)

    result = []
    for tup in zip(*args):
        result.append(table[tup])
        table[tup] += 1

    return result

_rank1d_functions = {
    'float64': lib.rank_1d_float64,
    'int64': lib.rank_1d_int64,
    'generic': lib.rank_1d_generic
}

_rank2d_functions = {
    'float64': lib.rank_2d_float64,
    'int64': lib.rank_2d_int64,
    'generic': lib.rank_2d_generic
}

_hashtables = {
    'float64': (lib.Float64HashTable, lib.Float64Vector),
    'int64': (lib.Int64HashTable, lib.Int64Vector),
    'generic': (lib.PyObjectHashTable, lib.ObjectVector)
}

max_stars_repo_name: flexlee/pandas

max_issues_repo_name: flexlee/pandas

max_forks_repo_name: flexlee/pandas


text:
import base64
from PIL import Image
import io
import cv2
from keras.models import load_model
import numpy as np
import matplotlib.pyplot as plt
import cv2
import pandas as pd
from PIL import Image
import imagehash
import base64
import face_recognition
img=[]
pho=[]
from difflib import SequenceMatcher

def similar(a, b):
    return SequenceMatcher(None, a, b).ratio()

def solve(img1):
    image = base64.b64decode(str(img1))
    imagePath = ("test.jpeg")
    img3 = Image.open(io.BytesIO(image))
    img3.save(imagePath, 'jpeg')
    co=0

    picture_of_me = face_recognition.load_image_file("test.jpeg")
    for i in img:
        image = base64.b64decode(str(i))
        imagePath = ("test1.jpeg")
        img3 = Image.open(io.BytesIO(image))
        img3.save(imagePath, 'jpeg')
        my_face_encoding = face_recognition.face_encodings(picture_of_me)[0]

        # my_face_encoding now contains a universal 'encoding' of my facial features that can be compared to any other picture of a face!

        unknown_picture = face_recognition.load_image_file("test1.jpeg")
        unknown_face_encoding = face_recognition.face_encodings(unknown_picture)[0]

        # Now we can see the two face encodings are of the same person with `compare_faces`!

        results = face_recognition.compare_faces([my_face_encoding], unknown_face_encoding)

        if results[0] == True:
            co=co+1
    return co

from flask import Flask, request, jsonify
app = Flask(__name__)
@app.route("/attendance")
def attend():
    img1=str(request.args['image'])
    co=solve(img1)
    if co>0:
        return jsonify({'success':'attended'})
    return jsonify({'success':'not attended'})

@app.route("/register")
def reg():
    img1=str(request.args['image'])
    phone=str(request.args['phone'])
    if not(phone in pho):
        img.append(img1)
        pho.append(phone)
        return jsonify({'success':'added'})

    return jsonify({'success':'rejected'})

if __name__ == "__main__":
    app.run(host='0.0.0.0',port=5000)


text:
import os
import re

import numpy as np
import pandas as pd

FILE_ROOT_PATH = os.path.dirname(os.path.abspath(__file__))
source = os.path.join(FILE_ROOT_PATH, '9889.xlsx')
output = os.path.join(FILE_ROOT_PATH, 'result_9889.xlsx')

def merge_sheets(path: str) -> pd.DataFrame:
    '''将每个表格的sheet页中日期、期货成交汇总、期货持仓汇总聚集到一起
    '''
    df = pd.read_excel(path, sheet_name=None)
    print(len(list(df.keys())))
    all_indexs = dict()
    all_sheet_df_res = pd.DataFrame()
    for sheet_index, sheet in enumerate(list(df.keys())):
        if sheet_index < len(list(df.keys())) + 1:     # 测试时候控制前几个
            print(sheet_index)
            df_sheet = pd.read_excel(path, sheet_name=sheet)
            row, col = df_sheet.shape
            child_indexs = []
            child_table1_flag = 0
            child_table2_flag = 0
            for r in range(row):
                each_row_list = list(df_sheet.loc[r])
                if "交易日期" in each_row_list:
                    key_date = str(each_row_list[7])
                    date_row = r
                    # print(key_date)
                elif "期货持仓汇总" in each_row_list:
                    # print(each_row_list)
                    # print("从此行开始截取数据", r)
                    first_start = r
                    child_indexs.append(first_start)
                    child_table1_flag = 1
                elif "合计" in each_row_list and child_table1_flag == 1:
                    # print(each_row_list)
                    # print("第一段结束", r)
                    first_end = r
                    child_indexs.append(first_end)
                    # 跳出前总结数据
                    if key_date:
                        all_indexs[key_date] = child_indexs
                elif "期权持仓汇总" in each_row_list:
                    # print(each_row_list)
                    # print("从此行开始截取数据", r)
                    second_start = r
                    child_indexs.append(second_start)
                    child_table2_flag = 1
                elif "合计" in each_row_list and child_table2_flag == 1:
                    # print(each_row_list)
                    # print("第二段结束", r)
                    second_end = r
                    if second_end not in child_indexs:
                        child_indexs.append(second_end)
                        # 跳出前总结数据
                        if key_date:
                            all_indexs[key_date] = child_indexs
            each_sheet_res = []     # 每个sheet页的结果
            # print(child_indexs)
            if len(child_indexs) == 2:
                df_sheet.loc[date_row][0] = df_sheet.loc[date_row][7]
                df_sheet.loc[date_row][1:] = np.nan
                each_sheet_res.append(df_sheet.loc[date_row])           # 日期行
                # print(df_sheet.loc[date_row][0])
                for i in range(child_indexs[0], child_indexs[1] + 1):
                    # print(i)
                    # print(df_sheet.loc[i])
                    each_sheet_res.append(df_sheet.loc[i])
            elif len(child_indexs) == 4:
                df_sheet.loc[date_row][0] = df_sheet.loc[date_row][7]
                df_sheet.loc[date_row][1:] = np.nan
                each_sheet_res.append(df_sheet.loc[date_row])           # 日期行
                # print(df_sheet.loc[date_row])
                for i in range(child_indexs[0], child_indexs[1] + 1):
                    # print(i)
                    # print(df_sheet.loc[i])
                    each_sheet_res.append(df_sheet.loc[i])
                for j in range(child_indexs[2], child_indexs[3] + 1):
                    # print(j)
                    # print(df_sheet.loc[j])
                    each_sheet_res.append(df_sheet.loc[j])
            # print(each_sheet_res)
            each_sheet_res_df = pd.DataFrame(each_sheet_res).reset_index(drop=True)
            all_sheet_df_res = pd.concat([all_sheet_df_res, each_sheet_res_df], axis=0)
        # break
    return all_sheet_df_res

if __name__ == "__main__":
    res = merge_sheets(source)
    res.to_excel(output, header=None, index=False)


text:
#python may27.py --prototxt mobilenet_ssd/MobileNetSSD_deploy.prototxt \
#	--model mobilenet_ssd/MobileNetSSD_deploy.caffemodel --input videos/dashcam_boston.mp4 \
#	--output output/output_01.avi

#works, does only detection on the first frame. was updated on apr30. added q to quit

from imutils.video import VideoStream
from imutils.video import FPS
from collections import OrderedDict
import argparse
import imutils
import time
import cv2
import numpy as np
#import pysnooper

#def supertracker(boxrange)

#@pysnooper.snoop()
# construct the argument parse and parse the arguments
ap = argparse.ArgumentParser()
ap.add_argument("-p", "--prototxt", required=True,
	help="path to Caffe 'deploy' prototxt file")
ap.add_argument("-m", "--model", required=True,
	help="path to Caffe pre-trained model")
ap.add_argument("-i", "--input", type=str,
	help="path to optional input video file")
ap.add_argument("-o", "--output", type=str,
	help="path to optional output video file")
ap.add_argument("-c", "--confidence", type=float, default=0.4,
	help="minimum probability to filter weak detections")
ap.add_argument("-s", "--skip-frames", type=int, default=40,
	help="# of skip frames between detections")
ap.add_argument("-t", "--tracker", type=str, default="kcf",
	help="OpenCV object tracker type")
ap.add_argument("-v", "--threshold-val", type=str, default= "10",
	help="Threshold value for bounding boxes")
args = vars(ap.parse_args())

# extract the OpenCV version info
(major, minor) = cv2.__version__.split(".")[:2]

# if we are using OpenCV 3.2 OR BEFORE, we can use a special factory
# function to create our object tracker
if int(major) == 3 and int(minor) < 3:
	tracker = cv2.Tracker_create(args["tracker"].upper())

else:
	# initialize a dictionary that maps strings to their corresponding
	# OpenCV object tracker implementations
	OPENCV_OBJECT_TRACKERS = {
		"csrt": cv2.TrackerCSRT_create,
		"kcf": cv2.TrackerKCF_create,
		"boosting": cv2.TrackerBoosting_create,
		"mil": cv2.TrackerMIL_create,
		"tld": cv2.TrackerTLD_create,
		"medianflow": cv2.TrackerMedianFlow_create,
		"mosse": cv2.TrackerMOSSE_create
	}

	# grab the appropriate object tracker using our dictionary of
	# OpenCV object tracker objects
	tracker = OPENCV_OBJECT_TRACKERS[args["tracker"]]()
	multiTracker = cv2.MultiTracker_create()

# initialize the list of class labels MobileNet SSD was trained to
# detect
CLASSES = ["background", "aeroplane", "bicycle", "bird", "boat",
	"bottle", "bus", "car", "cat", "chair", "cow", "diningtable",
	"dog", "horse", "motorbike", "person", "pottedplant", "sheep",
	"sofa", "train", "tvmonitor"]

# load our serialized model from disk
print("[INFO] loading model...")
net = cv2.dnn.readNetFromCaffe(args["prototxt"], args["model"])

# if a video path was not supplied, grab a reference to the webcam
if not args.get("input", False):
	print("[INFO] starting video stream...")
	vs = VideoStream(src=0).start()
	time.sleep(2.0)
	inputval = 0

# otherwise, grab a reference to the video file
else:
	print("[INFO] opening video file...")
	vs = cv2.VideoCapture(args["input"])
	inputval = 1

# initialize the video writer (we'll instantiate later if need be)
writer = None

# initialize the frame dimensions (we'll set them as soon as we read
# the first frame from the video)
W = None
H = None
initBB = None
bboxes = []

#initialize the fps collector for statistical purposes
a = time.strftime("%d_%b_%H_%M")
name = a+"_"+args["tracker"]+".txt"
fpsfile = open(name,"a")

########perform first detection########
#initialize total frames, to see how many frames we need in between detection and tracking.
totalFrames = 0
found = 0
skipframes = 30
currentsizeoftracker = 0
sizes = 0
newbboxes = []
thresholdval = 10
fps = FPS().start()
#loops until initial detection is found.
while found == 0:
	frame = vs.read()
	frame = frame[1] if args.get("input", False) else frame

	# check to see if we have reached the end of the stream
	#if frame is None:
	#	break

	# resize the frame (so we can process it faster) and grab the
	# frame dimensions
	frame = imutils.resize(frame, width=500)
	(H, W) = frame.shape[:2]

	if args["output"] is not None and writer is None:
		fourcc = cv2.VideoWriter_fourcc(*"MJPG")
		writer = cv2.VideoWriter(args["output"], fourcc, 30, (W, H), True)

	blob = cv2.dnn.blobFromImage(frame, 0.007843, (W, H), 127.5)
	net.setInput(blob)
	detections = net.forward()

	for i in np.arange(0, detections.shape[2]):
		# extract the confidence (i.e., probability) associated
		# with the prediction
		confidence = detections[0, 0, i, 2]

		# filter out weak detections by requiring a minimum
		# confidence
		if confidence > args["confidence"]:
			# extract the index of the class label from the
			# detections list
			idx = int(detections[0, 0, i, 1])

			#if CLASSES[idx]
			# if the class label is not a car, ignore it
			if CLASSES[idx] == "car":

				box = detections[0, 0, i, 3:7] * np.array([W, H, W, H])
				(startX, startY, endX, endY) = box.astype("int")
				#print(startX, startY, endX, endY)
				width = endX-startX
				height = endY-startY
				initBB = (startX,startY,width,height)
				#print(initBB)
				bboxes.append(initBB)
				found = 1
				totalFrames +=1

				#tracker.init(frame, initBB)
				#(x, y, w, h) = [int(v) for v in box]
				#cv2.rectangle(frame, (x, y), (x + w, y + h),(0, 255, 0), 2)
				#fps = FPS().start()
				#totalFrames+=1
				#found = 1

for bbox in bboxes:
	multiTracker.add(cv2.TrackerCSRT_create(),frame,bbox)
	retval	=multiTracker.getObjects()
	#print(type(bbox))
	#print(retval)
	#(x, y, w, h) = [int(v) for v in bbox]
	#cv2.rectangle(frame, (x, y), (x + w, y + h),(0, 255, 0), 2)

cv2.imshow("MultiTracker", frame)
fps.update()
anobox = []
dellist = []
while True:

	frame = vs.read()
	frame = frame[1] if args.get("input", False) else frame

	# check to see if we have reached the end of the stream
	if frame is None:
		break

	# resize the frame (so we can process it faster) and grab the
	# frame dimensions
	frame = imutils.resize(frame, width=500)
	(H, W) = frame.shape[:2]

	if totalFrames %skipframes ==0:

		newbboxes = []
		#print("re-detection after 30 secs") ##debuging purposes
		blober = cv2.dnn.blobFromImage(frame, 0.007843, (W, H), 127.5)
		net.setInput(blober)
		detections = net.forward()

		for i in np.arange(0, detections.shape[2]):
			# extract the confidence (i.e., probability) associated
			# with the prediction
			confidence = detections[0, 0, i, 2]

			# filter out weak detections by requiring a minimum
			# confidence
			if confidence > args["confidence"]:
				# extract the index of the class label from the
				# detections list
				idx = int(detections[0, 0, i, 1])

				#if CLASSES[idx]
				# if the class label is not a car, ignore it
				if CLASSES[idx] == "car":
					sizes+=1
					box = detections[0, 0, i, 3:7] * np.array([W, H, W, H])
					(startX, startY, endX, endY) = box.astype("int")
					#print(startX, startY, endX, endY)
					width = endX-startX
					height = endY-startY
					initBB = (startX,startY,width,height)
					#print(sizes)
					newbboxes.append(initBB)

		for lilbox in retval:
			(x1,y1,w1,h1) = [int(v) for v in bbox]
			for newbox in newbboxes:
				(x2,y2,w2,h2) = [int(v) for v in newbox]
				if ((abs(x1-x2) < thresholdval)):
					a = newbboxes.index(newbox)
					dellist.append(a)
				elif((abs(y1-y2) < thresholdval)):
					a = newbboxes.index(newbox)
					dellist.append(a)

		#print(dellist)
		#print(newbboxes)
		dellist =list(OrderedDict.fromkeys(dellist))
		a = dellist[::-1]
		#print(a)
		try:
			if len(newbboxes) < len(dellist):
				print(newbboxes)
				newbboxes =[]

			for index_i in a:
				if index_i <=len(newbboxes):
					del newbboxes[index_i]
				else:
					continue
		except IndexError:
			pass

		#2 #####################################
		if len(newbboxes)>0:
			for nbbox in newbboxes:
				multiTracker.add(cv2.TrackerCSRT_create(),frame,nbbox)

		#bboxes = []

	if args["output"] is not None and writer is None:
		fourcc = cv2.VideoWriter_fourcc(*"MJPG")
		writer = cv2.VideoWriter(args["output"], fourcc, 30, (W, H), True)

	(success, box1) = multiTracker.update(frame)
	retval	=multiTracker.getObjects()
	#print(type(retval))
	#print(retval)
	currentsizeoftracker = len(retval)
	#print(len(retval)) #for debuging purposes

	if success:
		for bbox in box1:
			#print(bbox)
			(x, y, w, h) = [int(v) for v in bbox]
			if x >0 and y >0:
				cv2.rectangle(frame, (x, y), (x + w, y + h),(0, 255, 0), 2)

	info = [
			("Tracker", args["tracker"]),
			("Success", "Yes" if success else "No"),
			#("FPS", "{:.2f}".format(fps.fps())),
	]

	# loop over the info tuples and draw them on our frame
	for (i, (k, v)) in enumerate(info):
		text = "{}: {}".format(k, v)
		cv2.putText(frame, text, (10, H - ((i * 20) + 20)),
			cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 0, 255), 2)

	# show the output frame
	cv2.imshow("MultiTracker", frame)
	key = cv2.waitKey(1) & 0xFF
	fps.update()
	totalFrames+=1

	#press q to quit
	if key == ord("q"):
		break

	##increasing confidence## with extra detection
	#30 is the number of skip-frames
	########################################

fps.stop() ##added line
print("[INFO] elasped time: {:.4f}".format(fps.elapsed())) ##added line
print("[INFO] approx. FPS: {:.4f}".format(fps.fps()))##Added line
fpsfile.write("Marijani Hussein \n \n")
fpsfile.write("tracker type: " +args["tracker"]+"\n")
fpsfile.write("time elapsed{:.4f} \n".format(fps.elapsed()))
fpsfile.write("FPS {:.4f} \n".format(fps.fps()))

# if we are using a webcam, release the pointer
if inputval == 0:
	vs.stop
#if not args.get("video", False):
#	vs.stop()

# otherwise, release the file pointer
else:
	vs.release()

# close all windows
cv2.destroyAllWindows()


text:
import tensorflow as tf
from tensorflow.keras import layers, optimizers, datasets, Sequential, regularizers
from resnet import ResNet18
import numpy as np
import random

import os
import sys

sys.path.append("../src")
from dopamine import Dopamine, dopamine

assert tf.__version__.startswith('2.')

tf.random.set_seed(22)
np.random.seed(22)
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

batchsz = 250
epochs = 180
validation_freq = 1
input_shape = (batchsz, 32, 32, 3)
model_path = 'data/'
initial_epoch = 116
dopamine_batch_size = batchsz

# 1. 归一化函数实现；cifar100 均值和方差，自己计算的。
img_mean = tf.constant([0.50736203482434500, 0.4866895632914611, 0.4410885713465068])
img_std = tf.constant([0.26748815488001604, 0.2565930997269337, 0.2763085095510783])

def normalize(x, mean=img_mean, std=img_std):
    x = (x - mean) / std
    return x

# 2. 数据预处理，仅仅是类型的转换。    [-1~1]
def preprocess(x, y):
    x = tf.pad(x, [[4, 4], [4, 4], [0, 0]])  # 上下填充4个0，左右填充4个0，变为[40, 40, 3]
    x = tf.image.random_crop(x, [32, 32, 3])
    x = tf.image.random_flip_left_right(x)
    # x: [0,255]=> -1~1   其次：normalizaion
    x = tf.cast(x, dtype=tf.float32) / 255.
    # 0~1 => D(0,1) 调用函数；
    x = normalize(x)
    y = tf.cast(y, dtype=tf.int32)
    return x, y

# 3. 学习率调整测率200epoch
def lr_schedule_300ep(epoch):
    if epoch < 60:
        return 0.1
    if epoch < 130:
        return 0.02
    if epoch < 160:
        return 0.004
    if epoch < 200:
        return 0.0008
    if epoch < 250:
        return 0.0003
    if epoch < 300:
        return 0.0001
    else:
        return 0.00006

def load_data():
    # 数据集的加载
    (x, y), (x_test, y_test) = datasets.cifar100.load_data()
    y = tf.squeeze(y)  # 或者tf.squeeze(y, axis=1)把1维度的squeeze掉。
    y_test = tf.squeeze(y_test)  # 或者tf.squeeze(y, axis=1)把1维度的squeeze掉。
    print(x.shape, y.shape, x_test.shape, y_test.shape)

    # 训练集和标签包装成Dataset对象
    train_db = tf.data.Dataset.from_tensor_slices((x, y))
    # 测试集和标签包装成Dataset对象
    test_db = tf.data.Dataset.from_tensor_slices((x_test, y_test))

    # train_db = train_db.take(1000).shuffle(50000).map(preprocess).batch(batchsz)
    # test_db = test_db.take(1000).map(preprocess).batch(batchsz)

    train_db = train_db.shuffle(50000).map(preprocess).batch(batchsz)
    test_db = test_db.map(preprocess).batch(batchsz)

    # 我们来取一个样本，测试一下sample的形状。
    sample = next(iter(train_db))
    print('sample:', sample[0].shape, sample[1].shape,
          tf.reduce_min(sample[0]),
          tf.reduce_max(sample[0]))  # 值范围为[0,1]

    return train_db, test_db

def create_params():
    optimizer = tf.keras.optimizers.SGD(lr=0.02, momentum=0.9, decay=5e-4)
    loss = tf.losses.SparseCategoricalCrossentropy(from_logits=True)
    acc = tf.keras.metrics.SparseCategoricalAccuracy()
    return optimizer, loss, acc

def create_models():
    lys = [
        layers.GlobalAveragePooling2D(),
        layers.Lambda(lambda x: x),
        layers.Dense(100, activation=None, kernel_regularizer=regularizers.l2(5e-4)),
    ]
    return ResNet18(), Sequential(lys)

def save_models(resnet_model, class_model, epoch, acc):
    acc = int(acc*10000)
    save_model(resnet_model, '%s%05d_%03d_my_model1.h5' % (model_path, acc, epoch))
    save_model(class_model, '%s%05d_%03d_my_model2.h5' % (model_path, acc, epoch))

def load_models(resnet_model, class_model):
    load_model(resnet_model, 'data/07551_my_model1.h5')
    # load_model(class_model, 'data/07551_my_model2.h5')

def fill_models(class_model):
    # vv = Dopamine(input_shape=[250, 512], batch_size=dopamine_batch_size, use_bias=True)
    vv = layers.Dropout(0.5)
    la = class_model.get_layer(index=1)
    class_model.pop()
    class_model.add(vv)
    class_model.add(la)

def load_model(model, file):
    if os.path.isfile(file):
        model.load_weights(file)

def save_model(model, file):
    model.save_weights(file)

def create_callbacks(resnet_model, class_model):
    # def on_epoch_begin(epoch, logs):
    #     print('----on_epoch_begin', epoch, logs)

    def on_epoch_end(epoch, logs):
        key = 'val_sparse_categorical_accuracy'
        acc = logs[key] if key in logs else 0.0
        save_models(resnet_model, class_model, epoch, acc)
        # print('----on_epoch_end', epoch, logs)

    # def on_batch_begin(batch, logs):
    #     print('----on_batch_begin', batch, logs)
    #
    # def on_batch_end(batch, logs):
    #     acc = logs['sparse_categorical_accuracy']
    #
    #     print('----on_batch_end%.5f' % acc, batch, logs)
    #
    # def on_train_begin(logs):
    #     print('----on_train_begin', logs)
    #
    # def on_train_end(logs):
    #     print('----on_train_end', logs)

    task_callback = tf.keras.callbacks.LambdaCallback(
        # on_epoch_begin=on_epoch_begin,
        on_epoch_end=on_epoch_end,
        # on_batch_begin=on_batch_begin,
        # on_batch_end=on_batch_end,
        # on_train_begin=on_train_begin,
        # on_train_end=on_train_end,
    )

    lrs_cb = tf.keras.callbacks.LearningRateScheduler(lr_schedule_300ep)
    csv_logger = tf.keras.callbacks.CSVLogger('data/training.log')
    tb_logger = tf.keras.callbacks.TensorBoard(
        log_dir='logs',
        histogram_freq=0,
        write_graph=False,
        write_images=False,
        update_freq='batch',  # epoch or batch
        profile_batch=2,
        embeddings_freq=0, )

    # callbacks = [lrs_cb, csv_logger, tb_logger]
    callbacks = [lrs_cb, tb_logger, csv_logger, task_callback]
    return callbacks

def main():
    train_db, test_db = load_data()

    resnet_model, class_model = create_models()
    model = Sequential([resnet_model, class_model])
    optimizer, loss, acc = create_params()
    model.compile(
        optimizer=optimizer,
        loss=loss,
        metrics=[acc]
    )
    model.build(input_shape=input_shape)
    load_models(resnet_model, class_model)
    # fill_models(class_model)
    # model.build(input_shape=input_shape)
    model.summary()

    callbacks = create_callbacks(resnet_model, class_model)
    model.fit(
        train_db,
        epochs=epochs,
        validation_data=test_db,
        initial_epoch=initial_epoch,
        validation_freq=validation_freq,
        callbacks=callbacks
    )
    model.evaluate(test_db)
    # save_models(resnet_model, class_model)

if __name__ == '__main__':
    main()


text:
#!/usr/bin/env python
#
# Copyright (C) 2019
# Christian Limberg
# Centre of Excellence Cognitive Interaction Technology (CITEC)
# Bielefeld University
#
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions
# and the following disclaimer in the documentation and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote
# products derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

import numpy as np

def embedding_tsne(x, y=None):
    from sklearn.manifold import TSNE
    x_embedding = TSNE(n_components=2, random_state=42).fit_transform(x)
    x_embedding_normalized = normalize_features(x_embedding)
    return x_embedding_normalized

def embedding_umap(x, y=None):
    import umap
    from sklearn.preprocessing import LabelEncoder

    params = {'n_neighbors': 100, 'random_state': 42}

    if not y is None:
        none_idx = y == None
        y[none_idx] = '__unlabeled__' # older sklearn versions don't accept None
        int_labels = LabelEncoder().fit_transform(y)
        int_labels[none_idx] = -1 # umap is representing unlabeled data as -1
        x_embedding = umap.UMAP(**params).fit_transform(x,y=int_labels)
    else:
        x_embedding = umap.UMAP(**params).fit_transform(x)
    x_embedding_normalized = normalize_features(x_embedding)
    return x_embedding_normalized

def a2vq_querying(x_embedding_normalized, mask_unlabeled, probas, view_size, overlap):
    ''' a2vq querying as proposed in paper '''
    best_view = (-1,-1)
    least_confidence = -999999999999

    view_range = np.arange(0, 1 - view_size + overlap, overlap)
    print('VIEW_RANGES: ', view_range)
    num_samples = np.zeros((len(view_range),len(view_range)))
    mean_confidence = np.zeros((len(view_range),len(view_range)))

    for i,x in enumerate(view_range):
        for j, y in enumerate(view_range):
            mask = np.logical_and(x_embedding_normalized[:, 0] > x, x_embedding_normalized[:, 0] < x+view_size)
            mask = np.logical_and(mask, x_embedding_normalized[:, 1] > y)
            mask = np.logical_and(mask, x_embedding_normalized[:, 1] < y+view_size)
            fused_mask = np.logical_and(mask,mask_unlabeled)
            probas_temp = probas[fused_mask]

            mean_confidence[i,j] = np.mean([(1-cpu) for cpu in probas_temp])
            num_samples[i,j] = len(probas_temp)

    num_samples_ratio = num_samples / np.max(num_samples)

    mean_confidence = np.nan_to_num(mean_confidence)

    cost_map = mean_confidence * num_samples_ratio

    max_inds = cost_map.flatten().argsort()[::-1]

    max_views = np.array([np.unravel_index(ind, cost_map.shape) for ind in max_inds])
    max_ranges = [(view_range[x],view_range[y]) for x,y in max_views]
    max_costs = cost_map.flatten()[max_inds]

    return max_ranges, max_costs

def filter_embedding(x_embedding, anchor_point, selection_size):
    ''' get boolean mask of samples within a specified selection bb rect (view)'''
    mask = np.logical_and(x_embedding[:,0] > anchor_point[0], x_embedding[:,0] < anchor_point[0]+selection_size[0])
    mask = np.logical_and(mask, x_embedding[:,1] > anchor_point[1])
    mask = np.logical_and(mask, x_embedding[:, 1] < anchor_point[1]+selection_size[1])
    return mask

def normalize_features(features, scale = (0,1)):
    normalized = np.zeros(features.shape)
    for i in range(features.shape[1]):
        min, max = np.min(features[:, i]), np.max(features[:, i])
        normalized[:, i] = (features[:, i] - min) / (max - min) * (scale[1]-scale[0]) + scale[0]
    return normalized

if __name__ == '__main__':
    pass

max_stars_repo_licenses: MIT Unlicense

max_issues_repo_licenses: MIT Unlicense

max_forks_repo_licenses: MIT Unlicense


text:
#-------------------------------------------------------------------------------
# Name:        module_filter
# Purpose:
#
# Author:      s6anloew
#
# Created:     03.09.2014
# Copyright:   (c) s6anloew 2014
# Licence:     <your licence>
#-------------------------------------------------------------------------------

#import matplotlib.pyplot as plt
from scipy import ndimage
import scipy
import os
import numpy as np
import gdal

from osgeo import gdal_array

class Filter_ndem():
    def __init__(self, inFile_knowlegde, inFile_nDEM, outFolder,filter_lst, no_data = -9999):
        self.infile_knwlgd = inFile_knowlegde
        self.infile_ndem = inFile_nDEM
        self.outfile_name = os.path.split(self.infile_knwlgd)[1]
        self.outfile  = outFolder
        self.filterlst = filter_lst
        #self.outfile = os.path.join(self.outfolder, "filterd_" + self.outfile_name )
        self.nodata = no_data

    def filter_application(self):
        ### the filter lst beinhaltet die namen, der funktionen, so dass auf die variable immer die neue
        ### funktion angewendet wird bsp. lst = [("value", (3,3))] -> lst[0][0] _>  lst[0][1] _>
        ### lst[0][1][0] -> lst[0][1][1]
        In_File_filler = gdal.Open(self.infile_ndem)
        In_file = gdal.Open(self.infile_knwlgd)
        driver = In_file.GetDriver()

        band_filler = In_File_filler.GetRasterBand(1)
        band_buildup  = In_file.GetRasterBand(1)
        rows = In_file.RasterYSize
        cols = In_file.RasterXSize

        self.outDS = driver.Create(self.outfile, cols, rows,1, gdal.GDT_Float32)
        self.outDS.SetGeoTransform(In_file.GetGeoTransform())
        self.outDS.SetProjection(In_file.GetProjection())

        #numpyBandType = gdal_array.GDALTypeCodeToNumericTypeCode(gdal.GDT_CFloat32)

        blockSize = 5000
        band_filtert = self.outDS.GetRasterBand(1)

        for i in range(0, rows, blockSize):
            if i + blockSize < rows:
                numRows = blockSize
            else:
                numRows = rows - i

            for j in range(0, cols, blockSize):
                if j + blockSize < cols:
                    numCols = blockSize
                else:
                    numCols = cols - j
                fil_start = band_filler.ReadAsArray(j,i,numCols,numRows)
                filter_ndem = band_buildup.ReadAsArray(j,i,numCols,numRows)
                start_mask = filter_ndem > 0
                mask = None
                for ii in range(len(self.filterlst)):

                    mask_func = self.filterlst[ii][0]
                    if mask == None:
                        mask = mask_func(start_mask, structure=np.ones((self.filterlst[ii][1][0],self.filterlst[ii][1][1])))
                    else:
                        mask = mask_func(mask, structure=np.ones((self.filterlst[ii][1][0],self.filterlst[ii][1][1])))
                buildup_filtert = np.choose(mask,(self.nodata,fil_start))
                mask = None
                band_filtert.WriteArray(buildup_filtert, j,i)
                band_filtert.SetNoDataValue(self.nodata)
                self.message = "The filtering worked as planed"

        self.outDS = None
        return [self.outfile_name, self.message]

dic_lst = [(ndimage.binary_fill_holes,[4, 4]),(ndimage.binary_closing, [3, 3]), (ndimage.binary_opening , [5, 6])]

##
##Filter_ndem("H:\\Laser\\Testings\\test\\pallien\\results\\knwldg_pallien_05m_1.tif", "H:\\Laser\\Testings\\test\\pallien\\ndem_05m_pallien.tif",
##"H:\\Laser\\Testings\\test\\pallien\\results\\filtered_pallien_1.tif", dic_lst).filter_application()
##Filter_ndem("H:\\Laser\\Testings\\test\\pallien\\results\\knwldg_pallien_05m_2.tif", "H:\\Laser\\Testings\\test\\pallien\\ndem_05m_pallien.tif",
##"H:\\Laser\\Testings\\test\\pallien\\results\\filtered_pallien_2.tif", dic_lst).filter_application()
Filter_ndem("H:\\Laser\\Testings\\test\\pallien\\results\\knwldg_pallien_05m_3.tif", "H:\\Laser\\Testings\\test\\pallien\\ndem_05m_pallien.tif",
"H:\\Laser\\Testings\\test\\pallien\\results\\filtered_pallien_3.tif", dic_lst).filter_application()

##Filter_ndem("H:\\Laser\\Testings\\test\\trier_west\\results\\knwldg_trier_west_05m_1.tif", "H:\\Laser\\Testings\\test\\trier_west\\ndem_05m_trier_west.tif",
##"H:\\Laser\\Testings\\test\\trier_west\\results\\filtered_trier_west_1.tif", dic_lst).filter_application()
##Filter_ndem("H:\\Laser\\Testings\\test\\trier_west\\results\\knwldg_trier_west_05m_2.tif", "H:\\Laser\\Testings\\test\\trier_west\\ndem_05m_trier_west.tif",
##"H:\\Laser\\Testings\\test\\trier_west\\results\\filtered_trier_west_2.tif", dic_lst).filter_application()
##Filter_ndem("H:\\Laser\\Testings\\test\\trier_west\\results\\knwldg_trier_west_05m_3.tif", "H:\\Laser\\Testings\\test\\trier_west\\ndem_05m_trier_west.tif",
##"H:\\Laser\\Testings\\test\\trier_west\\results\\filtered_trier_west_3.tif", dic_lst).filter_application()

max_stars_repo_name: AndyLoewe/CityEX

max_issues_repo_name: AndyLoewe/CityEX

max_forks_repo_name: AndyLoewe/CityEX

reason: import numpy,import scipy,from scipy