text:
from abc import ABC, abstractmethod
from copy import copy
import os

import tianshou as ts
import numpy as np
import torch

from . import Algorithm
from ..nodes import Node
from ..policies import TianshouPolicy

class OffPolicyAlgorithm(Algorithm):
    """Off-policy algorithm. Supports SAC and DDPG."""

    supported_flat_offpol_algos = {
            "SAC": ts.policy.SACPolicy,
            "DDPG": ts.policy.DDPGPolicy
            }

    class _Transition():
        def __init__(self, subtask_tr, env_info, child_feedback, algo_info):
            self.subtask_tr = subtask_tr
            self.env_info = env_info
            self.child_feedback = child_feedback
            self.algo_info = algo_info

        def __str__(self):
            return "subtask_tr: {}\n".format(self.subtask_tr) + \
            "child_feedback: {}\n".format(self.child_feedback) + \
            "algo_info: {}\n".format(self.algo_info)

    def __init__(self, name, flat_algo_name, model, fully_random_fraction, flat_algo_kwargs=None,
            buffer_size=100000, batch_size=128, learning_starts=0, grad_steps_per_env_step=1.0):
        """
        Args:
            model (Model): Model instance which provides...
            for SAC:
            * actor: Pytorch module that returns ((mu, sigma), hidden_state) in forward method. mu and sigma refer to value in R^n before squashing.
            * two critics
            for DDPG:
            * actor: Pytorch module that returns (action, hidden_state) in forward method. action refers to value in R^n before squashing.
            * one critic

            learning_starts: Learnings starts after this many transitions have been collected.
        """
        super().__init__(name)

        if flat_algo_name not in self.supported_flat_offpol_algos:
            raise ValueError("Flat off policy RL algorithm \"{}\" not supported.".
                    format(flat_algo_name))

        assert model is not None

        self._flat_algo_name = flat_algo_name
        self._flat_algo_class = self.supported_flat_offpol_algos[flat_algo_name]
        self._flat_algorithm = None
        if flat_algo_kwargs is None:
            self._flat_algo_kwargs = {}
        else:
            self._flat_algo_kwargs = copy(flat_algo_kwargs)
        self._model = model
        self._fully_random_fraction = fully_random_fraction
        self._tb_low_level_writer = None
        self._actions_taken = 0
        self._buffer_size = buffer_size
        self._batch_size = batch_size
        self._learning_starts = learning_starts
        self._grad_steps_per_env_step = grad_steps_per_env_step
        self._episode_transitions = []

        # if a target entropy is given and no optimizer for the entropy coefficient is
        # specified use adam and interpret third entry of tuple as learning rate
        if flat_algo_name == "SAC" and "alpha" in self._flat_algo_kwargs:
            if (isinstance(self._flat_algo_kwargs["alpha"], list)
                and not isinstance(self._flat_algo_kwargs["alpha"][0], torch.optim.Optimizer)):
                target_entropy = self._flat_algo_kwargs["alpha"][0]
                log_alpha = torch.tensor([self._flat_algo_kwargs["alpha"][1]], dtype = torch.float, requires_grad = True)
                alpha_optim = torch.optim.Adam([log_alpha], lr = self._flat_algo_kwargs["alpha"][2])
                self._flat_algo_kwargs["alpha"] = (target_entropy, log_alpha, alpha_optim)

        # set action range to [-1., 1.] in tianshou and rescale in graph_rl if needed
        self._flat_algo_kwargs["action_range"] = [-1., 1.]

        self._last_learning_step = None

        self._replay_buffer = ts.data.ReplayBuffer(size = buffer_size)

    def _map_action_to_default_space(self, action):
        res = (action - self._action_offset)/self._action_scale
        res = np.clip(res, -1., 1.)
        return res

    def _add_to_flat_replay_buffer(self, f_trans):
        # map action back to [-1, 1]^n before adding transition to buffer
        action = self._action_space.flatten_value(f_trans.action)
        mapped_action = self._map_action_to_default_space(action)
        self._replay_buffer.add(
                obs = self._observation_space.flatten_value(f_trans.obs),
                act = mapped_action,
                rew = f_trans.reward,
                done = f_trans.done,
                obs_next = self._observation_space.flatten_value(f_trans.new_obs))

    def get_policy(self, name, observation_space, action_space):
        self._observation_space = observation_space
        self._action_space = action_space
        flat_action_space = action_space.get_flat_space()
        self._action_scale = 0.5*(flat_action_space.high - flat_action_space.low)
        self._action_offset = 0.5*(flat_action_space.high + flat_action_space.low)
        flat_obs_dim = observation_space.get_flat_dim()
        flat_action_dim = action_space.get_flat_dim()
        flat_gym_action_space = flat_action_space.get_gym_space()

        self._model.create(flat_obs_dim, flat_action_dim)
        self._learning_rate_critics = self._model._learning_rate_critics
        self._learning_rate_actor = self._model._learning_rate_actor

        # actor and critic modules
        nets = {
                "actor": self._model.actor,
                "actor_optim": self._model.actor_optim,
                f"critic{'1' if self._flat_algo_name == 'SAC' else ''}": self._model.critics[0],
                f"critic{'1' if self._flat_algo_name == 'SAC' else ''}_optim": self._model.critic_optims[0]
                }
        if self._flat_algo_name == "SAC":
            # SAC expects two critics while DDPG wants only one
            nets["critic2"] = self._model.critics[1]
            nets["critic2_optim"] = self._model.critic_optims[1]
        elif self._flat_algo_name == "DDPG":
            # Gaussian noise with n-dim. mean and variance
            class GaussianNoiseND(ts.exploration.random.BaseNoise):
                """Gaussian noise with n-dim. mean and variance."""

                def __init__(self, mu=[0.0], sigma=[1.0]):
                    super().__init__()
                    self._mu = mu
                    assert np.all(0 <= sigma), "Noise std should not be negative."
                    self._sigma = sigma

                def __call__(self, size):
                    # Note: size has to coincide with shape of mu and sigma
                    return np.random.normal(self._mu, self._sigma)

            # DDPG needs a noise object
            # flatten noise sigma object to vector using action space
            sigma = action_space.flatten_value(self._flat_algo_kwargs["noise_sigma"])
            exploration_noise = GaussianNoiseND(mu=np.zeros_like(sigma), sigma=sigma)
            self._flat_algo_kwargs["exploration_noise"] = exploration_noise
            del self._flat_algo_kwargs["noise_sigma"]

        self._flat_algorithm = self._flat_algo_class(**nets, **self._flat_algo_kwargs,
                action_space=flat_gym_action_space)
        self._flat_algorithm.eval()

        policy = TianshouPolicy(
                name = name,
                observation_space = observation_space,
                action_space = action_space,
                ts_policy = self._flat_algorithm,
                hidden_net = self._model.actor,
                fully_random_fraction = self._fully_random_fraction)

        return policy

    @abstractmethod
    def _add_experience_to_flat_algo(self, parent_info, deterministic_episode, node_is_sink, sess_info):
        """Process transitions from ending episode and store them in flat replay buffer.

        At this point hindsight manipulations can be implemented.
        """

        pass

    def add_experience(self, env_obs, env_info, subtask_trans, parent_info,
            child_feedback, algo_info, interruption, sess_info):

        # save transition to episode transitions buffer
        transition = self._Transition(
                subtask_tr = subtask_trans,
                env_info = env_info,
                child_feedback = child_feedback,
                algo_info = algo_info
                )
        self._episode_transitions.append(transition)
        if self._verbose:
            print(transition)
        self._actions_taken += 1

        # tensorboard logging
        if self._tb_writer is not None:
            obs = self._observation_space.flatten_value(subtask_trans.obs)
            if self._log_q_values and len(self._episode_transitions) == 1:
                act = self._action_space.flatten_value(subtask_trans.action)
                with torch.no_grad():
                    q_values = [critic(obs[None], act[None]).numpy() for critic  in self._model.critics]
                    q_value = min(q_values)
                mode = "test" if sess_info.testing else "train"
                self._tb_writer.add_scalar(f"{self.name}/{mode}/q_value", q_value, sess_info.total_step)
            if self._flat_algo_name == "SAC" and not sess_info.testing and parent_info is None:
                with torch.no_grad():
                    net_out, _ = self._model.actor(obs[None])
                    _, sigma = net_out
                    entropy = np.log(sigma.numpy()).sum() + (0.5 + np.log(np.sqrt(2.*np.pi)))*len(sigma)
                self._tb_writer.add_scalar(f"{self.name}/train/entropy", entropy, sess_info.total_step)

        # if the subtask has ended (i.e. control will be returned to active parent
        # node), process episode transitions and add them to replay buffer
        # of flat algorithm
        if subtask_trans.ended or env_info.done:
            if parent_info:
                deterministic_episode = algo_info["is_deterministic"]
            else:
                deterministic_episode = False

            node_is_sink = child_feedback is Node.sink_identification_feedback
            self._add_experience_to_flat_algo(parent_info, deterministic_episode,
                    node_is_sink, sess_info)

    def learn(self, current_step):
        if self._last_learning_step is None:
            self._last_learning_step = current_step - 1

        elapsed_steps = current_step - self._last_learning_step
        n_grad_steps = int(self._grad_steps_per_env_step*elapsed_steps)
        assert(elapsed_steps > 0)
        if len(self._replay_buffer) >= self._batch_size and current_step > self._learning_starts:
            for _ in range(int(n_grad_steps)):
                batch, indices = self._replay_buffer.sample(self._batch_size)
                batch = self._flat_algorithm.process_fn(batch, self._replay_buffer, indices)
                results = self._flat_algorithm.learn(batch)

                # tensorboard logging of losses
                if self._tb_writer is not None:
                    for key, item in results.items():
                        self._tb_writer.add_scalar(f"{self.name}/train/{key}", item, current_step)

        self._last_learning_step = current_step

    def get_parameters(self):
        """Returns dictionary containing all learned parameters."""
        params = {
                "flat_algo_name": self._flat_algo_name,
                "model_params": self._model.get_parameters()
                }
        return params

    def get_state(self):
        """Returns dictionary containing state."""
        state = {
                "flat_algo_name": self._flat_algo_name,
                "model_state": self._model.get_state(),
                "model_params": self._model.get_parameters(),
                "replay_buffer": self._replay_buffer
                }
        return state

    def save_state(self, node_id, dir_path):
        """Saves state and returns dictionary with file paths."""
        replay_buffer_file_name = f"{node_id}_algo_replay_buffer.hdf5"
        self._replay_buffer.save_hdf5(os.path.join(dir_path, replay_buffer_file_name))
        state = {
                "flat_algo_name": self._flat_algo_name,
                "model_state": self._model.get_state(),
                "model_params": self._model.get_parameters(),
                "replay_buffer": replay_buffer_file_name
                }
        return state

    def _override_target_networks(self):
        """Override target networks with current networks."""
        tau = self._flat_algorithm._tau
        self._flat_algorithm._tau = 1.0
        self._flat_algorithm.sync_weight()
        self._flat_algorithm._tau = tau

    def load_parameters(self, params):
        """Load all learned parameters from dictionary."""
        assert self._flat_algo_name == params["flat_algo_name"]
        self._model.load_parameters(params["model_params"])
        self._override_target_networks()

    def load_state(self, state, dir_path = None):
        """Load state from dictionary."""
        assert self._flat_algo_name == state["flat_algo_name"]
        self._model.load_state(state["model_state"])
        self._model.load_parameters(state["model_params"])
        self._override_target_networks()
        if isinstance(state["replay_buffer"], ts.data.ReplayBuffer):
            self._replay_buffer = state["replay_buffer"]
        else:
            del self._replay_buffer
            self._replay_buffer = ts.data.ReplayBuffer.load_hdf5(
                    os.path.join(dir_path, state["replay_buffer"])
                    )

max_stars_repo_name: nicoguertler/graphrl


text:
import numpy as np
import unittest

from learningcurve import LearningCurve

class LearningCurveTest(unittest.TestCase):
    def test_all_x_and_y_parameters_are_mandatory(self):
        with self.assertRaises(ValueError):
            LearningCurve(0, [2], None, None, None, None)
        with self.assertRaises(ValueError):
            LearningCurve(0, [10], np.random.rand(10, 5), np.random.rand(10, 1), None, None)

        successful_lc = LearningCurve(0, [5], np.random.rand(10, 5), np.random.rand(10, 1), np.random.rand(2, 5), np.random.rand(2, 1))

    def test_lambda_and_hidden_layer_sizes_are_mandatory(self):
        with self.assertRaises(ValueError):
            LearningCurve(None, [5], np.random.rand(10, 5), np.random.rand(10, 1), np.random.rand(2, 5), np.random.rand(2, 1))
        with self.assertRaises(ValueError):
            LearningCurve(0, [], np.random.rand(10, 5), np.random.rand(10, 1), np.random.rand(2, 5), np.random.rand(2, 1))
        with self.assertRaises(ValueError):
            LearningCurve(0, None, np.random.rand(10, 5), np.random.rand(10, 1), np.random.rand(2, 5), np.random.rand(2, 1))

        successful_lc = LearningCurve(0, [5], np.random.rand(10, 5), np.random.rand(10, 1), np.random.rand(2, 5), np.random.rand(2, 1))

    def test_train_and_cv_data_should_be_in_proper_shape(self):
        with self.assertRaises(ValueError):
            LearningCurve(0, [5], np.random.rand(10, 5), np.random.rand(9, 1), np.random.rand(2, 5), np.random.rand(2, 1))
        with self.assertRaises(ValueError):
            LearningCurve(0, [5], np.random.rand(10, 5), np.random.rand(10, 1), np.random.rand(2, 6), np.random.rand(2, 1))
        with self.assertRaises(ValueError):
            LearningCurve(0, [5], np.random.rand(10, 5), np.random.rand(10, 1), np.random.rand(2, 5), np.random.rand(3, 1))
        with self.assertRaises(ValueError):
            LearningCurve(0, [5], np.random.rand(10, 5), np.random.rand(10, 1), np.random.rand(2, 5), np.random.rand(2, 2))

        successful_lc = LearningCurve(0, [5], np.random.rand(10, 5), np.random.rand(10, 1), np.random.rand(2, 5), np.random.rand(2, 1))

    def test_learning_curve_generate_should_return_generator(self):
        import types
        lc = LearningCurve(0, [3], np.random.rand(10, 5), (np.random.rand(10, 3) > 0.5).astype(int), np.random.rand(2, 5), (np.random.rand(2, 3) > 0.5).astype(int))
        data_points = lc.generate()
        self.assertIsInstance(data_points, types.GeneratorType)

    def test_learning_curve_generates_proper_touples(self):
        lc = LearningCurve(0, [3], np.random.rand(10, 5), (np.random.rand(10, 3) > 0.5).astype(int), np.random.rand(2, 5), (np.random.rand(2, 3) > 0.5).astype(int))
        data_points = lc.generate()
        first_point = data_points.__next__()
        self.assertEqual(len(first_point), 3)
        self.assertIsInstance(first_point[0], int)
        self.assertEqual(first_point[0], 1)

    def test_learning_curve_generator_should_work_with_custom_indices(self):
        lc = LearningCurve(0, [3], np.random.rand(10, 5), (np.random.rand(10, 3) > 0.5).astype(int), np.random.rand(2, 5), (np.random.rand(2, 3) > 0.5).astype(int))
        custom_indices = [1, 3, 4];
        data_points = lc.generate(custom_indices)
        data_sizes = [x for (x, error_train, error_cv) in data_points]
        self.assertEqual(custom_indices, data_sizes)

ext: py

max_stars_repo_name: sasoana/multiclassANN

max_issues_repo_name: sasoana/multiclassANN

max_forks_repo_name: sasoana/multiclassANN

reason: import numpy


text:
import argparse
import os
import numpy as np
import pickle as pkl
import h5py
from scipy.io import loadmat
from tqdm import tqdm
import cv2
import gc

import matplotlib
matplotlib.use('agg')  # use matplotlib without GUI support
import matplotlib.pyplot as plt

import torch
import torch.optim as optim

from lib.models.unet import UNet
from lib.utils.net_utils import load_checkpoint
from lib.utils.data_utils import padding_array

# =================PARAMETERS=============================== #
parser = argparse.ArgumentParser()

# network settings
parser.add_argument('checkpoint', type=str, default=None, help='optional reload model path')

parser.add_argument('--use_normal', action='store_true', help='whether to use normal map as network input')
parser.add_argument('--use_img', action='store_true', help='whether to use rgb image as network input')
parser.add_argument('--use_occ', action='store_true', help='whether to use occlusion as network input')
parser.add_argument('--no_contour', action='store_true', help='whether to remove the first channel of occlusion')
parser.add_argument('--only_contour', action='store_true', help='whether to keep only the first channel of occlusion')

parser.add_argument('--th', type=float, default=0.7)
parser.add_argument('--lr', type=float, default=0.0001, help='learning rate of optimizer')

# pth settings
parser.add_argument('--result_dir', type=str, default='/space_sdd/NYU/depth_refine')
parser.add_argument('--occ_dir', type=str, default='/space_sdd/NYU/test_order_nms_pred_pretrain')
parser.add_argument('--data_dir', type=str, default='/home/xuchong/Projects/occ_edge_order/data/dataset_real/NYUv2/data/val_occ_order_raycasting_woNormal_avgROI_1mm')

opt = parser.parse_args()
print(opt)
# ========================================================== #

# =================CREATE DATASET=========================== #
eigen_crop = [0, 480, 0, 640]

def read_jiao():
    ours = []
    jiao_pred_path = '/space_sdd/NYU/depth_predictions/jiao_pred_mat/'
    for i in range(654):
        f = loadmat(jiao_pred_path + str(i+1) + '.mat')
        f = f['pred']
        ours.append(f)
    ours = np.array(ours)
    return ours

def read_laina():
    laina_pred = h5py.File('/space_sdd/NYU/depth_predictions/laina_predictions_NYUval.mat', 'r')['predictions']
    laina_pred = np.array(laina_pred).transpose((0, 2, 1))
    laina_pred = laina_pred[:, eigen_crop[0]:eigen_crop[1], eigen_crop[2]:eigen_crop[3]]
    return laina_pred

def read_sharpnet():
    with open('/space_sdd/NYU/depth_predictions/sharpnet_prediction.pkl', 'rb') as f:
        ours = pkl.load(f)
    ours = np.array(ours)
    ours = ours[:, eigen_crop[0]:eigen_crop[1], eigen_crop[2]:eigen_crop[3]]
    return ours

def read_eigen():
    ours = loadmat('/space_sdd/NYU/depth_predictions/eigen_nyud_depth_predictions.mat')
    ours = ours['fine_predictions']
    ours = ours.transpose((2, 0, 1))
    out = []
    for line in ours:
        line = cv2.resize(line, (640, 480))
        out.append(line)
    out = np.array(out)
    return out

def read_dorn():
    ours = []
    list_dirs = open('/space_sdd/NYU/depth_predictions/NYUV2_DORN/list_dorn_order.txt', 'r').readlines()
    for line in list_dirs:
        line = line.strip()
        f = loadmat('/space_sdd/NYU/depth_predictions/NYUV2_DORN/NYUV2_DORN/' + line)
        pred = f['pred'][eigen_crop[0]:eigen_crop[1], eigen_crop[2]:eigen_crop[3]]
        ours.append(pred)
    ours = np.array(ours)
    return ours

def read_bts():
    ours = []
    list_dirs = open('/space_sdd/NYU/depth_predictions/result_bts_nyu/pred_bts.txt', 'r').readlines()
    tmp_dict = dict()
    for line in list_dirs:
        line = line.strip()
        num_tmp = line.rfind('_')
        key = int(line[num_tmp+1:-4])
        f = cv2.imread('/space_sdd/NYU/depth_predictions/result_bts_nyu/raw/' + line, -1)
        f = f / 1000
        pred = f[eigen_crop[0]:eigen_crop[1], eigen_crop[2]:eigen_crop[3]]
        tmp_dict[key] = pred
    keys = list(tmp_dict.keys())
    keys.sort()
    for key in keys:
        ours.append(tmp_dict[key])
    del tmp_dict
    gc.collect()
    ours = np.array(ours)
    return ours

def read_vnl():
    ours = pkl.load(open('/space_sdd/NYU/depth_predictions/pred_VNL.pkl', 'rb'))
    ours = np.array(ours) * 10
    ours = ours[:, eigen_crop[0]:eigen_crop[1], eigen_crop[2]:eigen_crop[3]]
    return ours
# ========================================================== #

# ================CREATE NETWORK AND OPTIMIZER============== #
net = UNet(use_occ=opt.use_occ, no_contour=opt.no_contour, only_contour=opt.only_contour,
           use_aux=(opt.use_normal or opt.use_img))
optimizer = optim.Adam(net.parameters(), lr=opt.lr)

load_checkpoint(net, optimizer, opt.checkpoint)
net.cuda()
net.eval()
# ========================================================== #

# load in occlusion list
occ_list = sorted([name for name in os.listdir(opt.occ_dir) if name.endswith(".npy")])

# load in normal list
normal_list = sorted([name for name in os.listdir(opt.data_dir) if name.endswith("-normal.png")])

# load in rgb list
img_list = sorted([name for name in os.listdir(opt.data_dir) if name.endswith("-rgb.png")])

for method in tqdm(['jiao', 'laina', 'sharpnet', 'eigen', 'dorn', 'bts', 'vnl']):
    # read in depths
    func = eval('read_{}'.format(method))
    depths = func()
    depths = torch.from_numpy(np.ascontiguousarray(depths)).float().unsqueeze(1)
    assert len(occ_list) == depths.shape[0], 'depth map and occlusion map does not match !'

    with torch.no_grad():
        for i in tqdm(range(len(occ_list)), desc='refining depth prediction from {}'.format(method)):
            depth_coarse = depths[i].unsqueeze(0).cuda()

            occlusion = np.load(os.path.join(opt.occ_dir, occ_list[i]))

            # remove predictions with small score
            mask = occlusion[:, :, 0] <= opt.th
            occlusion[mask, 1:] = 0

            occlusion = padding_array(occlusion)
            occlusion = occlusion.unsqueeze(0).cuda()

            # forward pass
            if opt.use_normal:
                aux = cv2.imread(os.path.join(opt.data_dir, normal_list[i]), -1) / (2 ** 16 - 1) * 2 - 1
            elif opt.use_img:
                aux = cv2.imread(os.path.join(opt.data_dir, img_list[i]), -1) / 255
            else:
                aux = None
            if aux is not None:
                aux = padding_array(aux).unsqueeze(0).cuda()
            pred = net(depth_coarse, occlusion, aux)

            depth_refine = pred.clamp(1e-9).squeeze().cpu().numpy()
            depth_init = depth_coarse.squeeze().cpu().numpy()

            img_name = occ_list[i].split('-')[0]
            refine_name = os.path.join(opt.result_dir, method, 'depth_refine', '{}.png'.format(img_name))
            init_name = os.path.join(opt.result_dir, method, 'depth_init', '{}.png'.format(img_name))
            save_name = os.path.join(opt.result_dir, method, 'depth_npy', '{}.npy'.format(img_name))

            if not os.path.isdir(os.path.dirname(refine_name)):
                os.makedirs(os.path.dirname(refine_name))
            if not os.path.isdir(os.path.dirname(init_name)):
                os.makedirs(os.path.dirname(init_name))
            if not os.path.isdir(os.path.dirname(save_name)):
                os.makedirs(os.path.dirname(save_name))

            plt.imsave(refine_name, depth_refine)
            plt.imsave(init_name, depth_init)

            np.save(save_name, depth_refine)

reason: import numpy,from scipy


text:
"""
Discrete Fourier Transform (:mod:`numpy.fft`)
=============================================

.. currentmodule:: numpy.fft

Standard FFTs
-------------

.. autosummary::
   :toctree: generated/

   fft       Discrete Fourier transform.
   ifft      Inverse discrete Fourier transform.
   fft2      Discrete Fourier transform in two dimensions.
   ifft2     Inverse discrete Fourier transform in two dimensions.
   fftn      Discrete Fourier transform in N-dimensions.
   ifftn     Inverse discrete Fourier transform in N dimensions.

Real FFTs
---------

.. autosummary::
   :toctree: generated/

   rfft      Real discrete Fourier transform.
   irfft     Inverse real discrete Fourier transform.
   rfft2     Real discrete Fourier transform in two dimensions.
   irfft2    Inverse real discrete Fourier transform in two dimensions.
   rfftn     Real discrete Fourier transform in N dimensions.
   irfftn    Inverse real discrete Fourier transform in N dimensions.

Hermitian FFTs
--------------

.. autosummary::
   :toctree: generated/

   hfft      Hermitian discrete Fourier transform.
   ihfft     Inverse Hermitian discrete Fourier transform.

Helper routines
---------------

.. autosummary::
   :toctree: generated/

   fftfreq   Discrete Fourier Transform sample frequencies.
   rfftfreq  DFT sample frequencies (for usage with rfft, irfft).
   fftshift  Shift zero-frequency component to center of spectrum.
   ifftshift Inverse of fftshift.

Background information
----------------------

Fourier analysis is fundamentally a method for expressing a function as a
sum of periodic components, and for recovering the function from those
components.  When both the function and its Fourier transform are
replaced with discretized counterparts, it is called the discrete Fourier
transform (DFT).  The DFT has become a mainstay of numerical computing in
part because of a very fast algorithm for computing it, called the Fast
Fourier Transform (FFT), which was known to Gauss (1805) and was brought
to light in its current form by Cooley and Tukey [CT]_.  Press et al. [NR]_
provide an accessible introduction to Fourier analysis and its
applications.

Because the discrete Fourier transform separates its input into
components that contribute at discrete frequencies, it has a great number
of applications in digital signal processing, e.g., for filtering, and in
this context the discretized input to the transform is customarily
referred to as a *signal*, which exists in the *time domain*.  The output
is called a *spectrum* or *transform* and exists in the *frequency
domain*.

Implementation details
----------------------

There are many ways to define the DFT, varying in the sign of the
exponent, normalization, etc.  In this implementation, the DFT is defined
as

.. math::
   A_k =  \\sum_{m=0}^{n-1} a_m \\exp\\left\\{-2\\pi i{mk \\over n}\\right\\}
   \\qquad k = 0,\\ldots,n-1.

The DFT is in general defined for complex inputs and outputs, and a
single-frequency component at linear frequency :math:`f` is
represented by a complex exponential
:math:`a_m = \\exp\\{2\\pi i\\,f m\\Delta t\\}`, where :math:`\\Delta t`
is the sampling interval.

The values in the result follow so-called "standard" order: If ``A =
fft(a, n)``, then ``A[0]`` contains the zero-frequency term (the sum of
the signal), which is always purely real for real inputs. Then ``A[1:n/2]``
contains the positive-frequency terms, and ``A[n/2+1:]`` contains the
negative-frequency terms, in order of decreasingly negative frequency.
For an even number of input points, ``A[n/2]`` represents both positive and
negative Nyquist frequency, and is also purely real for real input.  For
an odd number of input points, ``A[(n-1)/2]`` contains the largest positive
frequency, while ``A[(n+1)/2]`` contains the largest negative frequency.
The routine ``np.fft.fftfreq(n)`` returns an array giving the frequencies
of corresponding elements in the output.  The routine
``np.fft.fftshift(A)`` shifts transforms and their frequencies to put the
zero-frequency components in the middle, and ``np.fft.ifftshift(A)`` undoes
that shift.

When the input `a` is a time-domain signal and ``A = fft(a)``, ``np.abs(A)``
is its amplitude spectrum and ``np.abs(A)**2`` is its power spectrum.
The phase spectrum is obtained by ``np.angle(A)``.

The inverse DFT is defined as

.. math::
   a_m = \\frac{1}{n}\\sum_{k=0}^{n-1}A_k\\exp\\left\\{2\\pi i{mk\\over n}\\right\\}
   \\qquad m = 0,\\ldots,n-1.

It differs from the forward transform by the sign of the exponential
argument and the default normalization by :math:`1/n`.

Type Promotion
--------------

`numpy.fft` promotes ``float32`` and ``complex64`` arrays to ``float64`` and
``complex128`` arrays respectively. For an FFT implementation that does not
promote input arrays, see `scipy.fftpack`.

Normalization
-------------

The default normalization has the direct transforms unscaled and the inverse
transforms are scaled by :math:`1/n`. It is possible to obtain unitary
transforms by setting the keyword argument ``norm`` to ``"ortho"`` (default is
`None`) so that both direct and inverse transforms will be scaled by
:math:`1/\\sqrt{n}`.

Real and Hermitian transforms
-----------------------------

When the input is purely real, its transform is Hermitian, i.e., the
component at frequency :math:`f_k` is the complex conjugate of the
component at frequency :math:`-f_k`, which means that for real
inputs there is no information in the negative frequency components that
is not already available from the positive frequency components.
The family of `rfft` functions is
designed to operate on real inputs, and exploits this symmetry by
computing only the positive frequency components, up to and including the
Nyquist frequency.  Thus, ``n`` input points produce ``n/2+1`` complex
output points.  The inverses of this family assumes the same symmetry of
its input, and for an output of ``n`` points uses ``n/2+1`` input points.

Correspondingly, when the spectrum is purely real, the signal is
Hermitian.  The `hfft` family of functions exploits this symmetry by
using ``n/2+1`` complex points in the input (time) domain for ``n`` real
points in the frequency domain.

In higher dimensions, FFTs are used, e.g., for image analysis and
filtering.  The computational efficiency of the FFT means that it can
also be a faster way to compute large convolutions, using the property
that a convolution in the time domain is equivalent to a point-by-point
multiplication in the frequency domain.

Higher dimensions
-----------------

In two dimensions, the DFT is defined as

.. math::
   A_{kl} =  \\sum_{m=0}^{M-1} \\sum_{n=0}^{N-1}
   a_{mn}\\exp\\left\\{-2\\pi i \\left({mk\\over M}+{nl\\over N}\\right)\\right\\}
   \\qquad k = 0, \\ldots, M-1;\\quad l = 0, \\ldots, N-1,

which extends in the obvious way to higher dimensions, and the inverses
in higher dimensions also extend in the same way.

References
----------

.. [CT] Cooley, James W., and John W. Tukey, 1965, "An algorithm for the
        machine calculation of complex Fourier series," *Math. Comput.*
        19: 297-301.

.. [NR] Press, W., Teukolsky, S., Vetterline, W.T., and Flannery, B.P.,
        2007, *Numerical Recipes: The Art of Scientific Computing*, ch.
        12-13.  Cambridge Univ. Press, Cambridge, UK.

Examples
--------

For examples, see the various functions.

"""

from ._pocketfft import *
from .helper import *

from numpy._pytesttester import PytestTester
test = PytestTester(__name__)
del PytestTester

max_stars_repo_name: rocketbot-cl/recognition

max_issues_repo_name: rocketbot-cl/recognition

max_forks_repo_name: rocketbot-cl/recognition

reason: from numpy


text:
import caffe
import numpy as np
DEBUG = False

class RemoveJunkLayer(caffe.Layer):
    def setup(self, bottom, top):
        if len(bottom)!=2:
            raise Exception("Just need feat and id_labels")
    def reshape(self, bottom, top):
        top[0].reshape(*bottom[0].data.shape)
        top[1].reshape(*bottom[1].data.shape)
    def forward(self, bottom, top):
        feat = bottom[0].data
        labels = bottom[1].data
        ind = np.where(labels!=-1)[0]
        top_feat = feat[ind]
        top[0].reshape(*top_feat.shape)
        top[0].data[...] = top_feat
        top_label = labels[ind]
        top[1].reshape(*top_label.shape)
        top[1].data[...] = top_label
    def backward(self, top, propagate_down, bottom):
        for j in range(2):
            if not propagate_down[j]:
                continue
            else:
                labels = bottom[1].data
                ind = np.where(labels!=-1)[0]
                ind_junk = np.where(labels == -1)[0]
                assert len(ind)==top[0].diff.shape[0]
                for i in range(len(ind)):
                    bottom[j].diff[ind[i]] = top[0].diff[i]
                bottom[j].diff[ind_junk] = 0

max_stars_repo_name: mysayalHan/person-search

max_issues_repo_name: mysayalHan/person-search

max_forks_repo_name: mysayalHan/person-search


text:
'''
Utility functions taken from: https://luna16.grand-challenge.org/tutorial/
'''

import SimpleITK as sitk
import numpy as np

def load_itk_image(filepath):
	itkimage = sitk.ReadImage(filepath)
	img = sitk.GetArrayFromImage(itkimage)

	origins = np.array(list(reversed(itkimage.GetOrigin())))
	spacings = np.array(list(reversed(itkimage.GetSpacing())))

	return img, origins, spacings

max_stars_repo_name: jdapoorv/lung-cancer-detector

max_issues_repo_name: jdapoorv/lung-cancer-detector

max_forks_repo_name: jdapoorv/lung-cancer-detector


text:
#! /usr/bin/python
# -*- coding: utf-8 -*-

import logging
logger = logging.getLogger(__name__)

import argparse

# import time
import pickle

# import sys
import os
import glob
import numpy as np
# """ import modules from lar-cc/lib """
# sys.path.insert(0, os.path.expanduser('~/projects/lar-cc/lib/py'))
# sys.path.insert(0, '/home/mjirik/projects/lar-cc/lib/py')

# import import_library as il
# lib_path = il.find_library_path("larcc", "larcc.py")
# sys.path.append(lib_path)
# from larcc import * # noqa

# input of test file nrn100.py (with definetion of V and FV)
# V = vertex coordinates
# FV = lists of vertex indices of every face (1-based, as required by pyplasm)
#
# sys.path.insert(1, '/Users/paoluzzi/Documents/RICERCA/pilsen/ricerca/')
# from nrn100 import *

def writeFile(filename, vertexes, faces, ftype='auto', shift_obj=True,
              ignore_empty_vertex_warning=False):
    """
    filename
    vertexes
    faces
    """
    if ftype == 'auto':
        _, ext = os.path.splitext(filename)
        ftype = ext[1:]

    if ftype in ('pkl', 'pickle', 'p'):
        pickle.dump([vertexes, faces], open(filename, 'wb'))
    elif ftype == 'obj':
        if shift_obj:
            faces = (np.asarray(faces) + 1).tolist()
        with open(filename, "w") as f:
            for i, vertex in enumerate(vertexes):
                __writeVertexLineToObjFile(f, vertex,
                                           ignore_empty_vertex_warning)
                    # import ipdb; ipdb.set_trace() #  noqa BREAKPOINT

            for face in faces:
                fstr = "f"
                for i in range(0, len(face)):
                    fstr += " %i" % (face[i])

                fstr += "\n"

                f.write(fstr)

def __writeVertexLineToObjFile(f, vertex, ignore_empty_vertex_warning):
    try:
        f.write("v %s %s %s\n" % (
            str(vertex[0]),
            str(vertex[1]),
            str(vertex[2])
        ))
    except IndexError:
        if not ignore_empty_vertex_warning:
            logger.warning('empty vertex %i ' % (vertex))
        f.write("v 0 0 0\n")

def readFile(filename, ftype='auto', shift_obj=True):
    # is maskk
    if filename.find('*') == -1 and \
        filename.find('?') == -1 and \
            filename.find('[') == -1:

        V, F = readOneFile(filename, ftype=ftype, shift_obj=shift_obj)

    else:
        V = []
        F = []
        filelist = glob.glob(filename)
        for fl in filelist:
            V1, F1 = readOneFile(fl, ftype=ftype, shift_obj=shift_obj)
            lenV = len(V)
            V = V + V1
            # add  len of prev V to indexes
            F1 = (np.array(F1) + lenV).tolist()
            F = F + F1
            # import ipdb; ipdb.set_trace() #  noqa BREAKPOINT

    return V, F

def readOneFile(filename, ftype='auto', shift_obj=True):
    """
    File open one file. It can autodetect type.
    """
    if not os.path.isfile(filename):
        logger.error('File "%s" not found' % (filename))
        exit(2)
    if ftype == 'auto':
        _, ext = os.path.splitext(filename)
        ftype = ext[1:]
        logger.info('file type autodetect: ' + ftype)

    if ftype in ('pkl', 'pickle', 'p'):
        fdata = pickle.load(open(filename, "rb"))
        vertexes, faces = fdata
    elif ftype in ('rawc'):
        with open(filename, "r") as f:
            vertexes, faces = __readRawcStream(f)
    elif ftype == 'obj':
        with open(filename, "r") as f:
            vertexes, faces = __readObjStream(f)
        if shift_obj:
            faces = (np.asarray(faces) - 1).tolist()

    return vertexes, faces

def __readRawcStream(f):
    vertexes = []
    faces = []
    for line in f.readlines():
        lnarr = line.strip().split(' ')
        lenlarr = len(lnarr)
        if lenlarr == 2:
            # number of vertexes and number of faces
            pass
        elif lenlarr == 6:
            vertex = [
                float(lnarr[0]),
                float(lnarr[1]),
                float(lnarr[2])
            ]
            vertexes.append(vertex)
        elif lenlarr == 3:
            face = [
                int(lnarr[0]),
                int(lnarr[2]),
                int(lnarr[1])
            ]
            faces.append(face)

    return vertexes, faces

def __readObjStream(f):
    vertexes = []
    faces = []
    for line in f.readlines():
        lnarr = line.strip().split(' ')
        if lnarr[0] == 'v':
            try:
                vertex = [
                    int(lnarr[1]),
                    int(lnarr[2]),
                    int(lnarr[3])
                ]
            except ValueError:
                vertex = [
                    float(lnarr[1]),
                    float(lnarr[2]),
                    float(lnarr[3])
                ]

            vertexes.append(vertex)
        elif lnarr[0] == 'f':
            face = [0] * (len(lnarr) - 1)
            for i in range(1, len(lnarr)):
                face[i - 1] = int(lnarr[i])
            faces.append(face)

    return vertexes, faces

def main():

    logger = logging.getLogger()
    logger.setLevel(logging.WARNING)
    ch = logging.StreamHandler()
    logger.addHandler(ch)

    # logger.debug('input params')

    # input parser
    parser = argparse.ArgumentParser(
        description="Laplacian smoothing"
    )
    parser.add_argument(
        '-i', '--inputfile',
        default=None,
        required=True,
        help='input file'
    )
    parser.add_argument(
        '-o', '--outputfile',
        default='smooth.obj',
        help='input file'
    )
    parser.add_argument(
        '-mib', '--moveindexbasis',
        type=int,
        default=0,
        help='indexes are 0-based or 1-based')
    parser.add_argument(
        '-d', '--debug', action='store_true',
        help='Debug mode')
    parser.add_argument(
        '--console', action='store_true',
        help='Debug mode')

    args = parser.parse_args()
    if args.debug:
        logger.setLevel(logging.DEBUG)

    # t0 = time.time()
    V, FV = readFile(args.inputfile)
    logger.info("Data readed from '%s'" % (args.inputfile))
    if args.moveindexbasis is not 0:
        FV = (np.array(FV) + args.moveindexbasis).tolist()

    if args.console:
        import ipdb; ipdb.set_trace() #  noqa BREAKPOINT

    writeFile(args.outputfile, V, FV)
    logger.info("Data stored to '%s'" % (args.outputfile))

if __name__ == "__main__":
    main()

max_stars_repo_name: gilyclem/larVolumeToObj

max_issues_repo_name: gilyclem/larVolumeToObj

max_forks_repo_name: gilyclem/larVolumeToObj


text:
import numpy
from theano import gof
from theano.gof import Constant, Generic, Op
from theano.gof.sched import key_to_cmp
from basic import tensor
import theano
##########################
# Disk Access
##########################

class LoadFromDisk(Op):
    """
    An operation to load an array from disk

    See Also
        load

    @note: Non-differentiable.
    """
    def __init__(self, dtype, broadcastable, mmap_mode=None):
        self.dtype = numpy.dtype(dtype)  # turn "float64" into numpy.float64
        self.broadcastable = broadcastable
        if mmap_mode not in (None, 'c'):
            raise ValueError("The only supported values for mmap_mode "
                    "are None and 'c', got %s" % mmap_mode)
        self.mmap_mode = mmap_mode
        self._info = (dtype, broadcastable, mmap_mode)

    def __eq__(self, other):
        return (type(self) == type(other) and self._info == other._info)

    def __hash__(self):
        return hash((type(self),) + self._info)

    def make_node(self, path):
        if isinstance(path, str):
            path = Constant(Generic(), path)
        return gof.Apply(self, [path], [tensor(self.dtype,
                                        broadcastable=self.broadcastable)])

    def perform(self, node, inp, out):
        path = inp[0]
        if (path.split('.')[-1] == 'npz'):
            raise ValueError("Expected a .npy file, got %s instead" % path)
        result = numpy.load(path, mmap_mode=self.mmap_mode)
        if result.dtype != self.dtype:
            raise TypeError("Expected an array of type %s, got %s instead" %
                    (self.dtype, result.dtype))
        out[0][0] = result

    def __str__(self):
        return "Load{dtype: %s, broadcastable: %s, mmep: %s}" % self._info

def load(path, dtype, broadcastable, mmap_mode=None):
    """
    Load an array from an .npy file.

    :param path: A Generic symbolic variable, that will contain a string
    :param dtype: The data type of the array to be read.
    :param broadcastable: The broadcastable pattern of the loaded array,
      for instance, (False,) for a vector, (False, True) for a column,
      (False, False) for a matrix.
    :param mmap_mode: How the file will be loaded. None means that the
      data will be copied into an array in memory, 'c' means that the file
      will be mapped into virtual memory, so only the parts that are
      needed will be actually read from disk and put into memory.
      Other modes supported by numpy.load ('r', 'r+', 'w+') cannot
      be supported by Theano.

    >>> from theano import *
    >>> path = Variable(Generic())
    >>> x = tensor.load(path, 'int64', (False,))
    >>> y = x*2
    >>> fn = function([path], y)
    >>> fn("stored-array.npy")
    array([0, 2, 4, 6, 8], dtype=int64)
    """

    return LoadFromDisk(dtype, broadcastable, mmap_mode)(path)

##########################
# MPI
##########################

try:
    from mpi4py import MPI
except ImportError:
    mpi_enabled = False
else:
    comm = MPI.COMM_WORLD
    mpi_enabled = True

class MPIRecv(Op):
    """
    An operation to asynchronously receive an array to a remote host using MPI

    See Also
       MPIRecv
       MPIWait

    @note: Non-differentiable.
    """

    def __init__(self, source, tag, shape, dtype):
        self.source = source
        self.tag  = tag
        self.shape = shape
        self.dtype = numpy.dtype(dtype) # turn "float64" into numpy.float64
        self.broadcastable = (False,) * len(shape)
        self._info = (source, tag, shape, dtype)

    def __eq__(self, other):
        return (type(self) == type(other) and self._info == other._info)

    def __hash__(self):
        return hash((type(self),) + self._info)

    def make_node(self):
        return gof.Apply(self, [], [theano.Variable(Generic()),
                                    tensor(self.dtype,
                                           broadcastable=self.broadcastable)])
    def perform(self, node, inp, out):

        data = numpy.zeros(self.shape, dtype=self.dtype)
        request = comm.Irecv(data, self.source, self.tag)

        out[0][0] = request
        out[1][0] = data

    def __str__(self):
        return "MPIRecv{source: %d, tag: %d, shape: %s, dtype: %s}" % self._info

    def infer_shape(self, node, shapes):
        return [None, self.shape]

    def do_constant_folding(self, node):
        return False

class MPIRecvWait(Op):
    """
    An operation to wait on a previously received array using MPI

    See Also
       MPIRecv

    @note: Non-differentiable.
    """

    def __init__(self, tag):
        self.tag = tag

    def __eq__(self, other):
        return type(self) == type(other) and self.tag == other.tag

    def __hash__(self):
        return hash((type(self), self.tag))

    def make_node(self, request, data):
        return gof.Apply(self, [request, data],
                               [tensor(data.dtype,
                                       broadcastable=data.broadcastable)])
    def perform(self, node, inp, out):

        request = inp[0]
        data    = inp[1]

        request.wait()

        out[0][0] = data

    def __str__(self):
        return "MPIRecvWait"

    def infer_shape(self, node, shapes):
        return [shapes[1]]

    view_map = {0: [1]}

class MPISend(Op):
    """
    An operation to asynchronously Send an array to a remote host using MPI

    See Also
       MPIRecv
       MPISendWait

    @note: Non-differentiable.
    """

    def __init__(self, dest, tag):
        self.dest = dest
        self.tag  = tag
        self._info = (dest, tag)

    def __eq__(self, other):
        return (type(self) == type(other) and self._info == other._info)

    def __hash__(self):
        return hash((type(self),) + self._info)

    def make_node(self, data):
        return gof.Apply(self, [data],
                               [theano.Variable(Generic()), data.type()])
    view_map = {1: [0]}

    def perform(self, node, inp, out):

        data = inp[0]

        request = comm.Isend(data, self.dest, self.tag)

        out[0][0] = request
        out[1][0] = data

    def __str__(self):
        return "MPISend{dest: %d, tag: %d}" % self._info

class MPISendWait(Op):
    """
    An operation to wait on a previously sent array using MPI

    See Also:
       MPISend

    @note: Non-differentiable.
    """

    def __init__(self, tag):
        self.tag = tag

    def __eq__(self, other):
        return type(self) == type(other) and self.tag == other.tag

    def __hash__(self):
        return hash((type(self), self.tag))

    def make_node(self, request, data):
        return gof.Apply(self, [request, data],
                               [theano.Variable(Generic())])

    def perform(self, node, inp, out):
        request = inp[0]
        request.wait()
        out[0][0] = True

    def __str__(self):
        return "MPISendWait"

def isend(var, dest, tag):
    return MPISend(dest, tag)(var)

def send(var, dest, tag):
    return MPISendWait(tag)(*isend(var, dest, tag))

def irecv(shape, dtype, source, tag):
    return MPIRecv(source, tag, shape, dtype)()

def recv(shape, dtype, source, tag):
    return MPIRecvWait(tag)(*irecv(shape, dtype, source, tag))

# Ordering keys for scheduling
def mpi_send_wait_key(a):
    """ Wait as long as possible on Waits, Start Send/Recvs early """
    if isinstance(a.op, (MPIRecvWait, MPISendWait)):
        return 1
    if isinstance(a.op, (MPIRecv, MPISend)):
        return -1
    return 0

def mpi_tag_key(a):
    """ Break MPI ties by using the variable tag - prefer lower tags first """
    if isinstance(a.op, (MPISend, MPIRecv, MPIRecvWait, MPISendWait)):
        return a.op.tag
    else:
        return 0

mpi_send_wait_cmp = key_to_cmp(mpi_send_wait_key)
mpi_tag_cmp = key_to_cmp(mpi_tag_key)

mpi_keys = (mpi_send_wait_key, mpi_tag_key)
mpi_cmps = (mpi_send_wait_cmp, mpi_tag_cmp)

max_stars_repo_name: ganguli-lab/Theano

max_issues_repo_name: ganguli-lab/Theano

max_forks_repo_name: ganguli-lab/Theano

reason: import numpy,import theano,from theano


text:
from __future__ import division
import numpy as np
na = np.newaxis
from warnings import warn
import scipy.linalg

from simplex import mesh, proj_matrix
from dirichlet import log_censored_dirichlet_density
from density import kde
import parallel

def chunk_indices(T,npoints):
    return map(lambda x: x[-1]+1, np.array_split(np.arange(T),npoints))

def kldist(pvec,qvec):
    '''
    between two pmfs (or KDEs evaluated over same supports and normalized)
    sorta DEPRECATED...
    '''
    return np.sum(np.where(np.logical_or(pvec==0,qvec==0),0.,pvec*(np.log(pvec) - np.log(qvec))))

def kldist_samples(samples,q):
    r'''
    from phat(x) = \frac{1}{|samples|} \sum_{xbar \in samples} \delta(x-xbar)
    to q

    samples is a list of points in X
    q is a function with domain X, like from a call to density.kde
    must have \int{x \in X} q(x) \approx 1
    '''
    warn('untested')
    N = samples.shape[0]
    return -1./N * np.log(N * q(samples)).sum()

def get_autocorr(chains):
    '''
    component-by-component
    '''
    chains = np.array(chains)
    results = np.zeros(chains.shape)
    for chainidx, chain in enumerate(chains):
        for idx in range(chain.shape[1]):
            temp = chain[:,idx] - chain[:,idx].mean(0)
            temp = np.correlate(temp,temp,'full')
            results[chainidx,:,idx] = temp[temp.shape[0]//2:]
    return results

def get_statistic_convergence(chains,ncomputepoints):
    '''
    mean, var of components, and l2 distances to the truth
    '''
    chains = np.array(chains,ndmin=3)

    p = chains.shape[2]

    ### estimate true parameters using all samples
    truemean = chains[:,chains.shape[1]//2:,:].mean(0).mean(0) # vector of length p
    truevar = chains[:,chains.shape[1]//2:,:].reshape((-1,p)).var(0) # vector of length p

    ### compute statistics at the compute points
    # preallocate outputs
    means = np.empty((chains.shape[0],ncomputepoints,p))
    variances = np.empty(means.shape)

    # loop over chains
    for chainidx, chain in enumerate(chains):
        # loop over chunks
        for chunkidx, sampleidx in enumerate(chunk_indices(chain.shape[0], ncomputepoints)):
            # compute statistics up to the current time
            means[chainidx,chunkidx] = chain[sampleidx//2:sampleidx].mean(0)
            variances[chainidx,chunkidx] = chain[sampleidx//2:sampleidx].var(0)

    ### get distances to truth at the compute points
    mean_distances = np.sqrt(((means - truemean[na,na,:])**2).sum(-1))
    var_distances = np.sqrt(((variances - truevar[na,na,:])**2).sum(-1))

    return (means,variances), (truemean,truevar), (mean_distances, var_distances)

def get_Rhat(chains,ncomputepoints):
    '''
    see Monitoring Convergence of Iterative Simulations
    aka Multivariate Scale Reduction Factor
    '''
    chains_all = np.array(chains)

    outs = np.empty(ncomputepoints)
    for chunkidx, sampleidx in enumerate(chunk_indices(chains.shape[1],ncomputepoints)):
        chains = chains_all[:,sampleidx//2:sampleidx,:]

        m,n,p = chains.shape
        # get means
        mu_all = chains.mean(0).mean(0)
        mu_each = chains.mean(1)
        # B/n is between-chain covariance
        temp = mu_each - mu_all
        B_over_n = 1/(m-1) * temp.T.dot(temp)
        # W is within-chain covaraince
        temp = chains - mu_each[:,na,:]
        W = 1/(m*(n-1)) * np.tensordot(temp,temp,axes=([0,1],[0,1]))

        Vhat = (n-1)/n * W + (1+1/m) * B_over_n

        # project to subspaces on which the matrices are full rank (since they
        # are rank-deficient due to the simplex constraint)
        Vhatp, Wp = proj_matrix(Vhat), proj_matrix(W)
        # compute MSPRF
        outs[chunkidx] = scipy.linalg.eigvalsh(Vhatp,Wp).max() # same as spectral radius of W^-1 Vhat

    return outs

def get_kldivs(chains,ncomputepoints,meshsize=100,params={'alpha':2.,'beta':30.,'data':np.array([[0,2,0],[0,0,0],[0,0,0]])}):
    alpha, beta, data = params['alpha'], params['beta'], params['data']
    p = chains.shape[2]
    assert p == 3, 'this test only works on 3 dimensional examples'

    ### construct a 'true' density object by discrete approximate integration
    # get density evaluated on a mesh, (mesh3D, dvals)
    mesh3D = mesh(meshsize)
    dvals = log_censored_dirichlet_density(mesh3D, alpha, data=data)
    dvals = np.exp(dvals - dvals.max())
    dvals /= dvals.sum()

    # interpolate into a density function
    true_density = kde(0.05,mesh3D,dvals)

    ### get kl divergence to truth at cmopute points
    # preallocate outputs
    dists = np.zeros((chains.shape[0],ncomputepoints))

    # loop over chains
    for chainidx, chain in enumerate(chains):
        # loop over chunks
        for chunkidx, sampleidx in enumerate(chunk_indices(chain.shape[0], ncomputepoints)):
            # get relevant samples
            samples = chain[sampleidx//2:sampleidx]
            # compute kldiv against true_density
            dists[chainidx,chunkidx] = kldist_samples(samples,true_density)

    return dists

max_stars_repo_name: mattjj/dirichlet-truncated-multinomial

max_issues_repo_name: mattjj/dirichlet-truncated-multinomial

max_forks_repo_name: mattjj/dirichlet-truncated-multinomial

reason: import numpy,import scipy


text:
import numpy as np

def policy_evaluation_one_step(env, policy, value_table, gamma=0.9):
    # For each state
    for s_idx, s in enumerate(env.states):
        if env.terminal_table[s_idx] == 1:
            value_table[s_idx] = 0
            continue

        new_v = 0
        # For each action
        for a in range(4):
            pi_prob = policy[s_idx, a]
            s_prime_pmf = env.transition_table[s_idx, a]
            # For each possible next state
            for s_prime_idx, trans_prob in enumerate(s_prime_pmf):
                r = env.reward_table[s_idx, a]
                if env.terminal_table[s_prime_idx] == 0:
                    new_v += pi_prob * trans_prob * (r + gamma * value_table[s_prime_idx])
                else:
                    new_v += pi_prob * trans_prob * r
        value_table[s_idx] = new_v

def policy_improvement(env, policy, value_table, gamma=0.9):

    for i_s, s in enumerate(env.states):
        # Initialize q-values for current state and new policy
        q = np.zeros(4)
        new_policy = np.zeros(4)

        # Compute q-value for each action in current state
        for a in range(4):
            r = env.reward_table[i_s, a]
            next_s_pmf = env.transition_table[i_s, a]
            for i_next_s, prob in enumerate(next_s_pmf):
                q[a] += prob * (r + gamma * value_table[i_next_s])

        # Set policy to greedy w.r.t. Q-values computed above
        new_policy[np.argmax(q)] = 1
        policy[i_s] = new_policy

def epsilon_greedy_pi_from_q_table(env, q_table, epsilon):
    policy = np.zeros((len(env.states), 4))
    for i_s, s in enumerate(env.states):
        q_values = q_table[i_s]
        a_star = np.argmax(q_values)
        policy[i_s, :] = [epsilon/4]*4
        policy[i_s, a_star] += 1-epsilon
    return policy

if __name__ == '__main__':
    from lib.grid_world import grid_world_3x3 as env
    from lib.plot_utils import plot_policy, plot_value_table
    import matplotlib.pyplot as plt

    np.random.seed(2)
    env.reset()
    value_table = np.random.rand(130)
    policy = np.ones((len(env.states), 4)) * 0.25
    for i, s in enumerate(env.states):
        policy[i, :] = np.random.dirichlet([0.1, 0.1, 0.1, 0.1])

    plot_value_table(env, value_table)
    plot_policy(env, policy)
    plt.show()

max_stars_repo_name: JulianoLagana/deep-machine-learning

max_issues_repo_name: JulianoLagana/deep-machine-learning

max_forks_repo_name: JulianoLagana/deep-machine-learning


text:
# coding: utf-8

# In[1]:

# coding: utf-8

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.optim import lr_scheduler
from torch.autograd import Variable
import torchvision
from torchvision import datasets, models, transforms
import time
import os
import copy

os.environ["CUDA_VISIBLE_DEVICES"] = "3"

# 参数
learning_rate = 0.01
momentum = 0.9
epochs = 40
batch_size = 4
display_step = 1
shuffle = True
num_classes = 4

# In[2]:

class TwoStreamNet(nn.Module):

    def __init__(self):
        super(TwoStreamNet, self).__init__()

        # self.conv1 = nn.Conv2d(3, 96, 7, stride=2)
        # self.conv1_norm = nn.BatchNorm2d(96)
        # self.conv1_relu = nn.ReLU(inplace=True)
        # self.conv1_pool = nn.MaxPool2d(kernel_size=3, stride=2)
        #
        # self.conv2 = nn.Conv2d(96, 256, 5, stride=2)
        # self.conv2_norm = nn.BatchNorm2d(256)
        # self.conv2_relu = nn.ReLU(inplace=True)
        # self.conv2_pool = nn.MaxPool2d(kernel_size=3, stride=2)
        #
        # self.conv3 = nn.Conv2d(256, 512, 3, stride=1)
        # self.conv3_relu = nn.ReLU(inplace=True)
        #
        # self.conv4 = nn.Conv2d(512, 512, 3, stride=1)
        # self.conv4_relu = nn.ReLU(inplace=True)
        #
        # self.conv5 = nn.Conv2d(512, 512, 3, stride=1)
        # self.conv5_relu = nn.ReLU(inplace=True)
        # self.conv5_pool = nn.MaxPool2d(kernel_size=3, stride=2)

        self.features = nn.Sequential(
            nn.Conv2d(3, 96, 7, stride=2),
            nn.BatchNorm2d(96),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(kernel_size=3, stride=2),
            nn.Conv2d(96, 256, 5, stride=2),
            nn.BatchNorm2d(256),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(kernel_size=3, stride=2),
            nn.Conv2d(256, 512, 3, stride=1),
            nn.ReLU(inplace=True),
            nn.Conv2d(512, 512, 3, stride=1),
            nn.ReLU(inplace=True),
            nn.Conv2d(512, 512, 3, stride=1),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(kernel_size=3, stride=2),
        )
        self.classifier = nn.Sequential(
            nn.Linear(512 * 2 * 2, 4096),
            nn.ReLU(True),
            nn.Dropout(),
            nn.Linear(4096, 2048),
            nn.ReLU(True),
            nn.Dropout(),
            nn.Linear(2048, 4),
        )

    def forward(self, x):
        x = self.features(x)
        x = x.view(x.size(0), -1)
        x = self.classifier(x)
        x = F.softmax(x, dim=1)
        return x

model = TwoStreamNet()
print(model)

# In[4]:

# 数据准备
# crop:裁剪 resize:缩放 flip:翻转
data_transforms = {
    'train': transforms.Compose([
        transforms.Resize(256),
        transforms.RandomCrop(224),
        transforms.RandomHorizontalFlip(),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ]),
    'validation': transforms.Compose([
        transforms.Resize(256),
        transforms.CenterCrop(224),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ]),
}

# 空间域图像
data_dir_spatial = '../images/'
image_datasets_spatial = {x: datasets.ImageFolder(os.path.join(data_dir_spatial, x),
                                          data_transforms[x]) for x in ['train', 'validation']}
# torchvision.datasets.ImageFolder返回的是list，这里用torch.utils.data.DataLoader类将list类型的输入数据封装成Tensor数据格式
dataloders_spatial = {x: torch.utils.data.DataLoader(image_datasets_spatial[x],
                                             batch_size = batch_size,
                                             shuffle = shuffle,
                                             num_workers = 10) for x in ['train', 'validation']}
dataset_sizes_spatial = {x: len(image_datasets_spatial[x]) for x in ['train', 'validation']}

dataset_sizes = {x: len(image_datasets[x]) for x in ['train', 'validation']}

# 时间域图像
data_dir_temporal = '../flow_images/'
image_datasets_temporal = {x: datasets.ImageFolder(os.path.join(data_dir_temporal, x),
                                          data_transforms[x]) for x in ['train', 'validation']}
# torchvision.datasets.ImageFolder返回的是list，这里用torch.utils.data.DataLoader类将list类型的输入数据封装成Tensor数据格式
dataloders_temporal = {x: torch.utils.data.DataLoader(image_datasets_temporal[x],
                                             batch_size = batch_size,
                                             shuffle = shuffle,
                                             num_workers = 10) for x in ['train', 'validation']}
dataset_sizes_temporal = {x: len(image_datasets_temporal[x]) for x in ['train', 'validation']}

# In[5]:

# 是否使用GPU
use_gpu = torch.cuda.is_available()
if use_gpu:
    model = model.cuda()

print("use_gpu: " + str(use_gpu))

# In[6]:

# 开始训练
since = time.time()
best_model_wts = model.state_dict()
best_acc = 0.0
loss_train = [] # 训练集loss
acc_train = [] # 训练集正确率
loss_val = [] # 验证集loss
acc_val = [] # 验证集正确率
best_matrix = [[0 for i in range(num_classes)] for i in range(num_classes)]

# 定义损失函数，这里采用交叉熵函数
loss_fn = nn.CrossEntropyLoss()

for epoch in range(epochs):
    if epoch % display_step == 0:
        print('Epoch [{}/{}]:'.format(epoch + 1, epochs))

    # 定义优化函数，这里采用随机梯度下降法
    if epoch == 0:
        learning_rate = 1e-2
        optimizer = optim.SGD(model.parameters(), learning_rate, momentum)
    if epoch == 50000:
        learning_rate = 1e-3
        optimizer = optim.SGD(model.parameters(), learning_rate, momentum)
    if epoch == 70000:
        learning_rate = 1e-4
        optimizer = optim.SGD(model.parameters(), learning_rate, momentum)

    # 每一轮都跑一遍训练集和验证集
    for phase in ['train', 'validation']:
        if phase == 'train':
            i = 1
            j = 1
            # exp_lr_scheduler.step()
            model.train()  # 把module设成training模式，对Dropout和BatchNorm有影响
        else:
            i = 1
            j = 2
            model.eval()  # 把module设置为评估模式

        running_loss = 0.0
        running_corrects = 0
        matrix = [[0 for i in range(num_classes)] for i in range(num_classes)]

        # Iterate over data.
        for data_1, data_2 in zip(dataloders_spatial[phase], dataloders_temporal[phase]):
            # get the inputs
            inputs_1, labels_1 = data_1
            inputs_2, labels_2 = data_2

            # wrap them in Variable
            # if use_gpu:
            #     inputs = inputs.cuda()
            #     labels = labels.cuda()
            # else:
            #     inputs, labels = Variable(inputs), Variable(labels)

            # PyTorch更新至0.4.0后，将Variable和Tensor合并
            if use_gpu:
                inputs_1 = inputs_1.cuda()
                labels_1 = labels_1.cuda()
                inputs_2 = inputs_2.cuda()
                labels_2 = labels_2.cuda()

            # 先将网络中的所有梯度置0
            optimizer.zero_grad()

            # 网络的前向传播
            outputs_1 = model(inputs_1)
            outputs_2 = model(inputs_2)

            print('outputs_1:' + outputs_1)
            print('outputs_2:' + outputs_2)
            print('outputs_average:' + str((outputs_1 + outputs_2) / 2))

            # 计算损失
            # loss = loss_fn(outputs, labels)
            loss = loss_fn((outputs_1 + outputs_2) / 2, (labels_1 + labels_2) / 2)

            # 得到模型预测该样本属于哪个类别的信息
            # '_'就是一个变量，换成a也是可以的，没有特别的意思，不过一般用_表示的变量好像都是没什么用的一个临时变量，大概是
            # 一个编程习惯吧。所以这边'_,'没有特殊的含义，'_'就是一个变量，只是为了让preds取到max函数返回值的第二项，
            # 即找到的最大值的索引位置（对应到这里就是类别标签）
            # （max函数解释见https://pytorch.org/docs/stable/torch.html?highlight=max#torch.max）
            _, preds = torch.max(((outputs_1 + outputs_2) / 2).data, 1)

            # 训练时，应用回传和优化
            if phase == 'train':
                loss.backward()
                optimizer.step()

            # 记录当前batch_size的loss以及数据对应的分类准确数量
            running_loss += loss.item()
            running_corrects += torch.sum(preds == labels.data)
            if phase == 'validation':
                for k in range(0, num_classes):
                    matrix[labels.data.cpu().numpy()[k]][preds.cpu().numpy()[k]] += 1

            print('\t{} {}-{}: Loss: {:.4f} Acc: {:.4f}%'.format(phase, epoch + 1, i, loss.item()/4, torch.sum(preds == labels.data).item()/4.0*100))
            i = i + 1

        # 计算并打印这一轮训练的loss和分类准确率
        if j == 1:
            epoch_loss_train = running_loss / dataset_sizes['train']
            epoch_acc_train = running_corrects.item() / dataset_sizes['train']
            loss_train.append(epoch_loss_train)
            acc_train.append(epoch_acc_train)
        else:
            epoch_loss_val = running_loss / dataset_sizes['validation']
            epoch_acc_val = running_corrects.item() / dataset_sizes['validation']
            loss_val.append(epoch_loss_val)
            acc_val.append(epoch_acc_val)

        if epoch % display_step == 0 and j == 2:
            print('\ttrain Loss: {:.4f} Acc: {:.4f}%'.format(epoch_loss_train, epoch_acc_train*100))
            print('\tvalidation Loss: {:.4f} Acc: {:.4f}%'.format(epoch_loss_val, epoch_acc_val*100))

        # deep copy the model
        if phase == 'validation' and epoch_acc_val > best_acc:
            best_acc = epoch_acc_val
            best_model_wts = model.state_dict()
            print("网络参数更新")
            # 保存最优参数
            torch.save(best_model_wts, '../parameter/two_stream_fusion.pth')
            best_matrix = copy.deepcopy(matrix)
#             print("Model's state_dict:")
#             for param_tensor in best_model_wts:
#                 print(param_tensor, "\t", best_model_wts[param_tensor].size())
    time_elapsed = time.time() - since
    print('Time passed {:.0f}h {:.0f}m {:.0f}s'.format(time_elapsed // 3600, (time_elapsed % 3600) // 60, time_elapsed % 60))
    print('-' * 20)

# 计算训练所耗时间
time_elapsed = time.time() - since
print('Training complete in {:.0f}h {:.0f}m {:.0f}s'.format(time_elapsed // 3600, (time_elapsed % 3600) // 60, time_elapsed % 60))
print('Best validation Acc: {:4f}'.format(best_acc))

# In[ ]:

import numpy as np
import matplotlib
import matplotlib.pyplot as plt
get_ipython().run_line_magic('matplotlib', 'inline')

print('loss_train: ' + str(loss_train))
print('loss_val: ' + str(loss_val))
print('acc_train: ' + str(acc_train))
print('acc_val: ' + str(acc_val))

# 绘制第一个图，在一幅图上画两条曲线
plt.figure()
plt.title("Loss",fontsize=16)
plt.xlabel("Epochs")
plt.ylabel("Loss")
plt.xticks(np.arange(1, 41, 2.0))
plt.plot(range(1,epochs + 1), loss_train,color='r', linewidth = 3.0, label='train')
plt.plot(range(1,epochs + 1), loss_val,color='b', linewidth = 3.0, label='validation')
plt.legend()  # 设置图例和其中的文本的显示

# 绘制第二个图，在一幅图上画两条曲线
plt.figure()
plt.title("Predicted accuracy",fontsize=16)
plt.xlabel("Epochs")
plt.ylabel("Acc")
plt.xticks(np.arange(1, 41, 2.0))
plt.plot(range(1,epochs + 1), acc_train,color='r', linewidth = 3.0, label='train')
plt.plot(range(1,epochs + 1), acc_val,color='b', linewidth = 3.0, label='validation')
plt.legend()  # 设置图例和其中的文本的显示

plt.show()

# In[ ]:

def imshow(img):
    img = img / 2 + 0.5     # unnormalize
    npimg = img.numpy()
    plt.imshow(np.transpose(npimg, (1, 2, 0)))

classes = ('75', '77', '79', '81')

dataiter = iter(dataloders['validation'])
images, labels = dataiter.next()
print(images.size())
# print images
imshow(torchvision.utils.make_grid(images))
print('GroundTruth: ', ' '.join('%5s' % classes[labels[z]] for z in range(4)))

# test
outputs = model(images.cuda())
_, predicted = torch.max(outputs, 1)
print('Predicted: ', ' '.join('%5s' % classes[predicted[z]] for z in range(4)))

# In[ ]:

conc = {
    '0': '75  ',
    '1': '77  ',
    '2': '79  ',
    '3': '81  '
}

print("\t   Predicted\n")
print("\t   75\t77\t79\t81\n")
for i in range(0, num_classes):
    print("Actual ", end='')
    print(conc[str(i)], end='')
    for j in range(0, num_classes):
        print(str(best_matrix[i][j]) + '\t', end='')
    print('\n')

max_stars_repo_name: Jacen-Chou/Paste-Video-Classification

max_issues_repo_name: Jacen-Chou/Paste-Video-Classification

max_forks_repo_name: Jacen-Chou/Paste-Video-Classification


text:
# Copyright 2019 The Keras Tuner Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from abc import abstractmethod
import gc
import json
import os
import subprocess
import sys

import numpy as np
import tensorflow as tf
from tensorflow import python as tf_python

class GFileProxy(object):
    """Provides a subset of the tensorflow-2.0 API and proxies the
    calls through to the appropriate tensorflow API."""

    @abstractmethod
    def Open(self, name, mode):
        """Open a file.

        Args:
            name (str): name of the file
            mode (str): one of 'r', 'w', 'a', 'r+', 'w+', 'a+'. Append 'b' for
                bytes mode.

        Returns:
            GFile - a GFile object representing the opened file.
        """

    @abstractmethod
    def makedirs(self, path):
        """Creates a directory and all parent/intermediate directories.

        It succeeds if path already exists and is writable.

        Args:
            path (str): string, name of the directory to be created

        Raises:
            errors.OpError: If the operation fails.
        """

    @abstractmethod
    def exists(self, path):
        """Determines whether a path exists or not.
        Args:
            path: string, a path

        Returns:
            Boolean: True if the path exists, whether it's a file or a
               directory. False if the path does not exist and there are no
               filesystem errors.

        Raises:
            errors.OpError: Propagates any errors
                reported by the FileSystem API.
        """

    @abstractmethod
    def rmtree(self, path):
        """Deletes everything under path recursively.

        Args:
            path: string, a path

        Raises:
            errors.OpError: If the operation fails.
        """

    @abstractmethod
    def glob(self, pattern):
        """Returns a list of files that match the given pattern(s).

        Args:
            pattern: string or iterable of strings. The glob pattern(s).

        Returns:
            A list of strings containing filenames that
                match the given pattern(s).

        Raises:
            errors.OpError: If there are filesystem / directory listing errors.
        """

    @abstractmethod
    def remove(self, path):
        """Deletes the path located at 'path'.

        Args:
            path: string, a path

        Raises:
            errors.OpError: Propagates any errors reported
                by the FileSystem API.
        """

    @abstractmethod
    def copy(self, src, dst, overwrite=False):
        """Copies data from src to dst.

        Args:
            src: string, name of the file whose contents need to be copied
            dst: string, name of the file to which to copy to
            overwrite: boolean, if false its an error for
                newpath to be occupied by an existing file.

        Raises:
            errors.OpError: If the operation fails.
        """

class IOProxy(object):
    """Proxy for tf.io. In general, the GFile module is redirected to a
    custom proxy object, and all other calls are directly passed to `tf.io`"""

    def __init__(self):
        self.gfile = GFileProxy()

    def __getattr__(self, name):
        return getattr(tf.io, name)

class PythonProxy(object):
    """Proxy for tf.python. In general, the calls are passed directly to
    tf.python. Howerver, items in tf.python which exist in only one of tf 1.x
    or tf 2.x may have be forwarded to the appropriate module.
    """

    def __getattr__(self, name):
        return getattr(tf.python, name)

class TensorflowProxy(object):
    """A proxy object for tensorflow calls.

    This proxy system aims to simplify the use of tensorflow across different
    versions(1.11-1.14, 2.x) despite the fact that the APIs have drastically
    changed.
    """

    def __init__(self):
        self.io = IOProxy()
        self.python = tf_python

    def __getattr__(self, name):
        return getattr(tf, name)

class UtilsBase(object):
    """Base implementation of tensorflow utilities."""

    def __init__(self, tf_proxy):
        """Construct a utils class with the given tensorflow proxy."""
        self.tf_proxy = tf_proxy

    def compute_model_size(self, model):
        params = [self.tf_proxy.keras.backend.count_params(
            p) for p in set(model.trainable_weights)]
        return int(np.sum(params))

    def serialize_loss(self, loss):
        """Serialize the loss information for a model.

        Example:
            `json_loss_config = json.dumps(serialize_loss(model.loss))`

        Args:
            loss - One of the following:
                (str): Name of one of the loss functions known to Keras.
                (Callable): A function or callable object.
                    Must be registered as a Keras loss.
                (dict): A dictionary mapping output nodes to loss functions in
                    string or callable form. Loss functions must be represented
                    as a str or Callable, as above.
                (list): A list of loss functions, applied to the output nodes.
                    Loss functions must be represented as a str or Callable,
                    as above.
        """
        if isinstance(loss, str):
            return loss
        elif isinstance(loss, list):
            loss_out = []
            for l in loss:
                loss_out.append(self.serialize_loss(l))
            return loss_out
        elif isinstance(loss, dict):
            loss_out = {}
            for k, v in loss.items():
                loss_out[k] = self.serialize_loss(v)
            return loss_out
        else:
            return self.tf_proxy.keras.losses.serialize(loss)

    def deserialize_loss(self, loss):
        """ Deserialize a model loss, serialized by serialize_loss, above,
            returning a single loss function, list of losses, or dict of
            lossess, depending on what was serialized.

            Args:
                loss: JSON configuration representing the loss or losses.
        """

        if isinstance(loss, dict):
            loss_out = {}
            for output, l in loss.items():
                loss_out[output] = self.tf_proxy.keras.losses.deserialize_loss(
                    l)
            return loss_out
        elif isinstance(loss, list):
            loss_out = []
            for l in loss:
                loss_out.append(self.tf_proxy.keras.losses.deserialize_loss(l))
            return loss_out
        else:
            return self.tf_proxy.keras.losses.deserialize(loss)

    def freeze_graph(
            self,
            saved_model_path,
            output_graph_path,
            output_tensor_names):
        """Create a frozen version of the specified saved model.

        This cannot be done in the main session, as the names will conflict
        with the existing model. Rather than resorting to manipulating
        sessions in a error-prone and messy fashion, we simply spawn a
        subprocess to freeze the graph.

        Args:
            saved_model_path: str, path to the saved model directory.
            output_graph_path: str, path to the file where the output graph
                will be written.
            output_tensor_names: str, comma separated list of tensors to set
                as the outputs for the graph.
        """

        command = [
            sys.executable,
            "-m",
            "tensorflow.python.tools.freeze_graph",
            "--input_saved_model_dir=%s" % saved_model_path,
            "--output_node_names=%s" % output_tensor_names,
            "--placeholder_type_enum=3",  # int32
            "--output_graph=%s" % output_graph_path]

        process = subprocess.Popen(command)
        process.wait()

    def reload_model(
            self,
            config_file,
            weights_file,
            results_file,
            compile=False):
        """ Reconstructs a model from the outputs of a kerastuner run.

        Args:
            config_file(string): Configuration filename.
            weights_file(string): Keras weights filename.
            results_file(string): Results filename.
            compile(bool, optional): Defaults to False. If True, the optimizer
                and loss will be read from the Instance, and the model will be
                compiled.

        Returns:
            tf.keras.models.Model: The(optionally compiled) Model.
        """

        # Reconstruct the model.
        config = self.read_file(config_file)
        model = self.tf_proxy.keras.models.model_from_json(config)
        model.load_weights(weights_file)

        # If compilation is requested, we need to reload the results file to find
        # which optimizer and losses the model used.
        if compile:
            results_file = json.loads(self.read_file(results_file))
            loss = self.deserialize_loss(results_file["loss_config"])
            optimizer = self.tf_proxy.keras.optimizers.deserialize(
                results_file["optimizer_config"])
            model.compile(loss=loss, optimizer=optimizer)

        return model

    def get_input_ops(self, model):
        "Get the names of the input operations for the given model."
        return [node.op.name for node in model.inputs]

    def get_input_tensors(self, model):
        "Get the names of the input operations for the given model."
        return [node.op for node in model.inputs]

    def get_output_tensors(self, model):
        "Get the names of the output operations for the given model."
        return [node.op for node in model.outputs]

    def get_input_types(self, model):
        "Get the dtypes of the input operations for the given model."
        outputs = []
        for node in model.inputs:
            if hasattr(node, "dtype"):
                outputs.append(node.dtype)
            elif hasattr(node.op, "dtype"):
                outputs.append(node.op.dtype)

        return outputs

    def get_output_ops(self, model):
        "Get the names of the output operations for the given model."
        if isinstance(model.output, list):
            return [x.op.name for x in model.output]
        else:
            return [model.output.op.name]

    def write_graph(self, graph_def, filename, as_text=True):
        "Write the given graph_def to the specified file."
        self.tf_proxy.io.write_graph(graph_def,
                                     logdir=os.path.dirname(filename),
                                     name=os.path.basename(filename),
                                     as_text=as_text)

    def load_savedmodel(
            self,
            session,
            export_dir,
            tags=None):
        """Load a SavedModel into the given session."""
        return self.tf_proxy.saved_model.load(
            session,
            tags,
            export_dir)

    def save_savedmodel(self, model, path, tmp_path):
        """Write a SavedModel to the given path."""
        # Implemented by subclasses
        raise NotImplementedError

    def save_keras_model(self, model, path, tmp_path):
        """Save the given model as separate config and weights files."""
        config_path = "%s-config.json" % path
        weights_path = "%s-weights.h5" % path
        weights_tmp = "%s-weights.h5" % tmp_path

        self.write_file(config_path, model.to_json())

        model.save_weights(weights_tmp, overwrite=True)

        # Move the file, potentially across filesystems.
        self.tf_proxy.io.gfile.copy(
            weights_tmp, weights_path, overwrite=True)

        if self.tf_proxy.io.gfile.exists(weights_tmp):
            self.tf_proxy.io.gfile.remove(weights_tmp)

    def save_keras_bundle_model(self, model, path, tmp_path):
        """Save a Keras model bundle (config + weights) to the given path."""
        path += ".keras_bundle.h5"
        model.save(tmp_path)
        self.tf_proxy.io.gfile.copy(tmp_path, path, overwrite=True)
        self.tf_proxy.io.gfile.remove(tmp_path)

    def save_frozenmodel(self, model, path, tmp_path):
        """Save a frozen SavedModel to the given path."""

        # First, create a SavedModel in the tmp directory.
        saved_model_path = tmp_path + "savedmodel"
        saved_model_tmp_path = tmp_path + "savedmodel_tmp"
        self.save_savedmodel(
            model, saved_model_path, saved_model_tmp_path)

        # Extract the output tensor names, which are needed in the freeze_graph
        # call to determine which nodes are actually needed in the final graph.
        ops = self.get_output_ops(model)
        output_tensor_names = ','.join(ops)

        self.freeze_graph(
            saved_model_path, path, output_tensor_names)

    def save_optimized_model(
            self,
            model,
            output_path,
            tmp_path,
            toco_compatible=False):
        """Save an optimized, frozen SavedModel to the given path."""

        # To save an optimized model, we first freeze the model, then apply
        # the optimize_for_inference library.
        frozen_path = tmp_path + "_frozen"
        frozen_tmp_path = tmp_path + "_frozen_tmp"
        self.save_frozenmodel(model, frozen_path, frozen_tmp_path)

        # Parse the GraphDef, and determine the inputs and outputs
        with self.tf_proxy.io.gfile.Open(frozen_path, "rb") as f:
            graph_def = self.tf_proxy.python.GraphDef()
            graph_def.ParseFromString(f.read())
        input_ops = self.get_input_ops(model)
        input_types = self.get_input_types(model)
        output_ops = self.get_output_ops(model)

        transformed_graph_def = self.optimize_graph(
            frozen_path, input_ops, output_ops, input_types, toco_compatible)

        self.write_graph(
            transformed_graph_def,
            os.path.join(output_path, "optimized_graph.pb"),
            as_text=False)

    @abstractmethod
    def save_model(self, model, path, export_type="keras", tmp_path="/tmp/"):
        """Save the provided model to the given path.

        Args:
            model(Model): the Keras model to be saved.
            path (str): the directory in which to write the model.
            export_type (str, optional): Defaults to "keras". What format
                of model to export:

                # Tensorflow 1.x/2.x
                "keras" - Save as separate config (JSON) and weights (HDF5)
                    files.
                "keras_bundle" - Saved in Keras's native format (HDF5), via
                    save_model()

                # Currently only supported in Tensorflow 1.x
                "tf" - Saved in tensorflow's SavedModel format. See:
                    https://www.tensorflow.org/alpha/guide/saved_model
                "tf_frozen" - A SavedModel, where the weights are stored
                    in the model file itself, rather than a variables
                    directory. See:
                    https://www.tensorflow.org/guide/extend/model_files
                "tf_optimized" - A frozen SavedModel, which has
                    additionally been transformed via tensorflow's graph
                    transform library to remove training-specific nodes
                    and operations.  See:
                    https://github.com/tensorflow/tensorflow/tree/master/tensorflow/tools/graph_transforms
                "tf_lite" - A TF Lite model.
            tmp_path (str, optional): directory in which to store temporary
                files.
        """

    def write_file(self, path, contents):
        with self.tf_proxy.io.gfile.Open(path, 'w') as output:
            output.write(contents)

    def read_file(self, path, mode='r'):
        with self.tf_proxy.io.gfile.Open(path, mode) as i:
            return i.read()

    def create_directory(self, path, remove_existing=False):
        # Create the directory if it doesn't exist.
        if not self.tf_proxy.io.gfile.exists(path):
            self.tf_proxy.io.gfile.makedirs(path)

        # If it does exist, and remove_existing it specified,
        # the directory will be removed and recreated.
        elif remove_existing:
            self.tf_proxy.io.gfile.rmtree(path)
            self.tf_proxy.io.gfile.makedirs(path)

    def optimize_graph(
            self,
            frozen_model_path,
            input_ops,
            output_ops,
            input_types,
            toco_compatible):
        # Implemented by subclasses
        raise NotImplementedError

max_stars_repo_name: atinsood/keras-tuner

max_issues_repo_name: atinsood/keras-tuner

max_forks_repo_name: atinsood/keras-tuner


text:
from __future__ import division, print_function
import sys

import numpy as np
import lasagne
import theano
import theano.tensor as T

import learn_kernel
import generate

try:
    import progressbar as pb
    have_pb = True
except ImportError:
    have_pb = False

def load_network(results_file, n_test=None):
    with np.load(results_file) as d:
        args = d['args'][()]
        params = d['params'] if 'params' in d else d['net_params']
        sigma_val = d['sigma'][()]

    def gen_data(n=None, dtype=learn_kernel.floatX):
        return generate.generate_data(args, n or args.n_test, dtype=dtype)

    if getattr(args, 'net_code', False):
        learn_kernel.register_custom_net(args.net_code)

    # make the representation network; don't bother calling make_network since
    # it does a bunch of other things
    dim = gen_data()[0].shape[1]  # could be smarter about this
    input_p = T.matrix('input_p')
    input_q = T.matrix('input_q')
    in_p = lasagne.layers.InputLayer(shape=(None, dim), input_var=input_p)
    in_q = lasagne.layers.InputLayer(shape=(None, dim), input_var=input_q)
    net_p, net_q, reg = learn_kernel.net_versions[args.net_version](in_p, in_q)
    rep_p = lasagne.layers.get_output(net_p)

    print("Compiling...", file=sys.stderr, end='')
    get_rep = theano.function([input_p], rep_p)
    print("done.", file=sys.stderr)

    if getattr(args, 'opt_sigma', False):
        params = params[:-1]
    lasagne.layers.set_all_param_values(net_p, params)

    return get_rep, gen_data, sigma_val, args.linear_kernel

def eval_network(get_rep, gen_data, n_reps=10, null_samples=1000, n_test=None,
                 sigma=1, linear_kernel=False, hotelling=True):
    t = range(n_reps)
    if have_pb:
        t = pb.ProgressBar()(t)
    for i in t:
        X, Y = gen_data(n=n_test, dtype=np.float32)
        p_val, _, _ = learn_kernel.eval_rep(
            get_rep, X, Y, linear_kernel=linear_kernel, sigma=sigma,
            hotelling=hotelling, null_samples=null_samples)
        yield p_val

def main():
    import argparse
    parser = argparse.ArgumentParser(description='''
        Evaluate a learned representation (from learn_kernel.py) on
        repeatedly-generated test data; print out the p-value of the test on
        each set we try.'''.strip())
    parser.add_argument('filename')
    parser.add_argument('--n-reps', type=int, default=10,
                        help="How many test sets to evaluate on (default "
                             "%(default)s).")
    parser.add_argument('--null-samples', type=int, default=1000,
                        help="How many null samples to take (default "
                             "%(default)s).")
    parser.add_argument('--n-test', type=int, default=None,
                        help="How big of a test set to use.")

    g = parser.add_mutually_exclusive_group()
    g.add_argument('--hotelling', action='store_true', default=True,
                   help="If it's a linear kernel, use the Hotelling test "
                        "(default behavior).")
    g.add_argument('--permutation', action='store_false', dest='hotelling',
                   help="Do the permutation test even if it's a linear kernel.")
    args = parser.parse_args()

    get_rep, gen_data, sigma, linear_kernel = load_network(args.filename)
    p_vals = eval_network(
            get_rep, gen_data, n_reps=args.n_reps,
            n_test=args.n_test, null_samples=args.null_samples,
            sigma=sigma, linear_kernel=linear_kernel, hotelling=args.hotelling)
    for p in p_vals:
        if 0 <= p < 1:
            print('{:.3f}'.format(p)[1:])
        else:
            print('{:.2f}'.format(p))
    print()

if __name__ == '__main__':
    main()

max_stars_repo_name: Nirngen/opt-mmd

max_issues_repo_name: Nirngen/opt-mmd

max_forks_repo_name: Nirngen/opt-mmd

reason: import numpy,import theano