text:
## Copyright (C) 2018 Michael Pilosov

import numpy as np
import scipy.stats as sstats
from sklearn.neighbors import KernelDensity

r"""
This module defines supported distributions and associated utility functions.
They are as follows:
    :class:`cbayes.distributions.gkde` (needs test)
    :class:`cbayes.distributions.parametric_dist` (needs development)
    :method:`cbayes.distributions.supported_distributions` (tested)
    :method:`cbayes.distributions.assign_dist` (tested)
"""

def supported_distributions(distname=None):
    r"""
    TODO flesh out description.
    currently supports 'normal' and 'uniform'

    :param string distname: Name of a supported distributions.
    If None, returns dictionary of accepted keys.

    rtype: :class:`scipy.stats._distn_infrastructure`
    :returns: scipy distribution object

    rtype: :dict:
    :returns: dictionary with supported types.
    """
    #
    # both take kwags `loc` and `scale` of type `numpy.ndarray` or `list`
    # method `sample_set.set_dist` just creates a handle for the chosen distribution. The longer of
    # `loc` and `scale` is then inferred to be the dimension, which is written to sample_set.dim

    #: DICTIONARY OF SUPPORTED DISTRIBUTIONS:
    D = {
        "normal": sstats.norm,
        "uniform": sstats.uniform,
        "chi2": sstats.chi2,
        "beta": sstats.beta,
        "gamma": sstats.gamma,
    }

    # The following overloads supported keys into our dictionary of distributions.
    if distname is not None:
        if distname.lower() in ["gaussian", "gauss", "normal", "norm", "n"]:
            distname = "normal"
        elif distname.lower() in ["uniform", "uni", "u"]:
            distname = "uniform"
        elif distname.lower() in ["chi2", "c2", "chisquared", "chi_squared"]:
            distname = "chi2"
        elif distname.lower() in ["gamma", "g", "gam", "gm"]:
            distname = "gamma"
        elif distname.lower() in ["beta", "bt", "b"]:
            distname = "beta"
        try:
            return D[distname]
        except KeyError:
            print(
                "Please specify a supported distribution. Type `?supported_distributions`"
            )
    else:  # if d is unspecified, simply return the dictionary.
        return D

def assign_dist(distribution, **kwds):
    r"""
    TODO clean up description of how this is overloaded.
    If a string is passed, it will be matched against the options for `supported_distributions`
    attach the scipy.stats._continuous_distns class to our sample set object

    rtype: :class:`scipy.stats._distn_infrastructure`
    :returns: scipy distribution object
    """
    if type(distribution) is str:
        distribution = supported_distributions(distribution)
    if kwds is not None:
        return distribution(**kwds)
    else:
        return distribution

class gkde(object):
    r"""

    Custom wrapper around `scipy.stats.gaussian_kde` to conform
    to our prefered size indexing of (num, dim).

    """

    def __init__(self, data):
        self.kde_object = sstats.gaussian_kde(data.transpose())
        #: This is the primary difference
        self.d = self.kde_object.d
        self.n = self.kde_object.n

    def rvs(self, size=1):
        r"""
        Generates random variables from a kde object. Wrapper function for
        `scipy.stats.gaussian_kde.resample`.

        :param int size: number of random samples to generate
        :param tuple size: number of samples is taken to be the first argument
        """
        if type(size) is tuple:
            size = size[0]
        return self.kde_object.resample(size).transpose()
        # TODO write a test that makes sure this returns the correct shape

    def pdf(self, eval_points):
        r"""
        Generates random variables from a kde object. Wrapper function for
        `scipy.stats.gaussian_kde.pdf`.

        :param eval_points: points on which to evaluate the density.
        :type eval_points: :class:`numpy.ndarray` of shape (num, dim)
        """

        #: TODO write a test that makes sure this returns the correct shape
        try:
            num_samples = eval_points.shape[0]
        except AttributeError:
            num_samples = 1
        except IndexError:
            num_samples = 1
        p = self.kde_object.pdf(eval_points.transpose())
        return p

class skde(object):
    r"""

    Custom wrapper around `sklearn.neighbors.kde.KernelDensity` to conform
    to our prefered syntax calling (following scipy conventions)

    """

    def __init__(self, data, mirror=False, **kwds):
        self.mirror = mirror
        if kwds is None:
            if self.mirror:
                self.kde_object = KernelDensity(kernel="gaussian").fit(
                    np.vstack([-data, data]).reshape(-1, 1)
                )
            else:
                self.kde_object = KernelDensity(kernel="gaussian").fit(data)
        else:
            if self.mirror:
                self.kde_object = KernelDensity(**kwds).fit(
                    np.vstack([-data.reshape(-1, 1), data.reshape(-1, 1)])
                )
            else:
                self.kde_object = KernelDensity(**kwds).fit(data.reshape(-1, 1))
        try:
            self.d = data.shape[1]
        except IndexError:
            self.d = 1
        self.n = data.shape[0]

    def rvs(self, size=1):
        r"""
        Generates random variables from a kde object. Wrapper function for
        `sklearn.neighbors.kde.KernelDensity.sample`.

        :param int size: number of random samples to generate
        :param tuple size: number of samples is taken to be the first argument
        """
        if type(size) is tuple:
            size = size[0]

        if self.mirror:  # have to generate twice as many samples
            num_samps = 0
            samps = []
            while num_samps < size:
                samp_proposal = self.kde_object.sample()
                if samp_proposal > 0:
                    samps.append(samp_proposal)
                    num_samps += 1
            samps = np.array(samps).reshape(size, self.d)
        else:
            samps = self.kde_object.sample(size)

        return samps
        # TODO write a test that makes sure this returns the correct shape

    def pdf(self, eval_points):
        r"""
        Generates random variables from a kde object. Wrapper function for
        `sklearn.neighbors.kde.KernelDensity.score_samples`.

        :param eval_points: points on which to evaluate the density.
        :type eval_points: :class:`numpy.ndarray` of shape (num, dim)
        """

        #: TODO write a test that makes sure this returns the correct shape
        num_samples = eval_points.shape[0]
        if self.mirror:
            p = 2 * np.exp(self.kde_object.score_samples(eval_points)).reshape(
                num_samples
            )
        else:
            try:
                p = np.exp(self.kde_object.score_samples(eval_points))
            except ValueError:
                p = np.exp(self.kde_object.score_samples(eval_points.reshape(-1, 1)))
        return p

class parametric_dist(object):
    r"""

    TODO: add description.
    TODO: add actual math. this is supposed to mimick scipy.stats,
        except generalized to arbitrary mixtures, using familiar syntax
        that hides the complexity.

    """

    def __init__(self, dim=1):
        self.dim = dim  # this mimicks the scipy.stats.multivariate attribute
        self.distributions = {str(d): None for d in range(dim)}

    def rvs(self, size=None, seed=5397):
        r"""
        TODO: Add this.
        """
        D = self.distributions
        if size is None:
            size = len(D.keys())

        if type(size) is tuple:
            assert size[1] == len(D)  # make sure the dimensions are correct
            n = size[0]
        else:
            n = size

        for dist in D.keys():
            try:
                assert D[dist] is not None
            except AssertionError:
                raise (
                    ValueError(
                        """
                You are missing a distributionin key:%s, please use `self.set_dist`"""
                        % dist
                    )
                )
        output = np.concatenate(
            [
                D[dist]
                .rvs(size=(n, 1), random_state=(1 + seed + int(dist)))
                .reshape(-1, 1)
                for dist in D.keys()
            ],
            axis=1,
        )
        return output

    def pdf(self, eval_points):
        try:
            size = eval_points.shape
        except AttributeError:
            size = np.array([1, 1])
        D = self.distributions
        try:
            dim = size[1]
        except IndexError:
            dim = 1
            if len(D) != dim:
                raise (
                    IndexError(
                        "Could not infer dimensions. `eval_points` has the wrong shape."
                    )
                )
        try:
            n = size[0]
        except IndexError:
            n = 1
        try:
            eval_points = eval_points.reshape(n, dim)
        except AttributeError:  # trying to catch singletons without an error.
            pass
        except IndexError:
            pass
        output = np.ones(n)
        for ind, dist in enumerate(D.keys()):
            try:
                assert D[dist] is not None
            except AssertionError:
                raise (
                    ValueError(
                        """
                You are missing a distribution in key:%s, please use `self.set_dist`"""
                        % dist
                    )
                )
            try:
                output *= D[dist].pdf(eval_points[:, ind])
            except TypeError:
                output *= D[dist].pdf(eval_points)
        return output

    def ppf(self, eval_points):
        try:
            size = eval_points.shape
        except AttributeError:
            size = np.array([1, 1])
        D = self.distributions
        try:
            dim = size[1]
        except IndexError:
            dim = 1
            if len(D) != dim:
                raise (
                    IndexError(
                        "Could not infer dimensions. `eval_points` has the wrong shape."
                    )
                )
        try:
            n = size[0]
        except IndexError:
            n = 1
        try:
            eval_points = eval_points.reshape(n, dim)
        except AttributeError:  # trying to catch singletons without an error.
            pass
        except IndexError:
            pass
        output = np.ones(n)
        for ind, dist in enumerate(D.keys()):
            try:
                assert D[dist] is not None
            except AssertionError:
                raise (
                    ValueError(
                        """
                You are missing a distribution in key:%s, please use `self.set_dist`"""
                        % dist
                    )
                )
            try:
                output *= D[dist].ppf(eval_points[:, ind])
            except TypeError:
                output *= D[dist].ppf(eval_points)
        return output

    def cdf(self, eval_points):
        try:
            size = eval_points.shape
        except AttributeError:
            size = np.array([1, 1])
        D = self.distributions
        try:
            dim = size[1]
        except IndexError:
            dim = 1
            if len(D) != dim:
                raise (
                    IndexError(
                        "Could not infer dimensions. `eval_points` has the wrong shape."
                    )
                )
        try:
            n = size[0]
        except IndexError:
            n = 1
        try:
            eval_points = eval_points.reshape(n, dim)
        except AttributeError:  # trying to catch singletons without an error.
            pass
        except IndexError:
            pass
        output = np.ones(n)
        for ind, dist in enumerate(D.keys()):
            try:
                assert D[dist] is not None
            except AssertionError:
                raise (
                    ValueError(
                        """
                You are missing a distribution in key:%s, please use `self.set_dist`"""
                        % dist
                    )
                )
            try:
                output *= D[dist].cdf(eval_points[:, ind])
            except TypeError:
                output *= D[dist].cdf(eval_points)
        return output

    def evaluate(self, eval_points):
        return self.pdf(eval_points)

    def fit(self, dim):
        pass

    def mean(self):
        D = self.distributions

        for dist in D.keys():
            try:
                assert D[dist] is not None
            except AssertionError:
                raise (
                    ValueError(
                        """
                You are missing a distribution in key:%s, please use `self.set_dist`"""
                        % dist
                    )
                )

        return [D[dist].mean() for dist in D.keys()]

    def std(self):
        D = self.distributions

        for dist in D.keys():
            try:
                assert D[dist] is not None
            except AssertionError:
                raise (
                    ValueError(
                        """
                You are missing a distribution in key:%s, please use `self.set_dist`"""
                        % dist
                    )
                )

        return [D[dist].std() for dist in D.keys()]

    def assign_dist(self, dist="beta", kwds=None, dim=0):
        D = self.distributions
        if kwds is not None:
            D[str(dim)] = assign_dist(dist, **kwds)
        else:
            D[str(dim)] = assign_dist(dist)
        pass

    def set_dist(self, dist="beta", kwds=None, dim=0):
        D = self.distributions
        if kwds is not None:
            D[str(dim)] = assign_dist(dist, **kwds)
        elif dist == "beta" and kwds is None:
            D[str(dim)] = assign_dist(dist, a=1, b=1)
        else:
            D[str(dim)] = assign_dist(dist)
        pass

    def args(self):
        r"""
        TODO: Add this.
        """
        pass

max_stars_repo_name: mpilosov/ConsistentBayes

max_issues_repo_name: mpilosov/ConsistentBayes

max_forks_repo_name: mpilosov/ConsistentBayes

reason: import numpy,import scipy


text:
import sys
sys.path.append('../')
import matplotlib
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
import numpy as np
import seaborn as sns
from mpl_toolkits.axes_grid1 import make_axes_locatable

plt.rcParams["figure.figsize"] = [16, 9]
sns.set(font_scale=3.0)
sns.set_style(style='white')
sns.set_style(style='white')

from network import train_network, run_network_recall_limit

legends_instead_of_post = True
add_max_and_min_markers = True
captions = True
pre_rule = False
invert = True

N = 10
inter_sequence_time = 1.000
dt = 0.001
sequence = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
sequences = [sequence]

if pre_rule:
    tau_z_post_general = 0.005
    tau_w_general = 0.100
    training_time_general = 0.500
    tau_z_general = 0.050
    max_w_general = 20.0
    min_w_general = -10.0
    epochs_general = 4

else:
    tau_z_post_general = 0.005
    tau_w_general = 0.050
    training_time_general = 0.500
    tau_z_general = 0.100
    max_w_general = 5.0
    min_w_general = -10.0
    epochs_general = 4

fig = plt.figure(figsize=(16, 12))
ax1 = fig.add_subplot(321)
ax2 = fig.add_subplot(322)
ax3 = fig.add_subplot(323)
if not legends_instead_of_post:
    ax4 = fig.add_subplot(324)
ax5 = fig.add_subplot(325)
ax6 = fig.add_subplot(326)

if captions:
    size = 30
    fig.text(0.05, 0.97, 'a)', size=size)
    fig.text(0.50, 0.97, 'b)', size=size)
    fig.text(0.05, 0.65, 'c)', size=size)
    fig.text(0.05, 0.34, 'd)', size=size)
    fig.text(0.50, 0.32, 'f)', size=size)

plt.tight_layout()

# Some general parameters
epochs = epochs_general

pattern = 3
pattern_from = 2

# Training time
training_times_vector = np.arange(0.050, 1.050, 0.050)
w_self = []
w_transition = []
w_inhition = []

tau_z = tau_z_general
tau_z_post = tau_z_post_general
tau_w = tau_w_general
max_w = max_w_general
min_w = min_w_general
epocs = epochs_general

training_time = training_time_general

for training_time in training_times_vector:

    dic = train_network(N, dt, training_time, inter_sequence_time, sequences, tau_z,
                        tau_z_post, tau_w, epochs=epochs, max_w=max_w, min_w=min_w, save_w_history=False, pre_rule=pre_rule)

    w = dic['w']
    w_self.append(w[pattern, pattern])
    w_transition.append(w[pattern, pattern_from])
    w_inhition.append(w[pattern_from, pattern])

if invert:
    w_inhition = [-w for w in w_inhition]
ax1.plot(training_times_vector, w_transition, 'o:', color='blue', linewidth=4, markersize=13, label=r'$Exc_T$')
ax1.plot(training_times_vector, w_inhition, '^-', color='red', linewidth=6, markersize=18, label=r'$Inh$')
ax1.plot(training_times_vector, w_self, '*--', color='green', linewidth=2, markersize=15, label=r'$Exc_{self}$')

if add_max_and_min_markers:
    ax1.plot(training_times_vector, np.ones(len(w_self)) * max_w, '--', color='gray', label=r'$w_{max}$')
    ax1.plot(training_times_vector, -np.ones(len(w_self)) * min_w, ls=':', color='gray', label=r'$w_{min}$' )

ax1.axhline(0, ls='--', color='black', label='0')
ax1.axvline(training_time_general, ls='-', color='black')

ax1.set_xlabel('training time')
ax1.set_ylabel(r'$w$')
# ax1.legend(loc=4)

# Tau w
tau_w_vector = np.arange(0.010, 0.220, 0.010)
w_self = []
w_transition = []
w_inhition = []

tau_z = tau_z_general
tau_z_post = tau_z_post_general
tau_w = tau_w_general
max_w = max_w_general
min_w = min_w_general
epochs = epochs_general

for tau_w in tau_w_vector:

    dic = train_network(N, dt, training_time, inter_sequence_time, sequences, tau_z,
                        tau_z_post, tau_w, epochs=5, max_w=max_w, min_w=min_w, save_w_history=False, pre_rule=pre_rule)

    w = dic['w']
    w_self.append(w[pattern, pattern])
    w_transition.append(w[pattern, pattern_from])
    w_inhition.append(w[pattern_from, pattern])

if invert:
    w_inhition = [-w for w in w_inhition]
ax2.plot(tau_w_vector, w_transition, 'o:', color='blue', linewidth=4, markersize=13, label='Exc')
ax2.plot(tau_w_vector, w_inhition, '^-', color='red', linewidth=6, markersize=18, label='Inh')
ax2.plot(tau_w_vector, w_self, '*--', color='green', linewidth=2, markersize=15, label='Self')

if add_max_and_min_markers:
    ax2.plot(tau_w_vector, np.ones(len(w_self)) * max_w, '--', color='gray', label=r'$w_{max}$')
    ax2.plot(tau_w_vector, -np.ones(len(w_self)) * min_w, ':', color='gray', label=r'$w_{min}$')

ax2.axhline(0, ls='--', color='black', label='0')
ax2.axvline(tau_w_general, ls='-', color='black')

ax2.set_xlabel(r'$\tau_w$')
ax2.set_ylabel(r'$w$')
# ax2.legend(loc=4)

# tau_z

tau_z_vector = np.arange(0.050, 0.550, 0.050)
w_self = []
w_transition = []
w_inhition = []
pattern = 3
pattern_from = 2

tau_z = tau_z_general
tau_z_post = tau_z_post_general
tau_w = tau_w_general
max_w = max_w_general
min_w = min_w_general
epochs = epochs_general

for tau_z in tau_z_vector:

    dic = train_network(N, dt, training_time, inter_sequence_time, sequences, tau_z,
                        tau_z_post, tau_w, epochs=5, max_w=max_w, min_w=min_w, save_w_history=False, pre_rule=pre_rule)

    w = dic['w']
    w_self.append(w[pattern, pattern])
    w_transition.append(w[pattern, pattern_from])
    w_inhition.append(w[pattern_from, pattern])

if invert:
    w_inhition = [-w for w in w_inhition]
ax3.plot(tau_z_vector, w_transition, 'o:', color='blue', linewidth=4, markersize=13, label='Exc')
ax3.plot(tau_z_vector, w_inhition, '^-', color='red', linewidth=6, markersize=18, label='Inh')
ax3.plot(tau_z_vector, w_self, '*--', color='green', linewidth=2, markersize=15, label='Self')

if add_max_and_min_markers:
    ax3.plot(tau_z_vector, np.ones(len(w_self)) * max_w, '--', color='gray', label=r'$w_{max}$')
    ax3.plot(tau_z_vector, -np.ones(len(w_self)) * min_w, ls=':', color='gray', label=r'$w_{min}$')

ax3.axhline(0, ls='--', color='black', label='0')
ax3.axvline(tau_z_general, ls='-', color='black')

ax3.set_xlabel(r'$\tau_z$')
ax3.set_ylabel(r'$w$')

# tau_z_ post
if not legends_instead_of_post:
    tau_z_vector = np.arange(0.005, 0.105, 0.005)
    w_self = []
    w_transition = []
    w_inhition = []
    pattern = 3
    pattern_from = 2

    tau_z = tau_z_general
    tau_z_post = tau_z_post_general
    tau_w = tau_w_general
    max_w = max_w_general
    min_w = min_w_general
    epochs = epochs_general

    for tau_z_post in tau_z_vector:

        dic = train_network(N, dt, training_time, inter_sequence_time, sequences, tau_z,
                            tau_z_post, tau_w, epochs=5, max_w=max_w, min_w=min_w, save_w_history=False, pre_rule=pre_rule)

        w = dic['w']
        w_self.append(w[pattern, pattern])
        w_transition.append(w[pattern, pattern_from])
        w_inhition.append(w[pattern_from, pattern])

    if invert:
        w_inhition = [-w for w in w_inhition]
    ax4.plot(tau_z_vector, w_transition, 'o-', color='blue', linewidth=4, markersize=13, label='Exc')
    ax4.plot(tau_z_vector, w_inhition, '^-', color='red', linewidth=6, markersize=18, label='Inh')
    ax4.plot(tau_z_vector, w_self, '*-', color='green', linewidth=2, markersize=15, label='Self')

    ax4.axhline(0, ls='--', color='black', label='0')

    ax4.set_xlabel(r'$\tau_{z_{post}}$')
    # ax4.set_ylabel('weight value')
    #ax4.legend(loc=4)

# Max  w
max_w_vector = np.arange(1, 20.0, 2.0)
w_self = []
w_transition = []
w_inhition = []
pattern = 3
pattern_from = 2

tau_z = tau_z_general
tau_z_post = tau_z_post_general
tau_w = tau_w_general
max_w = max_w_general
min_w = min_w_general
epochs = epochs_general

for max_w in max_w_vector:

    dic = train_network(N, dt, training_time, inter_sequence_time, sequences, tau_z,
                        tau_z_post, tau_w, epochs=5, max_w=max_w, min_w=min_w, save_w_history=False, pre_rule=pre_rule)

    w = dic['w']
    w_self.append(w[pattern, pattern])
    w_transition.append(w[pattern, pattern_from])
    w_inhition.append(w[pattern_from, pattern])

if invert:
    w_inhibition = [-w for w in w_inhition]

ax5.plot(max_w_vector, w_transition, 'o-', color='blue', linewidth=4, markersize=13, label='Exc')
ax5.plot(max_w_vector, w_inhibition, '^-', color='red', linewidth=6, markersize=18, label='Inh')
ax5.plot(max_w_vector, w_self, '*-', color='green', linewidth=2, markersize=15, label='Self')

if add_max_and_min_markers:
    ax5.plot(max_w_vector, max_w_vector, '--', color='gray', label=r'$w_{max}$')
    ax5.plot(max_w_vector, -np.ones(len(w_self)) * min_w, ls=':', color='gray', label=r'$w_{min}$')

ax5.axhline(0, ls='--', color='black', label='0')
ax5.axvline(max_w_general, ls='-', color='black')

ax5.set_xlabel(r'$w_{max}$')
ax5.set_ylabel(r'$w$')

# min w
min_w_vector = -np.arange(1, 20.0 , 2.0)
w_self = []
w_transition = []
w_inhition = []
pattern = 3
pattern_from = 2

tau_z = tau_z_general
tau_z_post = tau_z_post_general
tau_w = tau_w_general
max_w = max_w_general
min_w = min_w_general
epochs = epochs_general

for min_w in min_w_vector:

    dic = train_network(N, dt, training_time, inter_sequence_time, sequences, tau_z,
                        tau_z_post, tau_w, epochs=5, max_w=max_w, min_w=min_w, save_w_history=False, pre_rule=pre_rule)

    w = dic['w']
    w_self.append(w[pattern, pattern])
    w_transition.append(w[pattern, pattern_from])
    w_inhition.append(w[pattern_from, pattern])

if invert:
    w_inhibition = [-w for w in w_inhition]
ax6.plot(min_w_vector, w_transition, 'o-', color='blue', linewidth=4, markersize=13, label='Exc')
ax6.plot(min_w_vector, w_inhibition, '^-', color='red', linewidth=6, markersize=18, label='Inh')
ax6.plot(min_w_vector, w_self, '*-', color='green', linewidth=2, markersize=15, label='Self')

if add_max_and_min_markers:
    ax6.plot(min_w_vector, np.ones(len(w_self)) * max_w, '--', color='gray', label=r'$w_{max}$')
    ax6.plot(min_w_vector, -min_w_vector, ':', color='gray', label=r'$w_{min}$')

ax6.axhline(0, ls='--', color='black', label=0)
ax6.axvline(min_w_general, ls='-', color='black')

ax6.set_xlabel(r'$w_{min}$')
ax6.set_ylabel(r'$w$')

# The legends
if legends_instead_of_post:
    handles, labels = ax1.get_legend_handles_labels()
    fig.legend(handles=handles, labels=labels, loc=(0.58, 0.42), fancybox=True, frameon=True, fontsize=32, ncol=2)
# Save the figure
name = './plot_producers/training_rule_quantities_pre_rule_' + str(pre_rule) + '.eps'
fig.savefig(name, frameon=False, dpi=110, bbox_inches='tight')
plt.close()

ext: py

reason: import numpy


text:
import os
import os.path as osp
import re

import numpy as np
from numpy import array, int32
from scipy.io import loadmat

from .base import BaseDataset

class MVN_pretrain(BaseDataset):
    def __init__(self, root, transforms, split):
        self.name = "MVN_pretrain"
        self.img_prefix = osp.join(root, "frames")
        self.anno_preloaded = 'data/anno_loaded/{}.txt'.format(split)
        super(MVN_pretrain, self).__init__(root, transforms, split)

    def _load_queries(self):
        if os.path.exists(self.anno_preloaded):
            print('Loading preloaded one...')
            with open(self.anno_preloaded, 'r') as fin:
                queries = eval(fin.read())
            return queries

        query_info = osp.join(self.root, "query_info.txt")
        with open(query_info, "rb") as f:
            raw = f.readlines()

        queries = []
        for line in raw:
            linelist = str(line, "utf-8").split(" ")
            pid = int(linelist[0])
            x, y, w, h = (
                float(linelist[1]),
                float(linelist[2]),
                float(linelist[3]),
                float(linelist[4]),
            )
            roi = np.array([x, y, x + w, y + h]).astype(np.int32)
            roi = np.clip(roi, 0, None)  # several coordinates are negative
            img_name = linelist[5].strip() + ".jpg"
            queries.append(
                {
                    "img_name": img_name,
                    "img_path": osp.join(self.img_prefix.replace('frames', 'query_portraits'), img_name),
                    "boxes": roi[np.newaxis, :],
                    "pids": np.array([pid]),
                }
            )
        print('-----------\n-----------\n{} pid:'.format(self.split))
        with open('data/anno_loaded/{}.txt'.format(self.split), 'w') as fout:
            fout.write(str(queries))
        return queries

    def _load_split_img_names(self):
        """
        Load the image names for the specific split.
        """
        assert self.split in ("train", "gallery")
        if self.split == "train":
            imgs = loadmat(osp.join(self.root, "frame_train.mat"))["img_index_train"]
        else:
            imgs = loadmat(osp.join(self.root, "frame_test.mat"))["img_index_test"]
        return [img[0][0] + ".jpg" for img in imgs]

    def _load_annotations(self):
        if self.split == "query":
            return self._load_queries()

        if os.path.exists(self.anno_preloaded):
            print('Loading preloaded one...')
            with open(self.anno_preloaded, 'r') as fin:
                annotations = eval(fin.read())
            pids = []
            for i in annotations:
                pids.append(i['pids'][0])
            np.savetxt('data/anno_loaded/pids_{}.txt'.format(self.split), pids)
            return annotations

        annotations = []
        imgs = self._load_split_img_names()
        print('Loading annotations >>>')
        for idx_img, img_name in enumerate(imgs):
            if idx_img % 1000 == 0:
                print('{}/{},\t'.format(idx_img, len(imgs)), end='')
            anno_path = osp.join(self.root, "annotations", img_name.replace('.jpg', ''))
            anno = loadmat(anno_path)
            box_key = "box_new"
            if box_key not in anno.keys():
                box_key = "anno_file"
            if box_key not in anno.keys():
                box_key = "anno_previous"

            rois = anno[box_key][:, 1:]
            ids = anno[box_key][:, 0]
            rois = np.clip(rois, 0, None)  # several coordinates are negative

            assert len(rois) == len(ids)

            rois[:, 2:] += rois[:, :2]
            ids[ids == -2] = 5555  # assign pid = 5555 for unlabeled people
            annotations.append(
                {
                    "img_name": img_name,
                    "img_path": osp.join(self.img_prefix, img_name),
                    "boxes": rois.astype(np.int32),
                    # FIXME: (training pids) 1, 2,..., 478, 480, 481, 482, 483, 932, 5555
                    "pids": ids.astype(np.int32),
                }
            )
        with open('data/anno_loaded/{}.txt'.format(self.split), 'w') as fout:
            fout.write(str(annotations))
        print('\n<<< End')
        return annotations

reason: import numpy,from numpy,from scipy


text:
"""
Struct that holds abstract_task, its part and handlers.
"""
from typing import Callable, Tuple

import numpy as np
from omtool.core.datamodel.abstract_task import AbstractTask
from omtool.core.datamodel.snapshot import Snapshot

class HandlerTask:
    """
    Struct that holds abstract_task, its part and handlers.
    """

    def __init__(
        self,
        task: AbstractTask,
        part=slice(0, None),
        handlers: list[Callable[[Tuple[np.ndarray, np.ndarray]], None]] = None,
    ):
        if handlers is None:
            handlers = []

        self.task = task
        self.part = part
        self.handlers = handlers

    def run(self, snapshot: Snapshot):
        """
        Launch the task and return its value
        """
        data = self.task.run(snapshot[self.part])

        for handler in self.handlers:
            handler(data)

max_stars_repo_name: Kraysent/OMTool

max_issues_repo_name: Kraysent/OMTool

max_forks_repo_name: Kraysent/OMTool


text:
import numpy as np
import pandas as pd
import pytest

from dptools import add_date_features
from dptools import add_text_features
from dptools import aggregate_data
from dptools import encode_factors

def test_aggregate_data_6():
    data = {'age': [27, np.nan, 30, 25, np.nan],
        'height': [170, 168, 173, 177, 165],
        'gender': ['female', 'male', np.nan, 'male', 'female'],
        'income': ['high', 'medium', 'low', 'low', 'no income']}
    df = pd.DataFrame(data)
    df = aggregate_data(df,
                        group_var = 'gender',
                        num_stats = ['min', 'max'],
                        fac_stats = 'mode')
    assert df.shape[1] == 6

def test_aggregate_data_8():
    data = {'age': [27, np.nan, 30, 25, np.nan],
        'height': [170, 168, 173, 177, 165],
        'gender': ['female', 'male', np.nan, 'male', 'female'],
        'income': ['high', 'medium', 'low', 'low', 'no income']}
    df = pd.DataFrame(data)
    df = aggregate_data(df,
                        group_var = 'gender',
                        num_stats = ['min', 'mean', 'max'],
                        fac_stats = ['mode', 'count'])
    assert df.shape[1] == 9

def test_add_text_features_9():
    data = {'age': [27, np.nan, 30, 25, np.nan],
        'height': [170, 168, 173, 177, 165],
        'gender': ['female', 'male', np.nan, 'male', 'female'],
        'income': ['high', 'medium', 'low', 'low', 'no income']}
    df = pd.DataFrame(data)
    df = add_text_features(df,
                           text_vars = ['income'],
                           tf_idf_feats = 5,
                           drop = True)
    assert df.shape[1] == 9

def test_add_text_features_10():
    data = {'age': [27, np.nan, 30, 25, np.nan],
        'height': [170, 168, 173, 177, 165],
        'gender': ['female', 'male', np.nan, 'male', 'female'],
        'income': ['high', 'medium', 'low', 'low', 'no income']}
    df = pd.DataFrame(data)
    df = add_text_features(df,
                           text_vars = ['income', 'gender'],
                           tf_idf_feats = 2,
                           drop = True)
    assert df.shape[1] == 10

def test_add_date_features_16():
    data = {'age': [27, 30],
        'height': [170, 168],
        'gender': ['female', 'male'],
        'date_of_birth': [np.datetime64('1993-02-10'), np.datetime64('1985-10-17')]}
    df = pd.DataFrame(data)
    df = add_date_features(df, date_vars = 'date_of_birth', time = False)
    assert df.shape[1] == 16

def test_encode_factors_label():
    data = {'age': [27, np.nan, 30, 25, np.nan],
        'height': [170, 168, 173, 177, 165],
        'gender': ['female', 'male', np.nan, 'male', 'female'],
        'income': ['high', 'medium', 'low', 'low', 'no income']}
    df = pd.DataFrame(data)
    df = encode_factors(df, method = 'label')
    assert df.shape[1] == 4

def test_encode_factors_dummy_8():
    data = {'age': [27, np.nan, 30, 25, np.nan],
        'height': [170, 168, 173, 177, 165],
        'gender': ['female', 'male', np.nan, 'male', 'female'],
        'income': ['high', 'medium', 'low', 'low', 'no income']}
    df = pd.DataFrame(data)
    df = encode_factors(df, method = 'dummy')
    assert df.shape[1] == 8

def test_encode_factors_dummy_7():
    data = {'age': [27, np.nan, 30, 25, np.nan],
        'height': [170, 168, 173, 177, 165],
        'gender': ['female', 'male', np.nan, 'male', 'female'],
        'income': ['high', 'medium', 'low', 'low', 'no income']}
    df = pd.DataFrame(data)
    df = encode_factors(df, factors = 'income', method = 'dummy')
    assert df.shape[1] == 7

max_stars_repo_name: kozodoi/dptools

max_issues_repo_name: kozodoi/dptools

max_forks_repo_name: kozodoi/dptools


text:
"""
This is the "update_formulae" module.

It performs a number of weighted updates that are needed for streaming learning, e.g.,

>>> convex_combination(10,20,0.3)
13.0

It also hosts certain decision formulae.
"""

import numpy as np

def convex_combination(a, b, weight):
    """

    :param a: one summand (e.g., partial sum)
    :param b: another summand (e.g., the new datapoint)
    :param weight: the weight of the update (e.g., the inverse of the sample size)
    :return: the updated value (e.g., the new partial sum)

    >>> assert(np.mean([3,7]) == convex_combination(3,7,0.5))
    >>> assert(np.mean([1,1,1,2]) == convex_combination(1,2,0.25))

    """
    return (1-weight) * a + weight * b

def update_effective_sample_size(
    effective_sample_size,
    batch_size,
    forgetting_factor
):
    """

    :param effective_sample_size:
    :param batch_size:
    :param forgetting_factor:
    :return:

    >>> update_effective_sample_size(1.0,1.0,1.0)
    (2.0, 1.0)

    """
    updated_sample_size = (
        effective_sample_size * forgetting_factor + batch_size
    )
    weight = 1 - (
        (effective_sample_size*1.0 - batch_size) /
        (effective_sample_size*1.0)
    )
    return updated_sample_size, weight

def rolling_window_update(old, new, w=100):
    """

    :param old: Old data
    :param new: New data
    :param w: Controls the size of the rolling window
    :return: The w most recent datapoints from the concatenation of old and new

    >>> rolling_window_update(old=[1,2,3], new=[4,5,6,7],w=5)
    array([3,4,5,6,7])

    """
    out = np.concatenate((old, new))
    if len(out) > w:
        out = out[(len(out)-w):]
    return out

def decision_rule(score, threshold=0.99, two_sided=True):
    """

    :param score: a score, assumed normalised (between 0 and 1) representing anomalousness
    :param threshold: a user-specified threshold above which an alert should be raised
    :param two_sided: if True, we flag anomalies that are either smaller than 1-threshold or larger than threhsold
    :return: a boolean flag

    >>> decision_rule(score=0.9)
    False
    >>> decision_rule(score=0.95, threshold=0.9)
    True
    >>> decision_rule(score=0.0001, threshold=0.99)
    True
    >>> decision_rule(score=0.001, two_sided=False)
    False
    """
    if two_sided:
        ans = np.logical_or(score < 1-threshold, score > threshold)
    else:
        ans = score > threshold
    return ans


text:
import streamlit as st
# To make things easier later, we're also importing numpy and pandas for
# working with sample data.
import numpy as np
import pandas as pd

df = pd.DataFrame({
  'first column': [1, 2, 3, 4],
  'second column': [10, 20, 30, 40]
})

if st.checkbox('Show dataframe'):
    chart_data = pd.DataFrame(
       np.random.randn(20, 3),
       columns=['a', 'b', 'c'])

    st.line_chart(chart_data)

# option = st.selectbox(
#     'Which number do you like best?',
#      df['first column'])
# 'You selected: ', option

option = st.sidebar.selectbox(
    'Which number do you like best?',
     df['first column'])
'You selected:', option

max_stars_repo_name: YA-androidapp/Tutorial-Streamit

max_issues_repo_name: YA-androidapp/Tutorial-Streamit

max_forks_repo_name: YA-androidapp/Tutorial-Streamit


text:
"""
高分辨率Patch特征抽取模块
"""
import cv2
import torch
import torch.nn as nn
import numpy as np

from .base import convnxn
from .base import MRFBlock
from .transformer import Encoder, VectorsEmbeddingModule
from .mae import MAE

from lib.models.utils.models import _tranpose_and_gather_feat

# 简单的使用CNN作为特征抽取器
class FineFeatExtractor(nn.Module):
    """
    特征抽取网络 从高清Patch中抽取细粒度特征 用于增强ReID部分
    主要要考虑到网络的简洁性
    """
    def __init__(self):
        super(FineFeatExtractor, self).__init__()

        self.first_conv = convnxn(3, 32, kernel=7, stride=1, padding=3)

        self.first_stage = nn.Sequential(
            MRFBlock(32, 1),
        )

        self.second_stage = nn.Sequential(
            convnxn(32, 64, kernel=3, stride=2, padding=1),
            MRFBlock(64, 2),
        )

        self.last_stage = nn.Sequential(
            convnxn(64, 128, kernel=3, stride=2, padding=1),
            MRFBlock(128, 3),
        )

    def forward(self, x):
        """
        x: 输入图像特征
        """
        x = self.first_conv(x)            # 32 * 112 * 112
        x = self.first_stage(x)           # 32 * 112 * 112
        x = self.second_stage(x)          # 64 * 56 * 56
        x = self.last_stage(x)            # 128 * 28 * 28

        return x

# 特征细化网络 (使用CNN抽取器的旧版本)
class FineFeatNet(nn.Module):
    """
    利用Transformer结构进行细粒度特征编码
    """
    def __init__(self):
        super(FineFeatNet, self).__init__()

        self.fine_extract = FineFeatExtractor()
        # ========================================================================
        # TODO 根据传参设置而不是硬编码  需要在dataset中设置输出的最大尺寸是112*112
        # ========================================================================
        self.pos_emb = VectorsEmbeddingModule(pool_size=(4, 4, 0), patch_num=49, embedding_dim=128, dropout=0.1)
        self.encoder = Encoder(layers=1, embedding_dim=128, reduced_dim=128, n_head=8, hidden_unit=512, dropout=0.1)

    def forward(self, low_outputs, batch):
        # =====================================================================================================
        # !!! 原始项目代码里需要累计梯度导致输出是一个列表 但实际上并没有用到梯度累计 有时间修改这部分内容 重构一下代码
        # =====================================================================================================
        if self.training:
            for i in range(len(low_outputs)):
                # 训练模式 patch从batch中直接加载
                # batch * k  * reid_dim 其中包括存在对象的也有不存在对象的
                low_output = low_outputs[i]
                id_head = _tranpose_and_gather_feat(low_output['id'], batch['ind'])
                # batch * k * 3 * h * w
                hd_patches = batch['hd_patches']
                # batch * k (0 / 1指示变量)
                selected_mask = batch['reg_mask']
                # reshape
                low_id_vectors = id_head[selected_mask > 0].contiguous()   # .view(-1, reid_dim)
                hd_patches_x = hd_patches[selected_mask > 0].contiguous()  # .view(-1, c, h, w)
                # (batch * total_k) * dim * patch * patch
                # 仅在此处放入显存
                hd_patches_x = hd_patches_x.to(low_id_vectors.device)
                fine_feat = self.fine_extract(hd_patches_x)
                # (batch * total_k) * (patch_nums + 1) * dim
                embeddings = self.pos_emb(fine_feat, low_id_vectors)
                # (batch * total_k) * (patch_nums + 1) * dim
                output = self.encoder(embeddings)

                # 将强化后的特征添加到原来的输出当中
                # (batch * total_k) *  dim
                fine_vectors = output[:, 0]
                low_outputs[i]["fine_id"] = torch.cat([low_id_vectors, fine_vectors], dim = 0)
            return low_outputs
        else:

            # 测试模式 patch需要从low_output里面decode出来
            # 1. 根据header的输出进行decode, 得到经过筛选之后的有效的位置的初始reid特征 (只对置信度大于某个阈值的位置进行计算)
            # 2. 根据解出的bbox得到在原始图像中的patch
            # 3. 利用low_reid和patch进行特征增强 选中的位置用于特征增强 没有选中的位置保持原来的low_reid
            # 4. 返回修改之后的特征 (batch * reid_dim * output_h * output_w)

            img0 = batch['img0']
            thres = batch['thres']
            transforms = batch['transforms']
            patch_size = batch['patch_size']
            # thres = 0.8
            origin_height, origin_width = img0.shape[0], img0.shape[1]
            # print(origin_height, origin_width)
            for i in range(len(low_outputs)):
                low_output = low_outputs[i]
                # reid_feat = low_output['id']
                # hm = low_output['hm']
                # reg = low_output['reg']
                # wh = low_output['wh']

                # print(reid_feat.size())
                # print(hm.size())
                # print(reg.size())
                # print(wh.size())

                hm = low_output['hm']
                # for simplest evaluate
                b, c, h, w = hm.size()
                assert b == 1

                hm = hm.clone().view(b, c, -1).squeeze(0).squeeze(0)
                sigmoid_hm = torch.sigmoid(hm)

                # print("====================================================================")
                selected_mask = (sigmoid_hm >= thres).int().cpu().numpy()

                # b * 2 * h * w
                reg = low_output['reg'].clone().view(b, 2, -1).squeeze(0).cpu().numpy()
                wh = low_output['wh'].clone().view(b, 4, -1).squeeze(0).cpu().numpy()

                # print(selected_mask)
                # print(selected_mask.sum())
                # print(hm.size())
                # print(reg.size())
                # print(wh.size())

                selected_reg = reg[:, selected_mask == 1].transpose()
                selected_wh = wh[:, selected_mask == 1].transpose()
                indices = np.argwhere(selected_mask > 0)
                # print(indices.shape)
                # print(selected_reg.shape)
                # print(selected_wh.shape)

                # main decode process
                pacthes_tensors = []
                for i in range(indices.shape[0]):
                    # print("-------------------- bbox i -----------------------")
                    center_id_pos = indices[i, 0]
                    center_reg = selected_reg[i, :]
                    wh_ltrb = selected_wh[i, :]

                    # print(center_id_pos)
                    # print(center_reg)
                    # print(wh_ltrb)
                    # 对应于输出图中的中心位置
                    center_y = center_id_pos // w
                    center_x = center_id_pos % w

                    center_y += center_reg[0]
                    center_x += center_reg[1]

                    lt_x = max((center_x - wh_ltrb[0]), 0)
                    lt_y = max((center_y - wh_ltrb[1]), 0)
                    rb_x = min((center_x + wh_ltrb[2]), w)
                    rb_y = min((center_y + wh_ltrb[3]), h)

                    lt_x = int(lt_x / w * origin_width)
                    lt_y = int(lt_y / h * origin_height)
                    rb_x = int(rb_x / w * origin_width)
                    rb_y = int(rb_y / h * origin_height)
                    # print(lt_x, lt_y, rb_x, rb_y)

                    crop_img = img0[lt_y:rb_y, lt_x:rb_x].copy()
                    if crop_img.shape[0] == 0 or crop_img.shape[1] == 0:
                        # invalid patch avoiding empty error
                        crop_img = img0[0:10, 0:10].copy()
                    crop_img = cv2.resize(crop_img, (patch_size, patch_size), cv2.INTER_CUBIC)

                    # visual
                    # cv2.imwrite("./crop/crop_%d_.jpg" % (self.num, ), crop_img)
                    # self.num += 1

                    # BGR -> RGB
                    crop_img = np.ascontiguousarray(crop_img[:, :, ::-1])
                    crop_img = transforms(crop_img).unsqueeze(0)
                    pacthes_tensors.append(crop_img)

                if len(pacthes_tensors) > 0:

                    low_reid_feat = low_output['id'].clone()
                    b, reid_dim, h, w = low_reid_feat.size()

                    # (batch * h * w) * reid_dim
                    low_reid_feat = low_reid_feat.permute(0, 2, 3, 1).view(-1, reid_dim)

                    # print(low_reid_feat.size())
                    indices = indices.reshape((-1,))
                    selected_low_reid_vectors = low_reid_feat[indices, :]

                    hr_patches_input = torch.cat(pacthes_tensors, dim=0).to(selected_low_reid_vectors.device)
                    fine_feat = self.fine_extract(hr_patches_input)
                    embeddings = self.pos_emb(fine_feat, selected_low_reid_vectors)
                    fine_vectors = self.encoder(embeddings)[:, 0]

                    # patches * 128
                    for (k, ind) in zip(range(len(indices)), indices):
                        centery = ind // w
                        centerx = ind % w
                        # instead
                        low_output['id'][0, :, centery, centerx] = fine_vectors[k]

            return low_outputs

# 特征细化网络 (使用MAE抽取器的新版本)
class MAEFeatNet(nn.Module):
    def __init__(
        self,
        mae_pretrain_path = None
    ):
        """
        MAE细粒度特征抽取器 参数设置需要和MAE预训练时保持一致
        Args:
            mae_pretrain_path: 预训练权重路径
        """
        # ========================================================================
        # TODO 根据传参设置而不是硬编码  需要在dataset中设置输出的最大尺寸是128*128
        # ========================================================================
        super(MAEFeatNet, self).__init__()
        # 设置MAE抽取器
        self.mae = MAE(
            128,
            16,
            128,
            4,
            2,
            0.75
        )
        if self.training:
            # 利用预训练模型来初始化进行完整训练
            self.mae.load_state_dict(torch.load(mae_pretrain_path))
            print("MAE loading finished.")
        # 去除MAE预训练时的解码器
        del self.mae.decoder

        # 设置ReID任务时的新解码器
        self.pos_embeddings = nn.parameter.Parameter(torch.zeros(size = (16 ** 2 + 1, 128), requires_grad=True))
        self.transformer = Encoder(1, 128, 128, 8, 512, 0.1)

    def forward(self, low_outputs, batch):
        if self.training:

            # 由于代码遗留问题low_outputs实际上是一个里面只有一个字典元素的列表

            for i in range(len(low_outputs)):
                # 训练模式 patch从batch中直接加载
                # batch * k  * reid_dim 其中包括存在对象的也有不存在对象的
                low_output = low_outputs[i]
                id_head = _tranpose_and_gather_feat(low_output['id'], batch['ind'])
                # batch * k * 3 * h * w
                hd_patches = batch['hd_patches']
                # batch * k (0 / 1指示变量)
                selected_mask = batch['reg_mask']
                # reshape
                low_id_vectors = id_head[selected_mask > 0].contiguous()   # .view(-1, reid_dim)
                hd_patches_x = hd_patches[selected_mask > 0].contiguous()  # .view(-1, c, h, w)
                # (batch * total_k) * dim * patch * patch
                # 仅在此处放入显存
                hd_patches_x = hd_patches_x.to(low_id_vectors.device)

                # 使用MAE从高清图像Patch中获取细粒度特征
                fine_feat = self.mae(hd_patches_x, encoder_mode = True)
                # (batch * total_k) * (patch_nums + 1) * dim
                embeddings = torch.cat([low_id_vectors.unsqueeze(1), fine_feat], dim=1) + self.pos_embeddings
                # (batch * total_k) * (patch_nums + 1) * dim
                output = self.transformer(embeddings)

                # 将强化后的特征添加到原来的输出当中
                # (batch * total_k) *  dim
                fine_vectors = output[:, 0]

                # 将原本没有增强过的和已经增强过的同时返回以供下一步处理
                low_outputs[i]["fine_id"] = torch.cat([low_id_vectors, fine_vectors], dim = 0)

            return low_outputs
        else:
            # !!! Note: 所谓的可插拔即low_outputs是否使用下面的增强 如果不增强完全可以不使用整个MAE部分

            # 测试模式 patch需要从low_output里面decode出来
            # 1. 根据header的输出进行decode, 得到经过筛选之后的有效的位置的初始reid特征 (只对置信度大于某个阈值的位置进行计算)
            # 2. 根据解出的bbox得到在原始图像中的patch
            # 3. 利用low_reid和patch进行特征增强 选中的位置用于特征增强 没有选中的位置保持原来的low_reid
            # 4. 返回修改之后的特征 (batch * reid_dim * output_h * output_w)

            img0 = batch['img0']
            thres = batch['thres']
            transforms = batch['transforms']
            patch_size = 128
            # thres = 0.8
            origin_height, origin_width = img0.shape[0], img0.shape[1]
            # print(origin_height, origin_width)
            for i in range(len(low_outputs)):
                low_output = low_outputs[i]
                # reid_feat = low_output['id']
                # hm = low_output['hm']
                # reg = low_output['reg']
                # wh = low_output['wh']

                # print(reid_feat.size())
                # print(hm.size())
                # print(reg.size())
                # print(wh.size())

                hm = low_output['hm']
                # for simplest evaluate
                b, c, h, w = hm.size()
                assert b == 1

                hm = hm.clone().view(b, c, -1).squeeze(0).squeeze(0)
                sigmoid_hm = torch.sigmoid(hm)

                # print("====================================================================")
                selected_mask = (sigmoid_hm >= thres).int().cpu().numpy()

                # b * 2 * h * w
                reg = low_output['reg'].clone().view(b, 2, -1).squeeze(0).cpu().numpy()
                wh = low_output['wh'].clone().view(b, 4, -1).squeeze(0).cpu().numpy()

                # print(selected_mask)
                # print(selected_mask.sum())
                # print(hm.size())
                # print(reg.size())
                # print(wh.size())

                selected_reg = reg[:, selected_mask == 1].transpose()
                selected_wh = wh[:, selected_mask == 1].transpose()
                indices = np.argwhere(selected_mask > 0)
                # print(indices.shape)
                # print(selected_reg.shape)
                # print(selected_wh.shape)

                # main decode process
                pacthes_tensors = []
                for i in range(indices.shape[0]):
                    # print("-------------------- bbox i -----------------------")
                    center_id_pos = indices[i, 0]
                    center_reg = selected_reg[i, :]
                    wh_ltrb = selected_wh[i, :]

                    # print(center_id_pos)
                    # print(center_reg)
                    # print(wh_ltrb)
                    # 对应于输出图中的中心位置
                    center_y = center_id_pos // w
                    center_x = center_id_pos % w

                    center_y += center_reg[0]
                    center_x += center_reg[1]

                    lt_x = max((center_x - wh_ltrb[0]), 0)
                    lt_y = max((center_y - wh_ltrb[1]), 0)
                    rb_x = min((center_x + wh_ltrb[2]), w)
                    rb_y = min((center_y + wh_ltrb[3]), h)

                    lt_x = int(lt_x / w * origin_width)
                    lt_y = int(lt_y / h * origin_height)
                    rb_x = int(rb_x / w * origin_width)
                    rb_y = int(rb_y / h * origin_height)
                    # print(lt_x, lt_y, rb_x, rb_y)

                    crop_img = img0[lt_y:rb_y, lt_x:rb_x].copy()
                    if crop_img.shape[0] == 0 or crop_img.shape[1] == 0:
                        # invalid patch avoiding empty error
                        crop_img = img0[0:10, 0:10].copy()
                    crop_img = cv2.resize(crop_img, (patch_size, patch_size), cv2.INTER_CUBIC)

                    # visual
                    # cv2.imwrite("./crop/crop_%d_.jpg" % (self.num, ), crop_img)
                    # self.num += 1

                    # BGR -> RGB
                    crop_img = np.ascontiguousarray(crop_img[:, :, ::-1])
                    crop_img = transforms(crop_img).unsqueeze(0)
                    pacthes_tensors.append(crop_img)

                if len(pacthes_tensors) > 0:

                    low_reid_feat = low_output['id'].clone()
                    b, reid_dim, h, w = low_reid_feat.size()

                    # (batch * h * w) * reid_dim
                    low_reid_feat = low_reid_feat.permute(0, 2, 3, 1).view(-1, reid_dim)

                    # print(low_reid_feat.size())
                    indices = indices.reshape((-1,))
                    selected_low_reid_vectors = low_reid_feat[indices, :]

                    hr_patches_input = torch.cat(pacthes_tensors, dim=0).to(selected_low_reid_vectors.device)

                    # 使用mae抽取特征
                    fine_feat = self.mae(hr_patches_input, encoder_mode = True)
                    embeddings = torch.cat([selected_low_reid_vectors.unsqueeze(1), fine_feat], dim=1) + self.pos_embeddings
                    fine_vectors = self.transformer(embeddings)[:, 0]

                    # patches * 128
                    for (k, ind) in zip(range(len(indices)), indices):
                        centery = ind // w
                        centerx = ind % w
                        # instead
                        low_output['id'][0, :, centery, centerx] = fine_vectors[k]

            return low_outputs

max_stars_repo_name: luyu-fan/fsoccermot

max_issues_repo_name: luyu-fan/fsoccermot

max_forks_repo_name: luyu-fan/fsoccermot


text:
import cv2
import numpy as np

import model.yolov4.tf as yolo

detector = yolo.YoloV4(
    names_path="./data/darknet/coco.names",
    weights_path="./data/darknet/yolov4.weights",
)

img, boxs = detector.predict(cv2.imread("./data/3.jpg"), detector.classes)

cv2.imshow('img', img)

image_h, image_w, _ = img.shape
for box in boxs:
    coor = np.array(box[:4], dtype=np.int32)
    cv2.imshow("c", img[coor[1]:coor[3], coor[0]:coor[2]])
    # cv2.imshow("a", img[coor[0]:coor[2], coor[1]:coor[3]])
cv2.waitKey(0)
cv2.destroyAllWindows()

max_stars_repo_name: chunibyo-wly/SmartCities

max_issues_repo_name: chunibyo-wly/SmartCities

max_forks_repo_name: chunibyo-wly/SmartCities


text:
from typing import Tuple

import numpy as np
from tensorflow.keras.datasets import boston_housing

def get_dataset() -> Tuple[Tuple[np.ndarray, np.ndarray], Tuple[np.ndarray, np.ndarray]]:
    (x_train, y_train), (x_test, y_test) = boston_housing.load_data()
    x_train = x_train.astype(np.float32)
    y_train = y_train.reshape((-1, 1)).astype(np.float32)
    x_test = x_test.astype(np.float32)
    y_test = y_test.reshape((-1, 1)).astype(np.float32)
    return (x_train, y_train), (x_test, y_test)

def main() -> None:
    (x_train, y_train), (x_test, y_test) = get_dataset()

    print(f"x_train shape: {x_train.shape}")
    print(f"y_train shape: {y_train.shape}")
    print(f"x_test shape: {x_test.shape}")
    print(f"y_test shape: {y_test.shape}")

if __name__ == "__main__":
    main()


text:
import tcc
import json
from .base_lot import Lot
import numpy as np
from . import manage_xyz
from .units import *
import sys
import time

class TCC(Lot):

    @property
    def TC(self):
        return self.options['job_data']['TC']

    @property
    def tcc_options(self):
        return self.options['job_data']['tcc_options']

    @tcc_options.setter
    def tcc_options(self,d):
        self.options['job_data']['tcc_options'] = d
        return

    @property
    def orbfile(self):
        return self.options['job_data']['orbfile']

    @orbfile.setter
    def orbfile(self,value):
        self.options['job_data']['orbfile'] = value

    def __init__(self,options):
        super(TCC,self).__init__(options)
        tcc_options_copy = self.tcc_options.copy()
        tcc_options_copy['atoms'] = self.atoms
        self.tcc_options = tcc_options_copy

    def run(self,coords):
        self.E=[]
        self.grada=[]

        for state in self.states:
            #print("on state %d" % state[1])
            multiplicity=state[0]
            ad_idx=state[1]
            grad_options = self.tcc_options.copy()
            grad_options['runtype'] = 'gradient'
            grad_options['castargetmult'] = multiplicity
            grad_options['castarget'] = ad_idx
            if not self.orbfile:
                grad_options['guess'] = self.orbfile
            print((" orbfile is %s" % self.orbfile))
            #results = self.TC.compute(coords,grad_options)
            job_id = self.TC.submit(coords,grad_options)
            results = self.TC.poll_for_results(job_id)
            while results['message'] == "job not finished":
                results=self.TC.poll_for_results(job_id)
                print((results['message']))
                print("sleeping for 1")
                time.sleep(1)

            print((json.dumps(results, indent=2, sort_keys=True)))
            self.orbfile = results['orbfile']
            self.E.append((multiplicity,ad_idx,results['energy'][ad_idx]))
            self.grada.append((multiplicity,ad_idx,results['gradient']))
        if self.do_coupling==True:
            state1=self.states[0][1]
            state2=self.states[1][1]
            nac_options = self.tcc_options.copy()
            nac_options['runtype'] = 'coupling'
            nac_options['nacstate1'] = 0
            nac_options['nacstate2'] = 1
            results = self.TC.compute(coords,nac_options)
            self.coup = results['coupling']

        self.hasRanForCurrentCoords=True
        return

    def get_energy(self,coords,multiplicity,state):
        if self.hasRanForCurrentCoords==False or (coords != self.currentCoords).all():
            self.currentCoords = coords.copy()
            self.run(coords)
        return self.search_PES_tuple(self.E,multiplicity,state)[0][2]*KCAL_MOL_PER_AU

    def get_gradient(self,coords,multiplicity,state):
        if self.hasRanForCurrentCoords==False or (coords != self.currentCoords).all():
            self.currentCoords = coords.copy()
            self.run(coords)
        tmp = self.search_PES_tuple(self.grada,multiplicity,state)[0][2]
        return np.asarray(tmp)*ANGSTROM_TO_AU

    def get_coupling(self,coords,multiplicity,state1,state2):
        if self.hasRanForCurrentCoords==False or (coords != self.currentCoords).any():
            self.currentCoords = coords.copy()
            self.run(coords)
        return np.reshape(self.coup,(3*len(self.coup),1))*ANGSTROM_TO_AU


text:
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import torch
import random
import numpy as np
import math

from lista_stop.common.utils import soft_threshold
from lista_stop.common.cmd_args import cmd_args
from lista_stop.dataset import ListaDataset
from lista_stop.baselines.ista import MetricEval
import pickle

def fista_test(eval, algo_step, num_itr):
    with torch.no_grad():
        x_hat = torch.zeros(eval.x.shape)
        y_hat = torch.zeros(eval.x.shape)
        t = 1.0
        nmse_dict = dict()
        for i in range(num_itr):
            x_hat, y_hat, t = algo_step(eval.A, x_hat, y_hat, eval.y, t)
            mse, nmse, mse_per_snr, nmse_per_snr = eval.compute(y_hat)
            nmse_dict[i] = nmse.cpu().item()
            print('%d samples, itr %d, mse: %.5f, nmse: %.5f'
                  % (eval.test_size, i+1, mse, nmse))
            print('nmse per snr', nmse_per_snr)
        return nmse_dict

class FISTA(object):
    def __init__(self, A, ld=0.05,  L=None):
        self.A = A
        if L > 0:
            self.L = L
        else:
            u, s, v = torch.svd(torch.matmul(A.t(), A))
            self.L = torch.max(s)
        self.ld = ld
        self.t = 1.0

    @staticmethod
    def fista_step(A, xk_1, yk, b, ld, t, L=None):
        if L is None:
            u, s, v = torch.svd(torch.matmul(A.t(), A))
            L = torch.max(s)
        g = yk + A.t().dot(b - A.dot(yk)) / L
        x = soft_threshold(ld/L, g)
        t_next = (1 + math.sqrt(1 + 4 * t ** 2)) / 2
        y = x + (t - 1) / t_next * (x - xk_1)
        return x, y

    @staticmethod
    def fista_step_batch(A, xk_1, yk, b, ld, t, L=None):
        """
        A: m * n
        x: size * n
        b: size * m
        """
        if L is None:
            u, s, v = torch.svd(torch.matmul(A.t(), A))
            L = torch.max(s)
        g = yk + torch.matmul(b - torch.matmul(yk, A.t()), A) / L
        x = soft_threshold(ld/L, g)
        t_next = (1 + math.sqrt(1 + 4 * t ** 2)) / 2
        y = x + (t - 1) / t_next * (x - xk_1)
        return x, y, t_next

if __name__ == '__main__':
    random.seed(cmd_args.seed)
    np.random.seed(cmd_args.seed)
    torch.manual_seed(cmd_args.seed)
    # initilize validation and test data
    db = ListaDataset(partition_sizes={'test': cmd_args.num_test,
                                       'val': cmd_args.num_val},
                      m=cmd_args.m,
                      n=cmd_args.n,
                      con_num=cmd_args.con_num,
                      pnz=cmd_args.pnz,
                      snr=cmd_args.snr,
                      mix=cmd_args.snr_mix)

    print('initialize the matrices A of size = ', cmd_args.m, cmd_args.n)

    if cmd_args.snr_mix is not None:
        mix = list(map(int, cmd_args.snr_mix.split("-")))
        n_data_per_snr = int(cmd_args.num_test / len(mix))

    algo_fista = FISTA(db.A, ld=cmd_args.rho, L=cmd_args.L)
    eval_class = MetricEval(db)
    nmse_dict = fista_test(eval_class, lambda a, x, y, b, t: algo_fista.fista_step_batch(a, x, y, b, algo_fista.ld, t, algo_fista.L), cmd_args.num_algo_itr)
    f = open('FISTA_converge.pkl', 'wb')
    pickle.dump(nmse_dict, f)
    f.close()


text:
import sys
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

def calculateGenesLeftAfterIntensityThresholding(intensities_df, threshold: float):
    leftover = len(intensities_df[intensities_df['Intensity_ratio']>threshold])
    return leftover

def plotSpotsLeftAfterIntensityThresholding(path_to_decoded_genes: str):
    df = pd.read_csv(path_to_decoded_genes)
    mask = df.Gene.notnull()
    recognized_df = df[mask]
    unrecognized_df =df[~mask]

    fig, ax = plt.subplots(1,1)
    ax.set_title("Number of spots remaining after thresholding based on max channel intensity normalized by sum of all channel intensities")
    ax.set_xlabel("Threshold (%)")
    ax.set_ylabel("# of genes that pasthe threshold")

    x = np.linspace(0.25, 1, 100)
    y_recognized= [calculateGenesLeftAfterIntensityThresholding(recognized_df, threshold) for threshold in x]
    ax.plot(x, y_recognized, 'g-', label="recognized spots")

    x = np.linspace(0.25, 1, 100)
    y_nonrecognized= [calculateGenesLeftAfterIntensityThresholding(unrecognized_df, threshold) for threshold in x]

    # find biggest difference between x and y
    differences = {threshold:recog - nonrecog for threshold, recog, nonrecog in zip(x, y_recognized,y_nonrecognized)}
    max_index = max(differences, key=differences.get)
    max_difference = differences[max_index]

    ax.plot(x, y_nonrecognized, 'r-', label = "non-recognized spots")
    ax.axvline(max_index, color='k', label="optimal threshold")
    fig.tight_layout()

    plt.legend()
    plt.savefig("decoding_intensity_QC.svg", format="svg", dpi=1200)

# argparsing

decoded_genes = sys.argv[1]
plotSpotsLeftAfterIntensityThresholding(decoded_genes)

max_stars_repo_name: WoutDavid/ST-nextflow-pipeline

max_issues_repo_name: WoutDavid/ST-nextflow-pipeline

max_forks_repo_name: WoutDavid/ST-nextflow-pipeline


text:
#!/usr/bin/env python

import logging
log = logging.getLogger(__name__)
import numpy as np
import matplotlib.pyplot as plt

ri = np.array([
       [ 1.55 ,  1.478],
       [ 1.795,  1.48 ],
       [ 2.105,  1.484],
       [ 2.271,  1.486],
       [ 2.551,  1.492],
       [ 2.845,  1.496],
       [ 3.064,  1.499],
       [ 4.133,  1.526],
       [ 6.2  ,  1.619],
       [ 6.526,  1.618],
       [ 6.889,  1.527],
       [ 7.294,  1.554],
       [ 7.75 ,  1.793],
       [ 8.267,  1.783],
       [ 8.857,  1.664],
       [ 9.538,  1.554],
       [10.33 ,  1.454],
       [15.5  ,  1.454]
      ])

def divide_bins( e, mul ):
    """
    :param e: 1d array of monotonic edges
    :param mul: integer multiplicity with which to divide edges, eg 2 or 3 splits each edge into 2 or 3 etc..
    :return ee: array with extra edges obtained by splitting the input edges

         +--------+--------+     3 values, 2 bins

         +----+---+---+----+     5 values, 4 bins    (mul 2)

         +--+-+-+-+-+-+--+-+     9 values, 8 bins

    """
    nb = len(e)-1           # number of bins is one less than number of values
    nbb = nb*mul            # bins multiply
    ee = np.zeros(nbb+1)    # add one to to give number of values

    print(" divide_bins mul %d len(e) %d len(ee) %d " % ( mul, len(e), len(ee) ))

    for i in range(len(e)-1):
        a = np.linspace( e[i], e[i+1], 1+mul )
        i0 = i*mul
        i1 = (i+1)*mul
        print( " %2d %7.4f %7.4f i0 %2d i1 %2d     %s " % (i, e[i], e[i+1], i0, i1, a ) )
        ee[i0:i1+1] = a
    pass
    if mul == 1: assert np.all( ee == e )

    non_monotonic = np.any(ee[:-1] > ee[1:])
    if non_monotonic:
        log.error("non_monotonic")
        print(ee)
    pass
    return ee

def edges_plot(e):
    fig, ax = plt.subplots(figsize=[12.8, 7.2])
    for m in range(1,10):
        ee = divide_bins(e, mul=m)
        yy = np.repeat( m, len(ee) )
        ax.scatter( ee, yy, label="mul %d " % m )
    pass
    ax.legend()
    fig.show()

if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)
    e = ri[:,0]
    #ee = divide_bins(e, mul=2 )
    edges_plot(e)

max_stars_repo_name: hanswenzel/opticks

max_issues_repo_name: hanswenzel/opticks

max_forks_repo_name: hanswenzel/opticks


text:
import cv2
import numpy as np
from matplotlib import pyplot as plt

img = cv2.imread('../Images/logan.jpg', cv2.IMREAD_GRAYSCALE)
img = cv2.medianBlur(img,5)

ret,th1 = cv2.threshold(img,170,255,cv2.THRESH_BINARY)
th2 = cv2.adaptiveThreshold(img,255,cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY,11,2)
th3 = cv2.adaptiveThreshold(img,255,cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY,11,2)

titles = ['Original Image', 'Global Thresholding (v = 170)', 'Adaptive Mean Thresholding', 'Adaptive Gaussian Thresholding']
images = [img, th1, th2, th3]
for i in range(4):
    plt.subplot(2,2,i+1),plt.imshow(images[i],'gray')
    plt.title(titles[i])
    plt.xticks([]),plt.yticks([])
plt.show()

max_stars_repo_name: Sparrowan/computer-vision-ai-saturdays

max_issues_repo_name: Sparrowan/computer-vision-ai-saturdays

max_forks_repo_name: Sparrowan/computer-vision-ai-saturdays


text:
# ---
# jupyter:
#   jupytext:
#     formats: ipynb,py:light
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.11.4
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

# +
import pandas as pd
import numpy as np
import matplotlib as plt
import sys
sys.path.insert(0, '../Imitator')
from plot_helper import *

colors = [
    "tab:blue",
    "tab:orange",
    "tab:green",
    "tab:red",
    "tab:purple",
    "tab:brown",
    "tab:pink",
]
# -

# # Load Data and plot Bars

# +
# -1) Move any csv file into your Data directory
# 0) Specify Data directory for the models
data_dir = "/raid/clark/summer2021/datasets/corrected-wander-full/data"

# 1) Specify the Dataset
df = pd.read_csv("wander_pre/wander_pre_percentage.csv", index_col=0)
columns = list(df.columns)
mazes = columns[1:-3]
df = df.assign(clean_names=list(map(get_network_name, list(df["Network"]))))

# 2) Get extra Pretrained/Not Pretrained Data
avg_valid = merge_loss_data(data_dir, df, "valid_loss", "pretrained", True)
replicate_valid = merge_loss_data(data_dir, df, "valid_loss", "pretrained", False)
avg_train = merge_loss_data(data_dir, df, "train_loss", "pretrained", True)
replicate_train = merge_loss_data(data_dir, df, "train_loss", "pretrained", False)
avg_accuracy = merge_loss_data(data_dir, df, "accuracy", "pretrained", True)
df = pd.merge(df, avg_valid, on="clean_names")
df = df.sort_values(by=['mean_completion'])

# 3) Pass in "df" into the function, plot_bars()
ax = plot_bars(df, "Completion")
plt.gcf().subplots_adjust(left=0.35)
ax.set_title("Pretrained Wander Dataset Percentage Navigated Per Model Replicate");
# -

ax = plot_average_scatter(avg_accuracy[avg_accuracy.clean_names != "ResNet18"])
ax.title("Wander Dataset Pretrained Average Maze Percentage Navigated vs Validation Accuracy", size=20);
ax.xlabel("Validation Accuracy", size=20)
ax.ylabel("Average Percentage Navigated", size=20)
ax.xticks(size=15)
ax.yticks(size=15)

# The other DataFrames like "avg_valid" can be passed into "plot_average_scatter"()
ax = plot_average_scatter(avg_valid)
ax.title("Wander Dataset Pretrained Average Maze Percentage Navigated vs Validation Loss", size=20);
ax.xlabel("Validation Loss", size=20)
ax.ylabel("Average Percentage Navigated", size=20)
ax.xticks(size=15)
ax.yticks(size=15)

# +
df = pd.read_csv("wander_notpretrained/wander_npre_percentage.csv", index_col=0)
columns = list(df.columns)
mazes = columns[1:-3]
df = df.assign(clean_names=list(map(get_network_name, list(df["Network"]))))
avg_valid = merge_loss_data(data_dir, df, "valid_loss", "notpretrained", True)
replicate_valid = merge_loss_data(data_dir, df, "valid_loss", "notpretrained", False)
avg_train = merge_loss_data(data_dir, df, "train_loss", "notpretrained", True)
replicate_train = merge_loss_data(data_dir, df, "train_loss", "notpretrained", False)
avg_accuracy = merge_loss_data(data_dir, df, "accuracy", "notpretrained", True)
df = pd.merge(df, avg_valid, on="clean_names")
df = df.sort_values(by=['mean_completion'])

# %load_ext autoreload
# %autoreload 2
ax = plot_bars(df, "Completion")
plt.gcf().subplots_adjust(left=0.35)
ax.set_title("Scratch Wander Dataset Percentage Navigated Per Model Replicate");
# -

ax = plot_average_scatter(avg_accuracy)
ax.title("Wander Dataset Scratch Average Maze Percentage Navigated vs Validation Accuracy", size=20);
ax.xlabel("Validation Accuracy", size=20)
ax.ylabel("Average Percentage Navigated", size=20)
ax.xticks(size=15)
ax.yticks(size=15)

ax = plot_average_scatter(avg_valid)
ax.title("Wander Dataset Scratch Average Maze Percentage Navigated vs Validation Loss", size=20);
ax.xlabel("Validation Loss", size=20)
ax.ylabel("Average Percentage Navigated", size=20)
ax.xticks(size=15)
ax.yticks(size=15)

pd.read_csv("wander_cmd/cmd_percentage.csv", index_col=0)

# +
data_dir = "/raid/clark/summer2021/datasets/corrected-wander-full/cmd_data"

df = pd.read_csv("wander_cmd/cmd_percentage.csv", index_col=0)
columns = list(df.columns)
mazes = columns[1:-3]
df = df.assign(clean_names=list(map(get_network_name, list(df["Network"]))))
avg_valid = merge_loss_data(data_dir, df, "valid_loss", "cmd", True)
replicate_valid = merge_loss_data(data_dir, df, "valid_loss", "cmd", False)
avg_train = merge_loss_data(data_dir, df, "train_loss", "cmd", True)
replicate_train = merge_loss_data(data_dir, df, "train_loss", "cmd", False)
df = pd.merge(df, avg_valid, on="clean_names")
df = df.sort_values(by=['mean_completion'])

# %load_ext autoreload
# %autoreload 2
ax = plot_bars(df, "Completion")
plt.gcf().subplots_adjust(left=0.35)
ax.set_title("Not Pretrained Percentage Navigated Per Wander Dataset Model Replicate");

# +
df = pd.read_csv("RNN/rnn_percentage.csv", index_col=0)
columns = list(df.columns)
mazes = columns[1:-3]
df = df.assign(clean_names=list(map(get_network_name, list(df["Network"]))))
avg_valid = merge_loss_data(data_dir, df, "valid_loss", "rnn", True)
replicate_valid = merge_loss_data(data_dir, df, "valid_loss", "rnn", False)
avg_train = merge_loss_data(data_dir, df, "train_loss", "rnn", True)
replicate_train = merge_loss_data(data_dir, df, "train_loss", "rnn", False)
df = pd.merge(df, avg_valid, on="clean_names")
df = df.sort_values(by=['mean_completion'])

# %load_ext autoreload
# %autoreload 2
ax = plot_bars(df, "Completion")
plt.gcf().subplots_adjust(left=0.35)
ax.set_title("Not Pretrained Percentage Navigated Per Wander Dataset Model Replicate");

# +
df = pd.read_csv("uniform_pretrained/uniform_pre_completion_results.csv", index_col=0)
columns = list(df.columns)
mazes = columns[1:-3]
df = df.assign(clean_names=list(map(get_network_name, list(df["Network"]))))
avg_valid = merge_loss_data(data_dir, df, "valid_loss", "pretrained", True)
replicate_valid = merge_loss_data(data_dir, df, "valid_loss", "pretrained", False)
avg_train = merge_loss_data(data_dir, df, "train_loss", "pretrained", True)
replicate_train = merge_loss_data(data_dir, df, "train_loss", "pretrained", False)
df = pd.merge(df, avg_valid, on="clean_names")
df = df.sort_values(by=['mean_completion'])

# %load_ext autoreload
# %autoreload 2
ax = plot_bars(df, "Completion")
plt.gcf().subplots_adjust(left=0.35)
ax.set_title("Pretrained Uniform Dataset Percentage Navigated Per Model Replicate");

# +
df = pd.read_csv("uniform_nonpre/uniform_nonpre_percentage.csv", index_col=0)
columns = list(df.columns)
mazes = columns[1:-3]
df = df.assign(clean_names=list(map(get_network_name, list(df["Network"]))))
avg_valid = merge_loss_data(data_dir, df, "valid_loss", "notpretrained", True)
replicate_valid = merge_loss_data(data_dir, df, "valid_loss", "notpretrained", False)
avg_train = merge_loss_data(data_dir, df, "train_loss", "pretrained", True)
replicate_train = merge_loss_data(data_dir, df, "train_loss", "notpretrained", False)
df = pd.merge(df, avg_valid, on="clean_names")
df = df.sort_values(by=['mean_completion'])

# %load_ext autoreload
# %autoreload 2
ax = plot_bars(df, "Completion")
plt.gcf().subplots_adjust(left=0.35)
ax.set_title("Scratch Uniform Dataset Percentage Navigated Per Model Replicate");

# +
df = pd.read_csv("handmade_pretrained/handmade_pre_percentage.csv", index_col=0)
columns = list(df.columns)
mazes = columns[1:-3]
df = df.assign(clean_names=list(map(get_network_name, list(df["Network"]))))
avg_valid = merge_loss_data(data_dir, df, "valid_loss", "pretrained", True)
replicate_valid = merge_loss_data(data_dir, df, "valid_loss", "pretrained", False)
avg_train = merge_loss_data(data_dir, df, "train_loss", "pretrained", True)
replicate_train = merge_loss_data(data_dir, df, "train_loss", "pretrained", False)
df = pd.merge(df, avg_valid, on="clean_names")
df = df.sort_values(by=['mean_completion'])

# %load_ext autoreload
# %autoreload 2
ax = plot_bars(df, "Completion")
plt.gcf().subplots_adjust(left=0.35)
ax.set_title("Pretrained Handmade Dataset Percentage Navigated Per Model Replicate");

# +
df = pd.read_csv("handmade_notpretrained/handmade_notpre_percentage.csv", index_col=0)
columns = list(df.columns)
mazes = columns[1:-3]
df = df.assign(clean_names=list(map(get_network_name, list(df["Network"]))))
avg_valid = merge_loss_data(data_dir, df, "valid_loss", "notpretrained", True)
replicate_valid = merge_loss_data(data_dir, df, "valid_loss", "notpretrained", False)
avg_train = merge_loss_data(data_dir, df, "train_loss", "notpretrained", True)
replicate_train = merge_loss_data(data_dir, df, "train_loss", "notpretrained", False)
df = pd.merge(df, avg_valid, on="clean_names")
df = df.sort_values(by=['mean_completion'])

# %load_ext autoreload
# %autoreload 2
ax = plot_bars(df, "Completion")
plt.gcf().subplots_adjust(left=0.35)
ax.set_title("Scratch Handmade Dataset Percentage Navigated Per Model Replicate");
# -

# +
# %load_ext autoreload
# %autoreload 2

ax = plot_average_scatter(avg_accuracy)
ax.title("Wander Dataset Scratch Average Maze Percentage Navigated vs Validation Accuracy", size=20);
ax.xlabel("Validation Accuracy", size=20)
ax.ylabel("Average Percentage Navigated", size=20)
ax.xticks(size=15)
ax.yticks(size=15)
# -

ax2 = plot_average_scatter(replicate_valid)
ax.title("Maze Percentage Navigated as a Function of Valid Loss", size=20);
ax.xlabel("Valid Loss", size=20)
ax.ylabel("Percentage Navigated", size=20)
ax.xticks(size=15)
ax.yticks(size=15)

ax3 = plot_average_scatter(avg_train)
ax.title("Average Maze Percentage Navigated as a Function of Training Loss", size=20);
ax.xlabel("Training Loss", size=20)
ax.ylabel("Average Percentage Navigated", size=20)
ax.xticks(size=15)
ax.yticks(size=15)

ax4 = plot_average_scatter(replicate_train)
ax.title("Maze Percentage Navigated as a Function of Training Loss", size=20);
ax.xlabel("Training Loss", size=20)
ax.ylabel("Percentage Navigated", size=20)
ax.xticks(size=15)
ax.yticks(size=15)

# %load_ext autoreload
# %autoreload 2
warnings.filterwarnings('ignore')
ax = plot_boxplot(df, mazes);
ax.title("Percentage Completed Per Maze", size=20)
ax.xlabel("Validation Maze", size=20)
ax.ylabel("Percentage Completed", size=20)
ax.xticks(size=10)
ax.yticks(size=10)

df = pd.read_csv("../Imitator/RNN_percentage_uniform_full_2_mazes.csv", index_col=0)
columns = list(df.columns)
mazes = columns[1:-3]

ax = plot_bars(df, "Completion")

# # Scraps

model_type = []
for m in list(df['Network']):
    if "notpretrained" in m:
        model_type.append("Non-Pretrained")
    else:
        model_type.append("Pretrained")

clean_names = list(map(get_network_name, list(df['Network'])))

df = df.assign(model_type = model_type, clean_names = clean_names)
df

# # Scatter Plot Code

clean_names = list(map(get_network_name, list(pretrained_df['Network'])))

pretrained_df = pretrained_df.assign(clean_names = clean_names)

data_dir = "/raid/clark/summer2021/datasets/wander-full/data"

# Import seaborn
import seaborn as sns
# Apply the default theme
sns.set_theme()

fig = plt.gcf()
fig.set_size_inches(12, 9)
sns.scatterplot(data=pretrained_df, x="mean", y="error", hue="Network", style="Network", s=200)
plt.legend(bbox_to_anchor=(1.01, 1), borderaxespad=0)

len(list(set(pretrained_df['clean_names'])))

colors = get_colors(24)

# +
sns_colors = list(sns.color_palette())

color_labels = []
for i in range(24 + 1):
    color = sns_colors[i % len(sns_colors)]
    for i in range(4):
        color_labels.append(color)
color_labels;

# +
fig = plt.gcf()
fig.set_size_inches(16, 9)
sns.set_theme(style="whitegrid")

ci_bounds = get_error(pretrained_df)

label_names = list(pretrained_df["clean_names"])
sparse_labels = []
for i in range(0, len(label_names)):
    if i % 4 == 0:
        sparse_labels.append(clean_names[i])
    else:
        sparse_labels.append("")

ax = sns.barplot(
    x="Network",
    y="mean",
    data=pretrained_df,
    palette=color_labels,
    yerr=ci_bounds,
    ecolor="grey",
#     saturation=1
)
ax.set_xticklabels(labels=sparse_labels, rotation=75)
ax.set_yticks(np.arange(0, 100, 10))
ax.set_ylim(bottom=0.0, top=100.0)
# -

avg_df = merge_loss_data(data_dir, pretrained_df, "valid_loss", clean_names, True)
ax = plot_average_scatter(avg_df)
ax.set_title("Averaged Steps Taken Over Valid Loss Per Model")
ax.set_xlabel("Valid Loss")

training_losses = []
for c in csvs:
    df = pandas.read_csv(data_dir + "/" + c)
    if len(df) == 0:
        training_losses.append(0.0)
    else:
        training_losses.append(min(df["valid_loss"]))
training_losses

losses

# +
data_files = os.listdir(data_dir)
data_files.sort()
csvs = list(filter(get_csv, data_files))
losses = get_losses(csvs, data_dir, "valid_loss")
means = pretrained_df['mean']
names = clean_names

# losses = np.array(losses).reshape(-1,4).mean(axis=1)
# means = np.array(means).reshape(-1,4).mean(axis=1)
# names = list(set(clean_names))

# df = pandas.DataFrame()
# df = df.assign(Network = names)
# df = df.assign(losses = losses)
# df = df.assign(mean_steps = means)
# -

np.array(losses).reshape(-1,4).mean(axis=1)


text:
import os
import pickle

import numpy as np
from PIL import Image
from torch.utils.data.dataset import Dataset

class CIFAR10(Dataset):
    def __init__(self, root, transform=None, train=True):
        self.train = train
        self.transform = transform
        if train:
            data_list = [
                "data_batch_1",
                "data_batch_2",
                "data_batch_3",
                "data_batch_4",
                "data_batch_5",
            ]
        else:
            data_list = ["test_batch"]
        data = []
        targets = []
        if root[0] == "~":
            # interprete `~` as the home directory.
            root = os.path.expanduser(root)
        for file_name in data_list:
            file_path = os.path.join(root, file_name)
            with open(file_path, "rb") as f:
                entry = pickle.load(f, encoding="latin1")
            data.append(entry["data"])
            targets.extend(entry["labels"])
        # Convert data (List) to NHWC (np.ndarray) works with PIL Image.
        data = np.vstack(data).reshape(-1, 3, 32, 32).transpose((0, 2, 3, 1))
        self.data = data
        self.targets = np.asarray(targets)

    def __getitem__(self, index):
        img, target = self.data[index], self.targets[index]
        img = Image.fromarray(img)
        if self.transform is not None:
            img = self.transform(img)
        item = {"img": img, "target": target}

        return item

    def __len__(self):
        return len(self.data)


text:
#coding=utf-8
from flask import Flask, jsonify, request, render_template,session
import pymongo
import pandas as pd
import datetime
import time
import numpy as np

class Database:
    def __init__(self):
        # self.client = pymongo.MongoClient("mongodb+srv://james:wolf0719@cluster0-oiynz.azure.mongodb.net/test?retryWrites=true&w=majority")
        self.client = pymongo.MongoClient('127.0.0.1', 27017)
        self.client.admin.authenticate('james', 'wolf0719')

    def set_db(self):
        return self.client.ufs


text:
# Copyright 2020 Google LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import gym
import numpy as np
import tensorflow.compat.v2 as tf

from typing import Any, Callable, Dict, List, Optional, Tuple, Union

from tf_agents.trajectories import time_step
from tf_agents.environments import gym_wrapper
from tf_agents import specs
from tf_agents.utils import nest_utils

from dice_rl.data.dataset import OnpolicyDataset, EnvStep, StepType

# pytype: disable=attribute-error

class GymOnpolicyDataset(OnpolicyDataset):
  """On-policy dataset for OpenAI Gym environment and Python policy."""

  def __init__(self, gym_env: gym.Env,
               policy_fn: Callable[[Any], Tuple[Any, Dict[Any, Any]]],
               policy_info_spec: Dict[Any, Any],
               episode_step_limit: Optional[int] = None):
    """Creates the dataset.

    Args:
      gym_env: An OpenAI Gym environment.
      policy_fn: The behavior policy. A function that takes in an observation
        from the environment and returns a tuple of sampled action and
        policy_info (a dictionary).
      policy_info_spec: A spec defining the policy_info returned by policy_fn.
      episode_step_limit: If specified, episodes are terminated after this many
        steps. Note that this leads to episodes with n+1 observations.
    """
    self._env = gym_env
    self._policy_fn = policy_fn
    self._episode_step_limit = episode_step_limit
    self._spec = self._create_spec(policy_info_spec)

    self._num_steps = 0
    self._num_total_steps = 0
    self._num_episodes = 0
    self._num_total_episodes = 0
    self._start_on_next_step = True

  def _create_spec(self, policy_info_spec):
    observation_spec = gym_wrapper.spec_from_gym_space(
        self._env.observation_space)
    action_spec = gym_wrapper.spec_from_gym_space(
        self._env.action_space)

    tf_agents_time_step_spec = time_step.time_step_spec(observation_spec)
    step_num_spec = specs.tensor_spec.from_spec(
        specs.BoundedArraySpec([],
                               dtype=np.int64,
                               minimum=0,
                               maximum=self._episode_step_limit,
                               name='step_num'))
    return EnvStep(tf_agents_time_step_spec.step_type, step_num_spec,
                   observation_spec, action_spec,
                   tf_agents_time_step_spec.reward,
                   tf_agents_time_step_spec.discount, policy_info_spec, {}, {})

  @property
  def spec(self) -> EnvStep:
    return self._spec

  @property
  def num_steps(self) -> Union[int, tf.Tensor]:
    return self._num_steps

  @property
  def num_total_steps(self) -> Union[int, tf.Tensor]:
    return self._num_total_steps

  @property
  def num_episodes(self) -> Union[int, tf.Tensor]:
    return self._num_episodes

  @property
  def num_total_episodes(self) -> Union[int, tf.Tensor]:
    return self._num_total_episodes

  @property
  def constructor_args_and_kwargs(self):
    args = [self._env, self._policy_fn, self._spec]
    kwargs = {'episode_step_limit': self._episode_step_limit}
    return args, kwargs

  def _start_new_episode(self):
    self._observation = self._env.reset()
    self._step_type = StepType.FIRST
    self._discount = 1.0
    self._start_on_next_step = False
    self._cur_step_num = 0

  def _get_step(self) -> EnvStep:
    if self._start_on_next_step:
      self._start_new_episode()

    if StepType.is_last(self._step_type):
      # This is the last (terminating) observation of the environment.
      self._start_on_next_step = True
      self._num_total_steps += 1
      self._num_episodes += 1
      # The policy is not run on the terminal step, so we just carry over the
      # reward, action, and policy_info from the previous step.
      return EnvStep(self._step_type, self._cur_step_num,
                     self._observation, self._action,
                     self._reward, self._discount,
                     self._policy_info, {}, {})

    self._action, self._policy_info = self._policy_fn(self._observation)
    self._next_observation, self._reward, done, _ = self._env.step(self._action)
    self._next_discount = float(not done)
    self._cur_step_num += 1

    if done or (self._episode_step_limit and
                self._cur_step_num >= self._episode_step_limit):
      self._next_step_type = StepType.LAST
    else:
      self._next_step_type = StepType.MID

    step = EnvStep(self._step_type, self._cur_step_num - 1,
                   self._observation, self._action,
                   self._reward, self._discount,
                   self._policy_info, {}, {})

    self._num_steps += 1
    self._num_total_steps += 1
    if StepType.is_first(self._step_type):
      self._num_total_episodes += 1

    self._observation = self._next_observation
    self._step_type = self._next_step_type
    self._discount = self._next_discount

    return step

  def get_step(self, batch_size: Optional[int] = None,
               num_steps: Optional[int] = None) -> EnvStep:
    if batch_size is not None:
      raise ValueError('This dataset does not support batched step sampling.')

    if num_steps is None:
      return self._get_step()

    env_steps = []
    for _ in range(num_steps):
      next_step = self._get_step()
      env_steps.append(next_step)

    return nest_utils.stack_nested_arrays(env_steps)

  def _get_episode(self, truncate_episode_at: Optional[int] = None) -> List[
      EnvStep]:

    self._start_new_episode()
    env_steps = []
    while True:
      next_step = self._get_step()
      env_steps.append(next_step)
      if next_step.is_last():
        break
      if truncate_episode_at and len(env_steps) >= truncate_episode_at:
        break

    return env_steps

  def get_episode(self, batch_size: Optional[int] = None,
                  truncate_episode_at: Optional[int] = None) -> Tuple[
                      EnvStep, np.ndarray]:
    if batch_size is None:
      episode = self._get_episode(truncate_episode_at)
      mask = np.ones((len(episode),))
      return nest_utils.stack_nested_arrays(episode), mask
    if batch_size <= 0:
      raise ValueError('Invalid batch size %s.' % batch_size)

    episodes = []
    episode_lengths = []
    for _ in range(batch_size):
      next_episode = self._get_episode(truncate_episode_at)
      episodes.append(next_episode)
      episode_lengths.append(len(next_episode))

    max_length = max(episode_lengths)
    for episode in episodes:
      episode.extend([episode[-1]] * (max_length - len(episode)))

    batched_episodes = nest_utils.stack_nested_arrays(
        [nest_utils.stack_nested_arrays(episode)
         for episode in episodes])

    mask = np.arange(max_length)[None, :] < np.array(episode_lengths)[:, None]

    return batched_episodes, mask

max_stars_repo_name: SnowflyLXF/FedDICE

max_issues_repo_name: SnowflyLXF/FedDICE


text:
# To add a new cell, type '#%%'
# To add a new markdown cell, type '#%% [markdown]'
#%% Change working directory from the workspace root to the ipynb file location. Turn this addition off with the DataScience.changeDirOnImportExport setting
# ms-python.python added
import os
try:
	os.chdir(os.path.join(os.getcwd(), 'eddy_src/q_learning_stock'))
	print(os.getcwd())
except:
	pass

import indicators
import util
import config
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import datetime
from bokeh.io import curdoc, output_notebook, show
from bokeh.layouts import row, column, gridplot
from bokeh.models import ColumnDataSource, RangeTool, BoxAnnotation, HoverTool
from bokeh.models.widgets import Slider, TextInput
from bokeh.plotting import figure, output_file

#%%
stocks = ['GCIIX', 'GCGIX', 'GCVIX']#, 'GERIX', 'GSLC', 'GEM', 'GARSX', 'GCSIX', 'GSIE', 'GICIX', 'EWZ']
# stocks = ['GGSIX', 'GOIIX', 'GIPIX']
select_indicators = ['MACD Line']#, 'RSI', 'Stochastic Oscillator %K', 'CCI']
start = '1/1/2016'
end = '31/12/2018'
action_periods_dict = {}

for stock in stocks:
	df = pd.read_csv('data/goldman/{}.csv'.format(stock), usecols=config.column_names, parse_dates=['Date'])
	df = df.loc[(df['Date'] > start) & (df['Date'] <= end)]
	df = indicators.create_indicator_columns(df)
	indicator_dict = {}

	for indicator in select_indicators:
		indicator_dict[indicator] = indicators.get_action_periods(df, indicator)

	util.plot_market_trends(df, indicator_dict, stock)
	action_periods_dict[stock] = indicator_dict

#%% [markdown]
## Plot the switching portfolio performance for goldman sachs
#%%
portfolios = ['balanced', 'equity_growth', 'growth', 'growth_income', 'satellite']
df_list = []
# Add the switched portfolio first due to limitations by plotting (for now)
df_list.append(pd.read_csv('data/goldman/best_portfolio_switch.csv'))
for portfolio in portfolios:
	df = pd.read_csv('data/goldman/portfolio/{}/portfolio_quarterly_return.csv'.format(portfolio))
	# Reduce the df to 1 symbol only (for now)
	df_list.append(df.loc[:11])

#%%
def plot_portfolio_comparisons(df_list: list, date_col='end_period', price_col='quarterly_return', symbol_col='symbol'):
    """**First dataframe must be the Portfolio with switching.**
    Currently only supports up to 6 dataframes in list.
    """
    x_min = datetime.datetime.strptime(df_list[0][date_col].min(), '%Y-%m-%d')
    x_max = datetime.datetime.strptime(df_list[0][date_col].max(), '%Y-%m-%d')
    y_min = df_list[0][price_col].min()
    y_max = df_list[0][price_col].max()

    # Temporary hack for date. Should have other date ranges
    date = []
    for i in df_list[0][date_col].values.tolist():
        date.append(datetime.datetime.strptime(i, '%Y-%m-%d'))

    p = figure(title="Portfolio Comparison",
           x_range=(x_min, x_max), y_range=(y_min, y_max), x_axis_type='datetime',
           background_fill_color="#fafafa")

    p.line(date, df_list[0][price_col].values.tolist(), legend="Portfolio with switching",
        line_color="black", line_dash="dashed")

    p.line(date, df_list[1][price_col].values.tolist(), legend="{}".format(df_list[1].iloc[0][symbol_col]))
    p.circle(date, df_list[1][price_col].values.tolist(), legend="{}".format(df_list[1].iloc[0][symbol_col]))

    p.line(date, df_list[2][price_col].values.tolist(), legend="{}".format(df_list[2].iloc[0][symbol_col]), line_color="olivedrab")
    p.circle(date, df_list[2][price_col].values.tolist(), legend="{}".format(df_list[2].iloc[0][symbol_col]),
            fill_color=None, line_color="olivedrab")

    p.line(date, df_list[3][price_col].values.tolist(), legend="{}".format(df_list[3].iloc[0][symbol_col]),
        line_color="gold", line_width=2)

    p.line(date, df_list[4][price_col].values.tolist(), legend="{}".format(df_list[4].iloc[0][symbol_col]),
        line_dash="dotted", line_color="indigo", line_width=2)

    p.line(date, df_list[5][price_col].values.tolist(), legend="{}".format(df_list[5].iloc[0][symbol_col]),
        line_color="coral", line_dash="dotdash", line_width=2)

    p.legend.location = "bottom_left"

    # output_file("data/portfolio_comparison.html", title="Portfolio Comparison")

    show(p)
plot_portfolio_comparisons(df_list)

#%% [markdown]
## Plot the switching portfolio performance according to 8 quarters

#%%
def plot_portfolio_8q_comparisons(df_list: list, date_col='end_period', price_col='quarterly_return', symbol_col='symbol'):

    # Temporary hack for date. Should have other date ranges
    # create 3 date ranges and convert them to datetime
    date = []
    for i in range(3):
        end_date = df_list[0][date_col].iloc[i*2+7]
        date.append(datetime.datetime.strptime(end_date, '%Y-%m-%d'))
    print(date)

    return_range = []
    for i in range(6):
        return_range.append([])
        for j in range(3):
            returns = (df_list[i][price_col].iloc[j*2+7]+df_list[i][price_col].iloc[j*2])/2
            return_range[i].append(returns)
    print(return_range)

    p = figure(title="Portfolio Comparison for 8 Quarters starting from 1 jan 2016",
           x_range=(date[0], date[2]), y_range=(-10, 10), x_axis_type='datetime',
           background_fill_color="#fafafa")

    p.line(date, return_range[0], legend="Portfolio with switching",
        line_color="black", line_dash="dashed")

    p.line(date, return_range[1], legend="{}".format(df_list[1].iloc[0][symbol_col]))
    p.circle(date, return_range[1], legend="{}".format(df_list[1].iloc[0][symbol_col]))

    p.line(date, return_range[2], legend="{}".format(df_list[2].iloc[0][symbol_col]), line_color="olivedrab")
    p.circle(date, return_range[2], legend="{}".format(df_list[2].iloc[0][symbol_col]),
            fill_color=None, line_color="olivedrab")

    p.line(date, return_range[3], legend="{}".format(df_list[3].iloc[0][symbol_col]),
        line_color="gold", line_width=2)

    p.line(date, return_range[4], legend="{}".format(df_list[4].iloc[0][symbol_col]),
        line_dash="dotted", line_color="indigo", line_width=2)

    p.line(date, return_range[5], legend="{}".format(df_list[5].iloc[0][symbol_col]),
        line_color="coral", line_dash="dotdash", line_width=2)

    p.legend.location = "top_right"

    # output_file("data/portfolio_comparison.html", title="Portfolio Comparison")

    show(p)
    # save_df = pd.DataFrame({'date': date, 'returns': return_range[0]})
    # save_df.to_csv('data/8_quarterly.csv')

#%%
plot_portfolio_8q_comparisons(df_list)

#%%