text:
import numpy as np
from scipy import ndimage
import warnings

def matlab_round(array):
    '''round equivalent to matlab function, which rounds .5 away from zero

    (used in matlab_histo so we can unit test against matlab code).

    But np.round() would rounds .5 to nearest even number, e.g.:
    - np.round(0.5) = 0, matlab_round(0.5) = 1
    - np.round(2.5) = 2, matlab_round(2.5) = 3

    Arguments
    ---------
    array : `np.array`
        the array to round

    Returns
    -------
    rounded_array : `np.array`
        the rounded array
    '''
    (fracPart, intPart) = np.modf(array)
    return intPart + (np.abs(fracPart) >= 0.5) * np.sign(fracPart)

def matlab_histo(array, nbins=101, binsize=None, center=None):
    '''Compute a histogram of array.

    How does this differ from MatLab's HIST function?  This function:
      - allows uniformly spaced bins only.
      + operates on all elements of MTX, instead of columnwise.
      + is much faster (approximately a factor of 80 on my machine).
      + allows specification of number of bins OR binsize. Default=101 bins.
      + allows (optional) specification of binCenter.

    Arguments
    ---------
    array : `np.array`
        the array to bin
    nbins : `int`
        the number of histogram bins
    binsize : `float` or None
        the size of each bin. if None, we use nbins to determine it as:
        `(array.max() - array.min()) / nbins`
    center : `float` or None
        the center position for the histogram bins. if None, this is `array.mean()`

    Returns
    -------
    N : `np.array`
        the histogram counts
    edges : `np.array`
        vector containing the centers of the histogram bins
    '''
    mini = array.min()
    maxi = array.max()

    if center is None:
        center = array.mean()

    if binsize is None:
        # use nbins to determine binsize
        binsize = (maxi-mini) / nbins

    nbins2 = int(matlab_round((maxi - center) / binsize) - matlab_round((mini - center) / binsize))
    if nbins2 != nbins:
        warnings.warn('Overriding bin number %d (requested %d)' % (nbins2, nbins))
        nbins = nbins2

    # np.histogram uses bin edges, not centers like Matlab's hist
    # compute bin edges (nbins + 1 of them)
    edge_left = center + binsize * (-0.499 + matlab_round((mini - center) / binsize))
    edges = edge_left + binsize * np.arange(nbins+1)
    N, _ = np.histogram(array, edges)

    # matlab version returns column vectors, so we will too.
    # to check: return edges or centers? edit comments.
    return (N.reshape(1, -1), edges.reshape(1, -1))

def rcosFn(width=1, position=0, values=(0, 1)):
    '''Return a lookup table containing a "raised cosine" soft threshold function

    Y =  VALUES(1) + (VALUES(2)-VALUES(1)) * cos^2( PI/2 * (X - POSITION + WIDTH)/WIDTH )

    this lookup table is suitable for use by `pointOp`

    Arguments
    ---------
    width : `float`
        the width of the region over which the transition occurs
    position : `float`
        the location of the center of the threshold
    values : `tuple`
        2-tuple specifying the values to the left and right of the transition.

    Returns
    -------
    X : `np.array`
        the x valuesof this raised cosine
    Y : `np.array`
        the y valuesof this raised cosine
    '''

    sz = 256   # arbitrary!

    X = np.pi * np.arange(-sz-1, 2) / (2*sz)

    Y = values[0] + (values[1]-values[0]) * np.cos(X)**2

    # make sure end values are repeated, for extrapolation...
    Y[0] = Y[1]
    Y[sz+2] = Y[sz+1]

    X = position + (2*width/np.pi) * (X + np.pi/4)

    return (X, Y)

def project_polar_to_cartesian(data):
    """Take a function defined in polar coordinates and project it into Cartesian coordinates

    Inspired by https://pyabel.readthedocs.io/en/latest/_modules/abel/tools/polar.html, which went
    the other way. Note that we currently don't implement the Cartesian to polar projection, but
    could do so based on this code fairly simply if it's necessary.

    Currently, this only works for square images and we require that the original image and the
    reprojected image are the same size. There should be a way to avoid both of these issues, but I
    can't think of a way to do that right now.

    Parameters
    ----------
    data : array_like
        The 2d array to convert from polar to Cartesian coordinates. We assume the first dimension
        is the polar radius and the second is the polar angle.

    Returns
    -------
    output : np.array
        The 2d array in Cartesian coordinates.

    """
    if np.isnan(data).any():
        data[np.isnan(data)] = 0
        warnings.warn("project_polar_to_cartesian won't work if there are any NaNs in the array, "
                      "so we've replaced all NaNs with 0s")
    nx = data.shape[1]
    ny = data.shape[0]
    if nx != ny:
        raise Exception("There's an occasional bug where we don't wrap the angle correctly if nx "
                        "and ny aren't equal, so we don't support this for now!")

    max_radius = data.shape[0]
    x_i = np.linspace(-max_radius, max_radius, nx, endpoint=False)
    y_i = np.linspace(-max_radius, max_radius, ny, endpoint=False)
    x_grid, y_grid = np.meshgrid(x_i, y_i)
    # need to flip the y indices so that negative is at the bottom (to correspond with how we have
    # the polar angle -- 0 on the right)
    y_grid = np.flipud(y_grid)

    r = np.sqrt(x_grid**2 + y_grid**2)

    theta = np.arctan2(y_grid, x_grid)
    # having the angle run from 0 to 2 pi seems to avoid most of the discontinuities
    theta = np.mod(theta, 2*np.pi)
    # need to convert from 2pi to pixel values
    theta *= nx/(2*np.pi)

    r_i, theta_i = r.flatten(), theta.flatten()
    # map_coordinates requires a 2xn array
    coords = np.vstack((r_i, theta_i))
    # we use mode="nearest" to deal with weird discontinuities that may pop up near the theta=0
    # line
    zi = ndimage.map_coordinates(data, coords, mode='nearest')
    output = zi.reshape((ny, nx))
    return output

if __name__ == "__main__":
    X, Y = rcosFn(width=1, position=0, values=(0, 1))

    import matplotlib.pyplot as plt
    plt.figure()
    plt.plot(X, Y)
    plt.show()

max_stars_repo_name: drwalton/pyrtools

max_issues_repo_name: drwalton/pyrtools

max_forks_repo_name: drwalton/pyrtools

reason: import numpy,from scipy


text:
"""
Support Vector Regression test on roles.salaries data.
"""
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from sklearn.preprocessing import StandardScaler
from sklearn.svm import SVR

df = pd.read_csv('./../../data/roles.salaries.csv')
X = df.iloc[:, 1:2].values
y = df.iloc[:, 2].values
print("X shape / y shape: ", X.shape, y.shape)
print("X:\n", X)
print("y:\n", y)

sc_X = StandardScaler()
sc_y = StandardScaler()
X = sc_X.fit_transform(X)
y = y.reshape(-1, 1)
y = sc_y.fit_transform(y)
print("Feature scaled X:\n", X)
print("Feature scaled y:\n", y)

model = SVR(kernel='rbf')
model.fit(X, y)

y_predicted = sc_y.inverse_transform(model.predict(sc_X.transform(np.array([[6.5]]))))
print("Support Vector Regression predict @ 6.5 role-level salary: ", y_predicted)

plt.scatter(X, y, color='red')
plt.plot(X, model.predict(X), color='blue')
plt.title('Support Vector Regression')
plt.xlabel('Role-Level')
plt.ylabel('Salary')
plt.show()

# Role-Level 1 - 10, step 0.1
X_grid = np.arange(min(X), max(X), 0.1)
X_grid = X_grid.reshape((len(X_grid), 1))

plt.scatter(X, y, color='red')
plt.plot(X_grid, model.predict(X_grid), color='blue')
plt.title('Smooth Support Vector Regression')
plt.xlabel('Role-Level')
plt.ylabel('Salary')
plt.show()

ext: py

reason: import numpy


text:
import numpy as np
import librosa
import os
from pytorch_sound.data.meta.ljspeech import LJSpeechMeta
from torch.utils.data import Dataset, DataLoader
from typing import Tuple

class AudioDataset(Dataset):

    def __init__(self, meta_frame, crop_length: int, seed: int = 1234):
        self.meta_frame = meta_frame
        self.column_name = 'audio_filename'
        self.crop_length = crop_length
        self.seed = seed
        np.random.seed(seed)

    def __getitem__(self, idx):
        # get selected file path
        file_path = self.meta_frame.iloc[idx][self.column_name]

        # load audio
        wav, _ = librosa.load(file_path, sr=None)
        # wav = librosa.effects.trim(wav)[0]

        # random crop
        if self.crop_length:
            rand_start = np.random.randint(0, (len(wav) - self.crop_length))
            cropped_wav = wav[rand_start:rand_start + self.crop_length]

            # crop on voiced part
            while np.abs(cropped_wav).max() < 0.05 and np.random.randint(5):
                rand = np.random.randint(0, max(len(wav) - 1 - self.crop_length, 1))
                cropped_wav = wav[rand:rand + self.crop_length]

            wav = cropped_wav

        # make mask
        wav_mask = np.ones_like(wav)

        return wav, wav_mask

    def __len__(self):
        return len(self.meta_frame)

def get_datasets(meta_dir: str, batch_size: int, num_workers: int, crop_length: int, random_seed: int
                 ) -> Tuple[DataLoader, DataLoader]:
    assert os.path.isdir(meta_dir), '{} is not valid directory path!'

    train_file, valid_file = LJSpeechMeta.frame_file_names[1:]

    # load meta file
    train_meta = LJSpeechMeta(os.path.join(meta_dir, train_file))
    valid_meta = LJSpeechMeta(os.path.join(meta_dir, valid_file))

    # create dataset
    train_dataset = AudioDataset(train_meta, crop_length=crop_length, seed=random_seed)
    valid_dataset = AudioDataset(valid_meta, crop_length=crop_length, seed=random_seed)

    # create data loader
    train_loader = DataLoader(train_dataset, batch_size=batch_size, num_workers=num_workers, shuffle=True)
    valid_loader = DataLoader(valid_dataset, batch_size=batch_size, num_workers=num_workers)

    return train_loader, valid_loader


text:
"""Methods for plotting photometry and SEDs."""
from numbers import Number
from typing import Sequence, Tuple

import matplotlib.pyplot as plt
import numpy as np
import sncosmo

from lgsm import sed_utils

def plot_photometry(
    photometry: np.ndarray,
    bandpasses: Sequence[str],
    redshift: np.ndarray = np.zeros(1),
    fig_settings: dict = None,
    ax_settings: dict = None,
    scatter_settings: dict = None,
    ax: plt.Axes = None,
) -> Tuple[plt.Figure, plt.Axes]:
    """Plots the given photometry

    Parameters
    ----------
    photometry: np.ndarray
        Array of photometry to plot. If 2 dimensional, the second dimension
        must match the length of bandpasses
    bandpasses: Sequence[str]
        The names of the bandpasses for which the photometry was calculated
    redshift: np.ndarray, default=np.zeros(1)
        The redshift at which the photometry was observed. Used to shift
        photometry back to the restframe. Must contain one redshift or one
        redshift per per row in photometry.
    fig_settings: dict, optional
        Settings to pass to the subplots constructor
    ax_settings: dict, optional
        Settings to pass to ax.set()
    scatter_settings: dict, optional
        Settings to pass to ax.scatter()
    ax: plt.Axes, optional
        A matplotlib.pyplot Axes object to plot the photometry on. If not
        provided, new figure and axes are created.

    Returns
    -------
    plt.Figure
        A matplotlib.pyplot figure. If an ax was passed, this is None.
    plt.Axes
        The matplotlib.pyplot axes the photometry is plotted on.
    """

    # check that the redshift is valid
    redshift = np.array(redshift)
    if np.any(redshift < 0):
        raise ValueError("redshift must be non-negative")

    # assign empty dicts to settings that are None
    if fig_settings is None:
        fig_settings = {}
    if ax_settings is None:
        ax_settings = {}
    if scatter_settings is None:
        scatter_settings = {}

    # if ax is passed, don't pass fig_settings
    if ax is not None and len(fig_settings) > 0:
        raise ValueError("If ax passed, don't pass fig_settings.")

    # make sure arrays are correct shape
    photometry = photometry.reshape(-1, len(bandpasses))
    redshift = redshift.reshape(-1, 1)

    # get the effective wavelengths of the bandpasses
    wave = np.array(
        photometry.shape[0]
        * [[sncosmo.get_bandpass(band).wave_eff for band in bandpasses]]
    ) / (1 + redshift)

    # shift photometry to restframe
    photometry = photometry + 2.5 * np.log10(1 + redshift)

    # if ax not provided, we will make a new figure and axis
    if ax is None:
        fig, ax = plt.subplots(**fig_settings)
        ax.invert_yaxis()
    else:
        fig = None

    # plot the photometry
    ax.scatter(wave.flatten(), photometry.flatten(), **scatter_settings)

    # set the axis labels
    ax.set(xlabel="Wavelength (AA)", ylabel="AB Magnitude")

    # set user-supplied settings
    ax.set(**ax_settings)

    # if we made a new figure and axis, return them
    return fig, ax

def _plot_sed(
    wave: np.ndarray,
    sed: np.ndarray,
    redshift: float = 0,
    normalize_at: float = None,
    sed_unit: str = "mag",
    plot_unit: str = None,
    fig_settings: dict = None,
    ax_settings: dict = None,
    plot_settings: dict = None,
    ax: plt.Axes = None,
) -> Tuple[plt.Figure, plt.Axes]:

    # check that the redshift is valid
    if redshift < 0:
        raise ValueError("redshift must be non-negative.")
    # check that normalize at is valid
    if normalize_at is not None:
        if not isinstance(normalize_at, Number):
            raise ValueError("normalize_at must be None or a number.")
        if normalize_at < wave.min() or normalize_at > wave.max():
            raise ValueError("normalize_at must be within range of wave.")
    # check that sed_unit and plot_unit are valid
    units_allowed = ["mag", "flambda", "fnu"]
    if sed_unit not in units_allowed[:-1]:
        raise ValueError(f"sed_unit must be one of {', '.join(units_allowed[-1])}")
    if plot_unit is None:
        plot_unit = sed_unit
    if plot_unit not in units_allowed:
        raise ValueError(f"plot_unit must be one of {', '.join(units_allowed)}")

    # assign empty dicts to settings that are None
    if fig_settings is None:
        fig_settings = {}
    if ax_settings is None:
        ax_settings = {}
    if plot_settings is None:
        plot_settings = {}

    # redshift the sed
    wave = wave * (1 + redshift)
    if sed_unit == "mag":
        sed = sed - 2.5 * np.log10(1 + redshift)
    else:
        sed = sed / (1 + redshift)

    # (possibly) convert sed units to the plotting units
    if sed_unit != plot_unit:
        sed = getattr(sed_unit, f"{sed_unit}_to_{plot_unit}")(sed, wave=wave)
    # (possibly) convert wavelength to frequency
    if plot_unit == "fnu":
        # note if this is called, "wave" is actually an array of frequencies
        wave = sed_utils.wave_to_freq(wave)[::-1]

    # (possibly) normalize the SED at the requested location
    if normalize_at is not None:
        idx = np.argmin(np.abs(wave - normalize_at))
        if plot_unit == "mag":
            sed -= sed[idx]
        else:
            sed /= sed[idx]

    # if ax not provided, we will make a new figure and axis
    if ax is None:
        fig, ax = plt.subplots(**fig_settings)
        # if we are plotting mags, invert the y-axis
        if plot_unit == "mag":
            ax.invert_yaxis()
    else:
        fig = None

    # plot the sed
    ax.plot(wave, sed, **plot_settings)

    # set the axis labels depending on plot_unit
    if plot_unit == "mag":
        xlabel = "Wavelength (AA)"
        ylabel = "AB Magnitude"
    elif plot_unit == "flambda":
        xlabel = "Wavelength (AA)"
        ylabel = "Flux Density (erg/s/AA/cm^2)"
    else:
        xlabel = "Frequency (Hz)"
        ylabel = "Flux Density (erg/s/Hz/cm^2)"
    ax.set(xlabel=xlabel, ylabel=ylabel)

    # set the user-supplied axis settings
    ax.set(**ax_settings)

    return fig, ax

def plot_sed(
    wave: np.ndarray,
    sed: np.ndarray,
    redshift: np.ndarray = np.zeros(1),
    normalize_at: float = None,
    sed_unit: str = "mag",
    plot_unit: str = None,
    fig_settings: dict = None,
    ax_settings: dict = None,
    plot_settings: dict = None,
    ax: plt.Axes = None,
) -> Tuple[plt.Figure, plt.Axes]:
    """Plots the given seds

    Attempts to broadcast all of the inputs so you can plot many SEDs at once.

    Parameters
    ----------
    wave: np.ndarray
        The wavelength grid(s)
    sed: np.ndarray
        The SED(s)
    redshift: np.ndarray, default=np.zeros(1)
        The redshift at which to plot the SED.
    normalize_at: float, optional
        The wavelength at which to normalize the SEDs
    sed_unit: str, default="mag"
        The unit of the input seds. Can be "mag" or "flambda"
    plot_unit: str, optional
        The unit in which to plot the seds. Can be "mag", "flambda", or
        "fnu". If None, assumed to be same as sed_unit
    fig_settings: dict, optional
        Settings to pass to the subplots constructor
    ax_settings: dict, optional
        Settings to pass to ax.set()
    plot_settings: dict, optional
        Settings to pass to ax.scatter(). If doesn't exactly match number
        of SEDs to plot, these settings will define property cycles.
    ax: plt.Axes, optional
        A matplotlib.pyplot Axes object to plot the photometry on. If not
        provided, new figure and axes are created.

    Returns
    -------
    plt.Figure
        A matplotlib.pyplot figure. If an ax was passed, this is None.
    plt.Axes
        The matplotlib.pyplot axes the photometry is plotted on.
    """

    # assign empty dicts to settings that are None
    if fig_settings is None:
        fig_settings = {}
    if ax_settings is None:
        ax_settings = {}
    if plot_settings is None:
        plot_settings = {}

    # if ax is passed, don't pass fig_settings
    if ax is not None and len(fig_settings) > 0:
        raise ValueError("If ax passed, don't pass fig_settings.")

    # save the settings that all function calls receive
    settings = {
        "normalize_at": normalize_at,
        "sed_unit": sed_unit,
        "plot_unit": plot_unit,
        "fig_settings": fig_settings,
        "ax_settings": ax_settings,
    }

    # enforce correct shapes
    wave = np.atleast_2d(wave)
    sed = np.atleast_2d(sed)
    redshift = np.reshape(redshift, (-1, 1))

    # if multiple wavelength grid, seds, or redshifts we provided
    # then we need to plot in a loop over them
    if wave.shape[0] > 1 or sed.shape[0] > 1 or redshift.size > 1:

        # broadcast wave and sed to a common shape
        wave, sed = np.broadcast_arrays(wave, sed)

        # if only one redshift, broadcast to shape for wave, sed
        if redshift.size == 1:
            redshift = np.broadcast_to(redshift, (wave.shape[0], 1))
        # if multiple redshifts, plot each sed at each redshift
        else:
            wave, sed, redshift = (
                np.tile(wave, redshift.size).reshape(-1, wave.shape[1]),
                np.tile(sed, redshift.size).reshape(-1, sed.shape[1]),
                np.tile(redshift.flatten(), wave.shape[0]).reshape(-1, 1),
            )

        # broadcast plot_settings too
        for key, vals in plot_settings.items():
            vals = np.array([vals]).flatten()
            vals = np.tile(vals, np.ceil(redshift.size / vals.size).astype("int"))
            plot_settings[key] = vals

        # if no ax supplied, the first call will create an ax, which we
        # will pass to all subsequent calls. We will also return the new
        # fig and ax
        if ax is None:
            fig, ax = _plot_sed(
                wave=wave[0],
                sed=sed[0],
                redshift=redshift[0],
                plot_settings={
                    key: plot_settings[key][0] for key in plot_settings.keys()
                },
                **settings,
            )
            for i, (W, S, Z) in enumerate(zip(wave[1:], sed[1:], redshift[1:])):
                _plot_sed(
                    wave=W,
                    sed=S,
                    redshift=Z,
                    ax=ax,
                    plot_settings={
                        key: plot_settings[key][i + 1] for key in plot_settings.keys()
                    },
                    **settings,
                )
            return fig, ax
        # if an ax was supplied, we can do a simple loop and provide it every time
        else:
            for i, (W, S, Z) in enumerate(zip(wave, sed, redshift)):
                fig, ax = _plot_sed(
                    wave=W,
                    sed=S,
                    redshift=Z,
                    ax=ax,
                    plot_settings={
                        key: plot_settings[key][i] for key in plot_settings.keys()
                    },
                    **settings,
                )
            return fig, ax

    # otherwise, just plot the single set given
    else:

        return _plot_sed(
            wave=wave[0],
            sed=sed[0],
            redshift=redshift[0],
            ax=ax,
            plot_settings=plot_settings,
            **settings,
        )

max_stars_repo_name: jfcrenshaw/LGSM

max_issues_repo_name: jfcrenshaw/lgsm

max_forks_repo_name: jfcrenshaw/lgsm


text:
import torch
import torch.nn as nn
import torch.optim as optim
from torch.autograd import Variable
from baseline.pytorch.torchy import pytorch_prepare_optimizer
import numpy as np
from baseline.progress import create_progress_bar
from baseline.reporting import basic_reporting
from baseline.utils import listify, get_model_file
from baseline.train import Trainer, create_trainer
import time

class Seq2SeqTrainerPyTorch(Trainer):

    def __init__(self, model, **kwargs):
        super(Seq2SeqTrainerPyTorch, self).__init__()
        self.steps = 0
        self.gpu = bool(kwargs.get('gpu', True))
        self.clip = float(kwargs.get('clip', 5))
        self.model = model
        self.optimizer, self.scheduler = pytorch_prepare_optimizer(self.model, **kwargs)
        self._input = model.make_input
        self.crit = model.create_loss()
        if self.gpu:
            self.model = torch.nn.DataParallel(model).cuda()
            self.crit.cuda()

    def _total(self, tgt):
        tgtt = tgt.data.long()
        return torch.sum(tgtt.ne(0)).item()

    def test(self, vs, reporting_fns, phase):
        self.model.eval()
        metrics = {}
        total_loss = total = 0
        steps = len(vs)
        epochs = 0
        if phase == 'Valid':
            self.valid_epochs += 1
            epochs = self.valid_epochs

        pg = create_progress_bar(steps)
        for batch_dict in vs:
            fx = self._input(batch_dict)
            tgt = fx[-1]
            fx = fx[:-1]
            pred = self.model(fx)
            loss = self.crit(pred, tgt)
            total_loss += loss.item()
            total += self._total(tgt)
            pg.update()
        pg.done()

        avg_loss = float(total_loss)/total
        metrics['avg_loss'] = avg_loss
        metrics['perplexity'] = np.exp(avg_loss)
        for reporting in reporting_fns:
            reporting(metrics, epochs, phase)
        return metrics

    def train(self, ts, reporting_fns):
        self.model.train()

        metrics = {}

        total_loss = total = 0
        duration = 0
        if self.scheduler is not None:
            self.scheduler.step()

        for batch_dict in ts:

            start_time = time.time()
            self.steps += 1
            self.optimizer.zero_grad()
            fx = self._input(batch_dict)
            tgt = fx[-1]
            fx = fx[:-1]
            pred = self.model(fx)
            loss = self.crit(pred, tgt)
            total_loss += loss.item()
            loss.backward()
            torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.clip)
            total += self._total(tgt)
            self.optimizer.step()
            duration += time.time() - start_time

            if self.steps % 500 == 0:
                print('Step time (%.3f sec)' % (duration / 500.))
                duration = 0
                avg_loss = float(total_loss)/total
                metrics['avg_loss'] = avg_loss
                metrics['perplexity'] = np.exp(avg_loss)
                for reporting in reporting_fns:
                    reporting(metrics, self.steps, 'Train')

        self.train_epochs += 1
        avg_loss = float(total_loss)/total
        metrics['avg_loss'] = avg_loss
        metrics['perplexity'] = np.exp(avg_loss)
        for reporting in reporting_fns:
            reporting(metrics, self.steps, 'Train')

        return metrics

def fit(model, ts, vs, es=None, **kwargs):

    do_early_stopping = bool(kwargs.get('do_early_stopping', True))
    epochs = int(kwargs.get('epochs', 20))
    model_file = get_model_file(kwargs, 'seq2seq', 'pytorch')

    if do_early_stopping:
        early_stopping_metric = kwargs.get('early_stopping_metric', 'avg_loss')
        patience = kwargs.get('patience', epochs)
        print('Doing early stopping on [%s] with patience [%d]' % (early_stopping_metric, patience))

    reporting_fns = listify(kwargs.get('reporting', basic_reporting))
    print('reporting', reporting_fns)

    after_train_fn = kwargs.get('after_train_fn', None)
    trainer = create_trainer(Seq2SeqTrainerPyTorch, model, **kwargs)

    min_metric = 10000
    last_improved = 0
    for epoch in range(epochs):

        trainer.train(ts, reporting_fns)

        if after_train_fn is not None:
            after_train_fn(model)

        test_metrics = trainer.test(vs, reporting_fns, phase='Valid')

        if do_early_stopping is False:
            model.save(model_file)

        elif test_metrics[early_stopping_metric] < min_metric:
            last_improved = epoch
            min_metric = test_metrics[early_stopping_metric]
            print('New min %.3f' % min_metric)
            model.save(model_file)

        elif (epoch - last_improved) > patience:
            print('Stopping due to persistent failures to improve')
            break

    if do_early_stopping is True:
        print('Best performance on min_metric %.3f at epoch %d' % (min_metric, last_improved))

    if es is not None:
        model.load(model_file)
        trainer = Seq2SeqTrainerPyTorch(model, **kwargs)
        trainer.test(es, reporting_fns, phase='Test')

max_issues_repo_name: ZhenyueChin/baseline

max_forks_repo_name: ZhenyueChin/baseline


text:
# Copyright 2014 Diamond Light Source Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
.. module:: strip_background
   :platform: Unix
   :synopsis: A plugin to automatically strip peaks and subtract background

.. moduleauthor:: Aaron D. Parsons <scientificsoftware@diamond.ac.uk>
"""
import logging
from scipy.signal import savgol_filter
import numpy as np
from savu.plugins.filters.base_filter import BaseFilter
from savu.plugins.driver.cpu_plugin import CpuPlugin
import time
from savu.plugins.utils import register_plugin
from copy import deepcopy

@register_plugin
class StripBackground(BaseFilter, CpuPlugin):

    def __init__(self):
        logging.debug("Stripping background")
        super(StripBackground, self).__init__("StripBackground")

    def process_frames(self, data):
        data = data[0]
        t1 = time.time()
        its = self.parameters['iterations']
        w = self.parameters['window']
        smoothed = self.parameters['SG_filter_iterations']
        SGwidth = self.parameters['SG_width']
        SGpoly = self.parameters['SG_polyorder']

        npts = len(data)
        x = np.arange(npts)  # set up some x indices
        #print data.shape
        filtered = savgol_filter(data, 35, 5) # make the start a bit a bit smoother
        # lets do it the crap, slow way first
        aved = np.zeros_like(filtered)
        bottomedgemain = x < w
        bottomedgerest = (x >= w) & (x < 2*w)

        mainpart = (x >= w) & (x < (npts-w))
        mainpartbottom = (x >= 0) & (x < (npts-2*w))
        mainparttop = (x >= 2*w) & (x < (npts))

        topedgemain = x >= (npts-w)
        topedgerest = (x >= (npts-2*w)) & (x >= (npts-w))

        for k in range(its):
            aved[mainpart] = (filtered[mainpartbottom] + filtered[mainpart] + filtered[mainparttop]) / 3.0
            aved[bottomedgemain] = (filtered[bottomedgemain] + filtered[bottomedgerest]) / 2.0
            aved[topedgemain] = (filtered[topedgemain] + filtered[topedgerest]) / 2.0
            filtered[aved<filtered] = aved[aved<filtered]
            if not (k/float(smoothed)-k/int(smoothed)):
                filtered=savgol_filter(filtered,35,5)

        t2 = time.time()
        logging.debug("Strip iteration took: %s ms", str((t2-t1)*1e3))
#         print (data - filtered).shape
        return [data - filtered, filtered]

    def setup(self):
        logging.debug('setting up the background subtraction')
        in_dataset, out_datasets = self.get_datasets()
        #print in_dataset, out_datasets

        in_meta = in_dataset[0].meta_data
        in_dictionary = in_meta.get_dictionary()
        #print in_dictionary
        stripped = out_datasets[0]
        stripped.create_dataset(in_dataset[0])
        stripped.meta_data.dict = deepcopy(in_dictionary)
        #print stripped.meta_data.dict
        background = out_datasets[1]
        background.create_dataset(in_dataset[0])
        background.meta_data.dict = deepcopy(in_dictionary)

        in_pData, out_pData = self.get_plugin_datasets()
        in_pData[0].plugin_data_setup('SPECTRUM', self.get_max_frames())
        out_pData[0].plugin_data_setup('SPECTRUM', self.get_max_frames())
        out_pData[1].plugin_data_setup('SPECTRUM', self.get_max_frames())

        logging.debug("****STRIP_BACKGROUND AXIS LABELS*** %s",
                      stripped.get_axis_labels())

    def get_max_frames(self):
        return 'single'

    def nOutput_datasets(self):
        return 2

max_stars_repo_name: nghia-vo/Savu

max_issues_repo_name: nghia-vo/Savu

max_forks_repo_name: nghia-vo/Savu


text:
import json
import numpy as np
import os

def main():
    data_path = os.path.join(".", "data")
    input_path = os.path.join(data_path, "sp500_constituents.csv")
    output_path = os.path.join(data_path, "sp500.json")

    sp500 = np.genfromtxt(input_path,
                          dtype=None,
                          delimiter=",",
                          names=True,
                          encoding="utf8")

    filtr = sp500[:]["conm"] == "S&P 500 Comp-Ltd"
    sp500 = sp500[filtr]
    filtr = sp500[:]["thru"] == ""
    sp500 = sp500[filtr]

    sp500_dict = dict()
    sp500_dict["sp500"] = sp500["co_tic"].tolist()

    with open(output_path, "w") as json_file:
        json.dump(sp500_dict, json_file, indent=4)

if __name__ == "__main__":
    main()


text:
"""Transforms for Panoptic FPN."""
from __future__ import absolute_import

import copy
from random import randint

import mxnet as mx
import numpy as np

from .. import bbox as tbbox
from .. import image as timage
from .. import mask as tmask

__all__ = ['transform_test', 'load_test',
           'PanopticFPNDefaultTrainTransform', 'PanopticFPNDefaultValTransform']

def transform_test(imgs, short=600, max_size=1000, mean=(0.485, 0.456, 0.406),
                   std=(0.229, 0.224, 0.225)):
    """A util function to transform all images to tensors as network input by applying
    normalizations. This function support 1 NDArray or iterable of NDArrays.

    Parameters
    ----------
    imgs : NDArray or iterable of NDArray
        Image(s) to be transformed.
    short : int, optional, default is 600
        Resize image short side to this `short` and keep aspect ratio.
    max_size : int, optional, default is 1000
        Maximum longer side length to fit image.
        This is to limit the input image shape, avoid processing too large image.
    mean : iterable of float
        Mean pixel values.
    std : iterable of float
        Standard deviations of pixel values.

    Returns
    -------
    (mxnet.NDArray, numpy.ndarray) or list of such tuple
        A (1, 3, H, W) mxnet NDArray as input to network, and a numpy ndarray as
        original un-normalized color image for display.
        If multiple image names are supplied, return two lists. You can use
        `zip()`` to collapse it.

    """
    if isinstance(imgs, mx.nd.NDArray):
        imgs = [imgs]
    for im in imgs:
        assert isinstance(im, mx.nd.NDArray), "Expect NDArray, got {}".format(type(im))

    tensors = []
    origs = []
    for img in imgs:
        img = timage.resize_short_within(img, short, max_size)
        orig_img = img.asnumpy().astype('uint8')
        img = mx.nd.image.to_tensor(img)
        img = mx.nd.image.normalize(img, mean=mean, std=std)
        tensors.append(img.expand_dims(0))
        origs.append(orig_img)
    if len(tensors) == 1:
        return tensors[0], origs[0]
    return tensors, origs

def load_test(filenames, short=600, max_size=1000, mean=(0.485, 0.456, 0.406),
              std=(0.229, 0.224, 0.225)):
    """A util function to load all images, transform them to tensor by applying
    normalizations. This function support 1 filename or list of filenames.

    Parameters
    ----------
    filenames : str or list of str
        Image filename(s) to be loaded.
    short : int, optional, default is 600
        Resize image short side to this `short` and keep aspect ratio.
    max_size : int, optional, default is 1000
        Maximum longer side length to fit image.
        This is to limit the input image shape, avoid processing too large image.
    mean : iterable of float
        Mean pixel values.
    std : iterable of float
        Standard deviations of pixel values.

    Returns
    -------
    (mxnet.NDArray, numpy.ndarray) or list of such tuple
        A (1, 3, H, W) mxnet NDArray as input to network, and a numpy ndarray as
        original un-normalized color image for display.
        If multiple image names are supplied, return two lists. You can use
        `zip()`` to collapse it.

    """
    if isinstance(filenames, str):
        filenames = [filenames]
    imgs = [mx.image.imread(f) for f in filenames]
    return transform_test(imgs, short, max_size, mean, std)

class PanopticFPNDefaultTrainTransform(object):
    def __init__(self, net=None, dataset="citys_panoptic", mean=(0.485, 0.456, 0.406),
                 std=(0.229, 0.224, 0.225), box_norm=(1., 1., 1., 1.), num_sample=256,
                 pos_iou_thresh=0.7,neg_iou_thresh=0.3, pos_ratio=0.5, ashape=128,
                 multi_stage=True, down_sample_rate=4, **kwargs):
        self._mean = mean
        self._std = std
        self._dataset = dataset
        self._anchors = None
        self._multi_stage = multi_stage
        self.down_sample_rate = down_sample_rate
        if net is None:
            return

        # use fake data to generate fixed anchors for target generation
        anchors = []  # [P2, P3, P4, P5]
        # in case network has reset_ctx to gpu
        anchor_generator = copy.deepcopy(net.rpn.anchor_generator)
        anchor_generator.collect_params().reset_ctx(None)
        if self._multi_stage:
            for ag in anchor_generator:
                anchor = ag(mx.nd.zeros((1, 3, ashape, ashape))).reshape((1, 1, ashape, ashape, -1))
                ashape = max(ashape // 2, 16)
                anchors.append(anchor)
        self._anchors = anchors
        # record feature extractor for infer_shape
        if not hasattr(net, 'features'):
            raise ValueError("Cannot find features in network, it is a Mask RCNN network?")
        self._feat_sym = net.features(mx.sym.var(name='data'))
        from ....model_zoo.rpn.rpn_target import RPNTargetGenerator
        self._target_generator = RPNTargetGenerator(
            num_sample=num_sample, pos_iou_thresh=pos_iou_thresh,
            neg_iou_thresh=neg_iou_thresh, pos_ratio=pos_ratio,
            stds=box_norm, **kwargs)

    def __call__(self, img, segms, inst, bbox):
        '''
        src : [H, W, 3]
        segms : [H, W]
        inst : [N, H, W]
        bbox: [N, 5]
        '''
        """Apply transform to training image/label."""
        h, w, _ = img.shape
        # to tensor
        img = mx.nd.image.to_tensor(img)
        img = mx.nd.image.normalize(img, mean=self._mean, std=self._std)
        sh = img.shape[1] // self.down_sample_rate
        sw = img.shape[2] // self.down_sample_rate
        segms = mx.image.imresize(segms.expand_dims(-1), sw, sh, interp=0)
        segms = segms.squeeze()

        # generate RPN target so cpu workers can help reduce the workload
        # feat_h, feat_w = (img.shape[1] // self._stride, img.shape[2] // self._stride)
        gt_bboxes = mx.nd.array(bbox[:, :4])
        if self._multi_stage:
            oshapes = []
            anchor_targets = []
            for feat_sym in self._feat_sym:
                oshapes.append(feat_sym.infer_shape(data=(1, 3, img.shape[1], img.shape[2]))[1][0])
            for anchor, oshape in zip(self._anchors, oshapes):
                anchor = anchor[:, :, :oshape[2], :oshape[3], :].reshape((-1, 4))
                anchor_targets.append(anchor)
            anchor_targets = mx.nd.concat(*anchor_targets, dim=0)
            cls_target, box_target, box_mask = self._target_generator(
                gt_bboxes, anchor_targets, img.shape[2], img.shape[1])
        return img, bbox.astype(img.dtype), segms, inst, cls_target, box_target, box_mask

max_stars_repo_name: Angzz/panoptic-fpn-gluon

max_issues_repo_name: mornydew/panoptic-fpn-gluon

max_forks_repo_name: mornydew/panoptic-fpn-gluon


text:
# Developed since: May 2020

import random
import unittest
import numpy
import upy2
from upy2 import u, U, undarray
from upy2.numpy_operators import UnaryOperator, BinaryOperator

class NotCloseError(AssertionError):
    """ Some numbers are not close enough to each other. """

    pass

class DerivativeApproximationError(AssertionError):
    """ Calculating a derivative approximation numerically failed. """

    pass

class DerivativeVerificationError(AssertionError):
    """ The numerically aproximated derivative did not match the
    symbolical prediction. """

    pass

class TestOperators(unittest.TestCase):
    def __init__(self, name):
        unittest.TestCase.__init__(self, name)
        self.U = U(1)

    def setUp(self):
        self.U.register()

    def tearDown(self):
        self.U.unregister()

    # Providers for tuples ``(nominal value, uncertainty)``.  The
    # ``uncertainty`` is a normally distributed random variable
    # always; there are several options for the distribution of the
    # ``nominal value`` ...

    def gauss(self):
        return (random.gauss(0, 1), random.gauss(0, 1))

    def expo(self):
        return (random.expovariate(1), random.gauss(0, 1))

    def in01(self):
        return (random.random(), random.gauss(0, 1))

    def inpm01(self):
        return (random.choice([-1, 1]) * random.random(), random.gauss(0, 1))

    def above1(self):
        return (1 + random.expovariate(1), random.gauss(0, 1))

    # Providers for inspection positions during testing
    # differentiation ...

    def cgauss(self):
        return random.gauss(0, 1) + 1j * random.gauss(0, 1)

    # Custom Assertions ...

    def assertClose(self, a, b):
        if not numpy.allclose(a, b):
            raise NotCloseError('{} not close to {}'.format(a, b))

    def assertDerivative(self, specimen, position, prediction,
            epsilon, epsilonfactor=None):
        """ *specimen* is the function whose derivative is to be
        approximated numerically at position *position* using a small
        displacement *epsilon*.  *prediction* is the prediction of the
        derivative of *specimen* at *position* using symbolic
        differentiation.  To assess the precision of the numerical
        differentiation, it is carried out using *two* displacements,
        (a) with displacement ``epsilon`` and (b) with displacement
        ``epsilon * epsilonfactor``.  *epsilonfactor* defaults to
        ``1j``. """

        if epsilonfactor is None:
            epsilonfactor = 1j

        position0 = position
        position1 = position + epsilon
        position1j = position + epsilon * epsilonfactor

        value0 = specimen(position0)
        value1 = specimen(position1)
        value1j = specimen(position1j)

        approximation1 = (value1 - value0) / epsilon
        approximation1j = (value1j - value0) / (epsilon * epsilonfactor)

        mean_approximation = (approximation1 + approximation1j) / 2
        approximation_uncertainty = abs(approximation1j -
                approximation1) + 1e-10 * abs(mean_approximation)
            # The relative uncertainty accounts for the finite
            # resolution of floating point numbers.

        relative_approximation_uncertainty = \
                approximation_uncertainty / abs(mean_approximation)
        if relative_approximation_uncertainty > 0.01:
            raise DerivativeApproximationError(
                    ("The approximation of {specimen} "
                     "couln't be approximated successfully at "
                     "{position}.  Relative approximation uncertainty: "
                     "{rel:.3f}").\
                    format(specimen=specimen, position=position,
                        rel=relative_approximation_uncertainty)
            )

        difference = numpy.abs(prediction - mean_approximation)
        if difference > 2 * approximation_uncertainty:
            raise DerivativeVerificationError(
                ('Could not verify the derivative of '
                 '{specimen} at {position}.  Prediction: {prediction}, '
                 'approximation: {approximation} with uncertainty '
                 '{uncertainty}, Excess: {excess:.2f}').\
                format(specimen=specimen, position=position,
                    prediction=prediction,
                    approximation=mean_approximation,
                    uncertainty=approximation_uncertainty,
                    excess=(difference / approximation_uncertainty))
            )

    # Testing assertions ...

    def test_assertions(self):
        with self.assertRaises(NotCloseError):
            self.assertClose(1, 2)

        with self.assertRaises(DerivativeApproximationError):
            random.seed(1812)
            for i in range(100):
                z = self.cgauss()
                self.assertDerivative(
                    specimen=(lambda z:
                        numpy.sin(z.real) + numpy.cos(z.imag)),
                    position=z,
                    prediction=42,
                    epsilon=1e-4,
                )

        with self.assertRaises(DerivativeVerificationError):
            random.seed(1815)
            for i in range(100):
                z = self.cgauss()
                self.assertDerivative(
                    specimen=numpy.cos,
                    position=z,
                    prediction=42,
                    epsilon=1e-4,
                )

        # Test successful differentiation:
        random.seed(1819)
        for i in range(100):
            z = self.cgauss()
            self.assertDerivative(
                specimen=numpy.cos,
                position=z,
                prediction=(-numpy.sin(z)),
                epsilon=1e-4,
            )

    # Testing construction ...

    def test_construction(self):
        random.seed( 658)
        for i in range(10):
            nom, unc = self.gauss()

            ua1 = nom +- u(unc)
            ua2 = nom + u(unc)
            ua3 = nom - u(unc)
            ua4 = undarray(nominal=nom, stddev=unc)
            ua5 = undarray(nominal=nom, stddev=(-unc))

            self.assertClose(ua1.stddev, abs(unc))
            self.assertClose(ua2.stddev, abs(unc))
            self.assertClose(ua3.stddev, abs(unc))
            self.assertClose(ua4.stddev, abs(unc))
            self.assertClose(ua5.stddev, abs(unc))

    # Testing the operators ...

    def test_positive(self):
        random.seed( 646)
        for i in range(10):
            nom, unc = self.gauss()
            ua = nom +- u(unc)
            ub = +ua
            self.assertClose(ub.stddev, abs(unc))

    def test_unegative(self):
        random.seed(1)
        for i in range(0, 10):
            nom, unc = self.gauss()
            ua1 = nom +- u(unc)
            ua2 = -ua1

            self.assertClose(ua1.stddev, abs(unc))
            self.assertClose(ua2.stddev, abs(unc))

    def test_absolute(self):
        ua1 = 1 +- u(0.1)
        self.assertClose(abs(ua1).stddev, 0.1)

        ua2 = 1 +- u(0.1j)
        self.assertClose(abs(ua2).stddev, 0)

        ua3 = 1j +- u(0.1j)
        self.assertClose(numpy.abs(ua3).stddev, 0.1)

        ua4 = -1 +- u(0.1 + 0.1j)
        self.assertClose(abs(ua4).stddev, 0.1)

        ua5 = -1j +- u(0.1 - 0.1j)
        self.assertClose(abs(ua5).stddev, 0.1)

        ua6 = 1j +- u(0.1)
        self.assertClose(abs(ua6).stddev, 0)

    def test_sqrt(self):
        random.seed( 658)
        for i in range(10):
            nom, unc = self.expo()
            ua = nom +- u(unc)
            ub = numpy.sqrt(ua)

            self.assertClose(ub.stddev, abs(unc) * 0.5 * nom ** (-0.5))

        random.seed(1953)
        for i in range(100):
            z = self.cgauss()
            self.assertDerivative(
                    specimen=numpy.sqrt, position=z,
                    prediction=(0.5 * z ** (-0.5)),
                    epsilon=1e-4,
            )

    def test_square(self):
        random.seed( 700)
        for i in range(10):
            nom, unc = self.gauss()
            ua = nom +- u(unc)
            ub = numpy.square(ua)

            self.assertClose(ub.stddev, abs(unc) * 2 * abs(nom))

        random.seed(1955)
        for i in range(100):
            z = self.cgauss()
            self.assertDerivative(
                    specimen=numpy.square, position=z,
                    prediction=(2 * z),
                    epsilon=1e-4,
            )

    def test_sin(self):
        random.seed(2)
        for i in range(10):
            nom, unc = self.gauss()
            ua1 = undarray(nominal=nom, stddev=unc)
            ua2 = numpy.sin(ua1)

            self.assertClose(ua2.stddev, abs(unc * numpy.cos(nom)))

        random.seed(1656)
        for i in range(100):
            z = self.cgauss()
            self.assertDerivative(
                    specimen=numpy.sin, position=z,
                    prediction=numpy.cos(z),
                    epsilon=1e-4,
            )

    def test_cos(self):
        random.seed(1632)
        for i in range(10):
            nom, unc = self.gauss()
            ua1 = nom +- u(unc)
            ua2 = numpy.cos(ua1)

            self.assertClose(ua2.stddev, abs(-unc * numpy.sin(nom)))

        random.seed(1957)
        for i in range(100):
            z = self.cgauss()
            self.assertDerivative(
                    specimen=numpy.cos, position=z,
                    prediction=(-numpy.sin(z)),
                    epsilon=1e-4,
            )

    def test_tan(self):
        random.seed( 702)
        for i in range(10):
            nom, unc = self.gauss()
            ua = nom +- u(unc)
            ub = numpy.tan(ua)

            self.assertClose(ub.stddev, abs(unc) * (1 + numpy.tan(nom) ** 2))

        random.seed(1959)
        for i in range(100):
            z = self.cgauss()
            self.assertDerivative(
                    specimen=numpy.tan, position=z,
                    prediction=(1 + numpy.tan(z) ** 2),
                    epsilon=1e-4,
            )

    def test_arcsin(self):
        random.seed( 704)
        for i in range(10):
            nom, unc = self.inpm01()
            ua = nom +- u(unc)
            ub = numpy.arcsin(ua)

            self.assertClose(ub.stddev, abs(unc) / numpy.sqrt(1 - nom ** 2))

        random.seed(2001)
        for i in range(100):
            z = self.cgauss()
            self.assertDerivative(
                    specimen=numpy.arcsin, position=z,
                    prediction=(1 / numpy.sqrt(1 - z ** 2)),
                    epsilon=1e-4,
            )

    def test_arccos(self):
        random.seed( 709)
        for i in range(10):
            nom, unc = self.inpm01()
            ua = nom +- u(unc)
            ub = numpy.arccos(ua)

            self.assertClose(ub.stddev, abs(unc) / numpy.sqrt(1 - nom ** 2))

        random.seed(2002)
        for i in range(100):
            z = self.cgauss()
            self.assertDerivative(
                    specimen=numpy.arccos, position=z,
                    prediction=(-1 / numpy.sqrt(1 - z ** 2)),
                    epsilon=1e-4,
            )

    def test_arctan(self):
        random.seed( 710)
        for i in range(10):
            nom, unc = self.gauss()
            ua = nom +- u(unc)
            ub = numpy.arctan(ua)

            self.assertClose(ub.stddev, abs(unc) / (1 + nom ** 2))

        random.seed(2004)
        for i in range(100):
            z = self.cgauss()
            self.assertDerivative(
                    specimen=numpy.arctan, position=z,
                    prediction=(1 / (1 + z ** 2)),
                    epsilon=1e-4,
            )

    def test_sinh(self):
        random.seed( 712)
        for i in range(10):
            nom, unc = self.gauss()
            ua = nom +- u(unc)
            ub = numpy.sinh(ua)

            self.assertClose(ub.stddev, abs(unc * numpy.cosh(nom)))

        random.seed( 632)
        for i in range(100):
            z = self.cgauss()
            self.assertDerivative(
                    specimen=numpy.sinh, position=z,
                    prediction=numpy.cosh(z),
                    epsilon=1e-4,
            )

    def test_cosh(self):
        random.seed( 715)
        for i in range(10):
            nom, unc = self.gauss()
            ua = nom +- u(unc)
            ub = numpy.cosh(ua)

            self.assertClose(ub.stddev, abs(unc * numpy.sinh(nom)))

        random.seed( 633)
        for i in range(100):
            z = self.cgauss()
            self.assertDerivative(
                    specimen=numpy.cosh, position=z,
                    prediction=numpy.sinh(z),
                    epsilon=1e-4,
            )

    def test_tanh(self):
        random.seed( 716)
        for i in range(10):
            nom, unc = self.gauss()
            ua = nom +- u(unc)
            ub = numpy.tanh(ua)

            self.assertClose(ub.stddev, abs(unc) * (1 - numpy.tanh(nom) ** 2))

        random.seed( 636)
        for i in range(100):
            z = self.cgauss()
            self.assertDerivative(
                    specimen=numpy.tanh, position=z,
                    prediction=(1 - numpy.tanh(z) ** 2),
                    epsilon=1e-4,
            )

    def test_arcsinh(self):
        random.seed( 723)
        for i in range(10):
            nom, unc = self.gauss()
            ua = nom +- u(unc)
            ub = numpy.arcsinh(ua)

            self.assertClose(ub.stddev, abs(unc) / numpy.sqrt(nom ** 2 + 1))

        random.seed( 637)
        for i in range(100):
            z = self.cgauss()
            self.assertDerivative(
                    specimen=numpy.arcsinh, position=z,
                    prediction=(1 / numpy.sqrt(z ** 2 + 1)),
                    epsilon=1e-4,
            )

    @unittest.expectedFailure
    def test_arccosh(self):
        random.seed( 730)
        for i in range(10):
            nom, unc = self.above1()
            ua = nom +- u(unc)
            ub = numpy.arccosh(ua)

            self.assertClose(ub.stddev, abs(unc) / numpy.sqrt(nom ** 2 - 1))

        # This succeeds:
        random.seed( 644)
        for i in range(100):
            x = 1 + random.expovariate(1)
            self.assertDerivative(
                    specimen=numpy.arccosh, position=x,
                    prediction=(1 / numpy.sqrt(x ** 2 - 1)),
                    epsilon=1e-4,
            )

        # This fails:
        random.seed( 656)
        for i in range(100):
            x = random.gauss(0, 1) + 0j
            self.assertDerivative(
                    specimen=numpy.arccosh, position=x,
                    prediction=(1 / numpy.sqrt(x ** 2 - 1)),
                    epsilon=1e-5,  # There is a case with x very close
                        # to 1.0, requiring this precision.  (With ``+
                        # 0j`` in the definition of *x*, an assertion
                        # is violated before reaching this case.)
            )

        # This fails:
        random.seed( 640)
        for i in range(100):
            z = self.cgauss()
            self.assertDerivative(
                    specimen=numpy.arccosh, position=z,
                    prediction=(1 / numpy.sqrt(z ** 2 - 1)),
                    #prediction=(1 / (z ** 2 - 1) ** 0.5),
                    epsilon=1e-4,
            )

    def test_arctanh(self):
        random.seed( 734)
        for i in range(10):
            nom, unc = self.inpm01()
            ua = nom +- u(unc)
            ub = numpy.arctanh(ua)

            self.assertClose(ub.stddev, abs(unc) / (1 - nom ** 2))

        random.seed(1346)
        for i in range(100):
            z = self.cgauss()
            self.assertDerivative(
                    specimen=numpy.arctanh, position=z,
                    prediction=(1 / (1 - z **2)),
                    epsilon=1e-4,
            )

    def test_exp(self):
        random.seed(1638)
        for i in range(10):
            nom, unc = self.gauss()
            ua1 = nom + u(unc)
            ua2 = numpy.exp(ua1)

            # y = exp(x) > 0
            # d_x y = exp(x)
            # u(y) = u(x) abs[d_x y] = u(x) * exp(x) = u(x) y
            # u(y) / y = u(x)
            # The absolute uncertainty of x turns out as the same as
            # the relative uncertainty of y.

            self.assertClose(ua2.stddev / ua2.nominal, abs(unc))

        random.seed(1348)
        for i in range(100):
            z = self.cgauss()
            self.assertDerivative(
                    specimen=numpy.exp, position=z,
                    prediction=(numpy.e ** z),
                    epsilon=1e-4,
            )

    def test_exp2(self):
        random.seed( 737)
        for i in range(10):
            nom, unc = self.gauss()
            ua = nom +- u(unc)
            ub = numpy.exp2(ua)

            # y = 2 ** x = e ** (ln 2 * x)
            # d_x y = ln 2 * 2 ** x

            self.assertClose(ub.stddev, abs(unc) * numpy.log(2) * 2 ** nom)

        random.seed(1349)
        for i in range(100):
            z = self.cgauss()
            self.assertDerivative(
                    specimen=numpy.exp2, position=z,
                    prediction=(numpy.log(2) * 2 ** z),
                    epsilon=1e-4,
            )

    def test_log(self):
        random.seed(1648)
        for i in range(10):
            nom, unc = self.expo()
            ua1 = nom +- u(unc)
            ua2 = numpy.log(ua1)

            # y = ln x
            # d_x y = 1 / x
            # u(y) = u(x) / x
            # The relative uncertainty of x is the absolute
            # uncertainty of y.

            self.assertClose(ua2.stddev, ua1.stddev / ua1.nominal)

        random.seed(1351)
        for i in range(100):
            z = self.cgauss()
            self.assertDerivative(
                    specimen=numpy.log, position=z,
                    prediction=(1 / z),
                    epsilon=1e-4,
            )

    def test_log2(self):
        random.seed(1653)
        for i in range(10):
            nom, unc = self.expo()
            ua1 = nom +- u(unc)
            ua2 = numpy.log2(ua1)

            # y = ln x / ln 2
            # u(y) = 1/ln 2 u(x) / x

            self.assertClose(ua2.stddev,
                    1/numpy.log(2) * ua1.stddev / ua1.nominal)

        random.seed(1352)
        for i in range(100):
            z = self.cgauss()
            self.assertDerivative(
                    specimen=numpy.log2, position=z,
                    prediction=(1 / numpy.log(2) / z),
                    epsilon=1e-4,
            )

    def test_log10(self):
        random.seed(1655)
        for i in range(10):
            nom, unc = self.expo()
            ua1 = nom +- u(unc)
            ua2 = numpy.log10(ua1)

            # y = ln x / ln 10
            # u(y) = 1/ln 10 u(x) / x

            self.assertClose(ua2.stddev,
                    1/numpy.log(10) * ua1.stddev / ua1.nominal)

        random.seed(1354)
        for i in range(100):
            z = self.cgauss()
            self.assertDerivative(
                    specimen=numpy.log10, position=z,
                    prediction=(1 / numpy.log(10) / z),
                    epsilon=1e-4,
            )

    def test_add(self):
        random.seed( 707)
        for i in range(10):
            nom1, unc1 = self.gauss()
            nom2, unc2 = self.gauss()
            ua1 = nom1 +- u(unc1)
            ua2 = nom2 +- u(unc2)

            uy = ua1 + ua2

            self.assertClose(uy.variance, ua1.variance + ua2.variance)

        random.seed(1406)
        for i in range(100):
            z1 = self.cgauss()
            z2 = self.cgauss()

            self.assertDerivative(
                    specimen=(lambda z: numpy.add(z, z2)), position=z1,
                    prediction=1,
                    epsilon=1e-4,
            )
            self.assertDerivative(
                    specimen=(lambda z: numpy.add(z1, z)), position=z2,
                    prediction=1,
                    epsilon=1e-4,
            )

    def test_subtract(self):
        random.seed( 710)
        for i in range(10):
            nom1, unc1 = self.gauss()
            nom2, unc2 = self.gauss()
            ua1 = nom1 +- u(unc1)
            ua2 = nom2 +- u(unc2)

            uy = ua1 - ua2

            self.assertClose(uy.variance, ua1.variance + ua2.variance)

        random.seed(1409)
        for i in range(100):
            z1 = self.cgauss()
            z2 = self.cgauss()

            self.assertDerivative(
                    specimen=(lambda z: numpy.subtract(z, z2)), position=z1,
                    prediction=1,
                    epsilon=1e-4,
            )
            self.assertDerivative(
                    specimen=(lambda z: numpy.subtract(z1, z)), position=z2,
                    prediction=-1,
                    epsilon=1e-4,
            )

    def test_multiply(self):
        random.seed( 712)
        for i in range(10):
            nom1, unc1 = self.gauss()
            nom2, unc2 = self.gauss()
            ua1 = nom1 +- u(unc1)
            ua2 = nom2 +- u(unc2)

            uy = ua1 * ua2
            # y = x1 * x2
            #
            # d_x1 y = x2
            # d_x2 y = x1
            #
            # Var[y] = x2^2 Var[x1] + x1^2 Var[x2]
            #
            # u(y) = sqrt(x2^2 u(x1)^2 + x1^2 u(x2)^2)
            # u(y) / y = u(y) / (x1 x2)
            #   = sqrt([u(x1) / x1]^2 + [u(x2) / x2]^2)
            #
            # The relative uncertainties add in an rms. manner.
            # Furthermore:
            #
            # [u(y) / y]^2 = [u(x1) / x1]^2 + [u(x2) / x2]^2
            #
            # Var[y]/(y^2) = Var[x1]/(x1^2) + Var[x2]/(x2^2)

            self.assertClose(uy.variance,
                    nom2 ** 2 * ua1.variance +
                    nom1 ** 2 * ua2.variance)
            self.assertClose((uy.stddev / uy.nominal) ** 2,
                        (ua1.stddev / ua1.nominal) ** 2 +
                        (ua2.stddev / ua2.nominal) ** 2)

        random.seed(1402)
        for i in range(100):
            z1 = self.cgauss()
            z2 = self.cgauss()

            self.assertDerivative(
                    specimen=(lambda z: numpy.multiply(z, z2)), position=z1,
                    prediction=z2,
                    epsilon=1e-4,
            )
            self.assertDerivative(
                    specimen=(lambda z: numpy.multiply(z1, z)), position=z2,
                    prediction=z1,
                    epsilon=1e-4,
            )

    def test_divide(self):
        random.seed( 723)
        for i in range(10):
            nom1, unc1 = self.gauss()
            nom2, unc2 = self.gauss()
            ua1 = nom1 +- u(unc1)
            ua2 = nom2 +- u(unc2)

            uy = ua1 / ua2
            # y = x1 / x2
            #
            # d_x1 y = 1/x2 = y/x1
            # d_x2 y = -x1 / x2^2 = -y/x2
            #
            # Var[y] = Var[x1] [d_x1 y]^2 + Var[x2] [d_x2 y]^2
            #   = y^2 Var[x1]/(x1^2) + y^2 Var[x2]/(x2^2)
            #
            # Thus:
            #
            # Var[y]/(y^2) = Var[x1]/(x1^2) + Var[x2]/(x2^2)
            #
            # or:
            #
            # (u(y) / y)^2 = (u(x1) / x1)^2 + (u(x2) / x2)^2
            #
            # just as in the multiplication case.

            self.assertClose(uy.variance / uy.nominal ** 2,
                    ua1.variance / nom1 ** 2 +
                    ua2.variance / nom2 ** 2)

        random.seed(1413)
        for i in range(100):
            z1 = self.cgauss()
            z2 = self.cgauss()

            self.assertDerivative(
                    specimen=(lambda z: numpy.divide(z, z2)), position=z1,
                    prediction=(1 / z2),
                    epsilon=1e-4,
            )
            self.assertDerivative(
                    specimen=(lambda z: numpy.divide(z1, z)), position=z2,
                    prediction=(-z1 / (z2 ** 2)),
                    epsilon=1e-4,
            )

    def test_power(self):
        random.seed( 742)
        for i in range(10):
            nom1, unc1 = self.expo()
            nom2, unc2 = self.gauss()
            ua1 = nom1 +- u(unc1)
            ua2 = nom2 +- u(unc2)

            uy = ua1 ** ua2
            # y = x1 ** x2
            #
            # d_x1 y = x2 * x1 ** (x2 - 1)
            #
            # d_x2 y = d_x2 exp(ln x1) ** x2 = d_x2 exp(ln x1 * x2)
            #   = exp(ln x1 * x2) (ln x1)
            #   = ln x1 * x1 ** x2

            d1 = nom2 * nom1 ** (nom2 - 1)
            d2 = numpy.log(nom1) * nom1 ** nom2

            self.assertClose(uy.variance,
                    ua1.variance * d1 ** 2 + ua2.variance * d2 ** 2)

        random.seed(1416)
        for i in range(100):
            z1 = self.cgauss()
            z2 = self.cgauss()

            self.assertDerivative(
                    specimen=(lambda z: numpy.power(z, z2)), position=z1,
                    prediction=(z2 * (z1 ** (z2 - 1))),
                    epsilon=1e-4,
            )
            self.assertDerivative(
                    specimen=(lambda z: numpy.power(z1, z)), position=z2,
                    prediction=(numpy.log(z1) * (z1 ** z2)),
                    epsilon=1e-4,
            )

    def test_arctan2(self):
        random.seed( 740)
        for i in range(10):
            nomx, uncx = self.gauss()
            nomy, uncy = self.gauss()
            ux = nomx +- u(uncx)
            uy = nomy +- u(uncy)

            ub = upy2.uarctan2(uy, ux)

            dx = -nomy / (nomx ** 2 + nomy ** 2)
            dy = nomx / (nomx ** 2 + nomy ** 2)
            self.assertClose(ub.variance,
                    ux.variance * (dx ** 2) + uy.variance * (dy ** 2))

        random.seed(1419)
        for i in range(100):
            x = random.gauss(0, 1)
            y = random.gauss(0, 1)
            # ``numpy.arctan2`` is supported only for real-valued
            # arguments.

            self.assertDerivative(
                    specimen=(lambda x: numpy.arctan2(y, x)), position=x,
                    prediction=(-y / (x ** 2 + y ** 2)),
                    epsilon=1e-4, epsilonfactor=-1,
            )
            self.assertDerivative(
                    specimen=(lambda y: numpy.arctan2(y, x)), position=y,
                    prediction=(x / (x ** 2 + y ** 2)),
                    epsilon=1e-4, epsilonfactor=-1,
            )

    def test_string_conversion(self):
        unary_operator = UnaryOperator(
                ufunc=numpy.positive,
                uufunc=upy2.upositive,
        )
        self.assertEqual(str(unary_operator),
                "<upy unary operator for numpy <ufunc 'positive'> and upy <<ufunc 'positive'> uufunc>>")
        self.assertEqual(repr(unary_operator),
                "<UnaryOperator(ufunc=<ufunc 'positive'>, uufunc=<<ufunc 'positive'> uufunc>)>")

        binary_operator = BinaryOperator(
                ufunc=numpy.add,
                uufunc=upy2.uadd,
        )
        self.assertEqual(str(binary_operator),
                "<upy binary operator for numpy <ufunc 'add'> and upy <<ufunc 'add'> uufunc>>")
        self.assertEqual(repr(binary_operator),
                "<BinaryOperator(ufunc=<ufunc 'add'>, uufunc=<<ufunc 'add'> uufunc>)>")

max_stars_repo_name: friedrichromstedt/upy

max_issues_repo_name: friedrichromstedt/upy

max_forks_repo_name: friedrichromstedt/upy


text:
import math
import pytest

import numpy as np

from bluesky.callbacks import CallbackCounter
from bluesky.examples import stepscan
from bluesky.tests.utils import DocCollector
from bluesky.callbacks.stream import LiveDispatcher

# Do not run these test if streamz is not installed
try:
    import streamz
    has_streamz = True
except ImportError:
    has_streamz = False

requires_streamz = pytest.mark.skipif(not has_streamz,
                                      reason='Missing streamz library')

class NegativeStream(LiveDispatcher):
    """Stream that only adds metadata to start document"""
    def start(self, doc):
        doc.update({"stream_level": "boring"})
        super().start(doc)

    def event(self, doc):
        modified = dict()
        for key, val in doc['data'].items():
            modified['modified_{}'.format(key)] = -math.fabs(val)
        doc['data'] = modified
        return super().event(doc)

class AverageStream(LiveDispatcher):
    """Stream that averages data points together"""
    def __init__(self, n=None):
        self.n = n
        self.in_node = None
        self.out_node = None
        self.averager = None
        super().__init__()

    def start(self, doc):
        """
        Create the stream after seeing the start document

        The callback looks for the 'average' key in the start document to
        configure itself.
        """
        # Grab the average key
        self.n = doc.get('average', self.n)
        # Define our nodes
        if not self.in_node:
            self.in_node = streamz.Source(stream_name='Input')

        self.averager = self.in_node.partition(self.n)

        def average_events(cache):
            average_evt = dict()
            desc_id = cache[0]['descriptor']
            # Check that all of our events came from the same configuration
            if not all([desc_id == evt['descriptor'] for evt in cache]):
                raise Exception('The events in this bundle are from '
                                'different configurations!')
            # Use the last descriptor to avoid strings and objects
            data_keys = self.raw_descriptors[desc_id]['data_keys']
            for key, info in data_keys.items():
                # Information from non-number fields is dropped
                if info['dtype'] in ('number', 'array', 'integer'):
                    # Average together
                    average_evt[key] = np.mean([evt['data'][key]
                                                for evt in cache], axis=0)
            return {'data': average_evt, 'descriptor': desc_id}

        self.out_node = self.averager.map(average_events)
        self.out_node.sink(super().event)
        super().start(doc)

    def event(self, doc):
        """Send an Event through the stream"""
        self.in_node.emit(doc)

    def stop(self, doc):
        """Delete the stream when run stops"""
        self.in_node = None
        self.out_node = None
        self.averager = None
        super().stop(doc)

def test_straight_through_stream(RE, hw):
    # Just a stream that sinks the events it receives
    ss = NegativeStream()
    # Create callback chain
    c = CallbackCounter()
    d = DocCollector()
    ss.subscribe(c)
    ss.subscribe(d.insert)
    # Run a basic plan
    RE(stepscan(hw.det, hw.motor), {'all': ss})
    # Check that our metadata is there
    assert c.value == 10 + 1 + 2  # events, descriptor, start and stop
    assert d.start[0]['stream_level'] == 'boring'
    desc = d.descriptor[d.start[0]['uid']][0]
    events = d.event[desc['uid']]
    print(desc)
    print([evt['data'] for evt in events])
    tmp_valid = all([evt['data'][key] <= 0
                     for evt in events
                     for key in evt['data'].keys()])
    assert tmp_valid
    tmp_valid = all([key in desc['data_keys']
                     for key in events[0]['data'].keys()])
    assert tmp_valid

@requires_streamz
def test_average_stream(RE, hw):
    # Create callback chain
    avg = AverageStream(10)
    c = CallbackCounter()
    d = DocCollector()
    avg.subscribe(c)
    avg.subscribe(d.insert)
    # Run a basic plan
    RE(stepscan(hw.det, hw.motor), {'all': avg})
    assert c.value == 1 + 1 + 2  # events, descriptor, start and stop
    # See that we made sensible descriptor
    start_uid = d.start[0]['uid']
    assert start_uid in d.descriptor
    desc_uid = d.descriptor[start_uid][0]['uid']
    assert desc_uid in d.event
    evt = d.event[desc_uid][0]
    assert evt['seq_num'] == 1
    assert all([key in d.descriptor[start_uid][0]['data_keys']
                for key in evt['data'].keys()])
    # See that we returned the correct average
    assert evt['data']['motor'] == -0.5  # mean of range(-5, 5)
    assert evt['data']['motor_setpoint'] == -0.5  # mean of range(-5, 5)
    assert start_uid in d.stop
    assert d.stop[start_uid]['num_events'] == {'primary': 1}

max_stars_repo_name: AbbyGi/bluesky

max_issues_repo_name: AbbyGi/bluesky

max_forks_repo_name: AbbyGi/bluesky


text:
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import cv2
import numpy as np
import argparse
import onnxruntime as ort
from pathlib import Path
from tqdm import tqdm

class PicoDet():
    def __init__(self,
                 model_pb_path,
                 label_path,
                 prob_threshold=0.4,
                 iou_threshold=0.3):
        self.classes = list(
            map(lambda x: x.strip(), open(label_path, 'r').readlines()))
        self.num_classes = len(self.classes)
        self.prob_threshold = prob_threshold
        self.iou_threshold = iou_threshold
        self.mean = np.array(
            [103.53, 116.28, 123.675], dtype=np.float32).reshape(1, 1, 3)
        self.std = np.array(
            [57.375, 57.12, 58.395], dtype=np.float32).reshape(1, 1, 3)
        so = ort.SessionOptions()
        so.log_severity_level = 3
        self.net = ort.InferenceSession(model_pb_path, so)
        self.input_shape = (self.net.get_inputs()[0].shape[2],
                            self.net.get_inputs()[0].shape[3])

    def _normalize(self, img):
        img = img.astype(np.float32)
        img = (img / 255.0 - self.mean / 255.0) / (self.std / 255.0)
        return img

    def resize_image(self, srcimg, keep_ratio=False):
        top, left, newh, neww = 0, 0, self.input_shape[0], self.input_shape[1]
        origin_shape = srcimg.shape[:2]
        im_scale_y = newh / float(origin_shape[0])
        im_scale_x = neww / float(origin_shape[1])
        scale_factor = np.array([[im_scale_y, im_scale_x]]).astype('float32')

        if keep_ratio and srcimg.shape[0] != srcimg.shape[1]:
            hw_scale = srcimg.shape[0] / srcimg.shape[1]
            if hw_scale > 1:
                newh, neww = self.input_shape[0], int(self.input_shape[1] /
                                                      hw_scale)
                img = cv2.resize(
                    srcimg, (neww, newh), interpolation=cv2.INTER_AREA)
                left = int((self.input_shape[1] - neww) * 0.5)
                img = cv2.copyMakeBorder(
                    img,
                    0,
                    0,
                    left,
                    self.input_shape[1] - neww - left,
                    cv2.BORDER_CONSTANT,
                    value=0)  # add border
            else:
                newh, neww = int(self.input_shape[0] *
                                 hw_scale), self.input_shape[1]
                img = cv2.resize(
                    srcimg, (neww, newh), interpolation=cv2.INTER_AREA)
                top = int((self.input_shape[0] - newh) * 0.5)
                img = cv2.copyMakeBorder(
                    img,
                    top,
                    self.input_shape[0] - newh - top,
                    0,
                    0,
                    cv2.BORDER_CONSTANT,
                    value=0)
        else:
            img = cv2.resize(
                srcimg, self.input_shape, interpolation=cv2.INTER_AREA)

        return img, scale_factor

    def get_color_map_list(self, num_classes):
        color_map = num_classes * [0, 0, 0]
        for i in range(0, num_classes):
            j = 0
            lab = i
            while lab:
                color_map[i * 3] |= (((lab >> 0) & 1) << (7 - j))
                color_map[i * 3 + 1] |= (((lab >> 1) & 1) << (7 - j))
                color_map[i * 3 + 2] |= (((lab >> 2) & 1) << (7 - j))
                j += 1
                lab >>= 3
        color_map = [color_map[i:i + 3] for i in range(0, len(color_map), 3)]
        return color_map

    def detect(self, srcimg):
        img, scale_factor = self.resize_image(srcimg)
        img = self._normalize(img)

        blob = np.expand_dims(np.transpose(img, (2, 0, 1)), axis=0)

        outs = self.net.run(None, {
            self.net.get_inputs()[0].name: blob,
            self.net.get_inputs()[1].name: scale_factor
        })

        outs = np.array(outs[0])
        expect_boxes = (outs[:, 1] > 0.5) & (outs[:, 0] > -1)
        np_boxes = outs[expect_boxes, :]

        color_list = self.get_color_map_list(self.num_classes)
        clsid2color = {}

        for i in range(np_boxes.shape[0]):
            classid, conf = int(np_boxes[i, 0]), np_boxes[i, 1]
            xmin, ymin, xmax, ymax = int(np_boxes[i, 2]), int(np_boxes[
                i, 3]), int(np_boxes[i, 4]), int(np_boxes[i, 5])

            if classid not in clsid2color:
                clsid2color[classid] = color_list[classid]
            color = tuple(clsid2color[classid])

            cv2.rectangle(
                srcimg, (xmin, ymin), (xmax, ymax), color, thickness=2)
            print(self.classes[classid] + ': ' + str(round(conf, 3)))
            cv2.putText(
                srcimg,
                self.classes[classid] + ':' + str(round(conf, 3)), (xmin,
                                                                    ymin - 10),
                cv2.FONT_HERSHEY_SIMPLEX,
                0.8, (0, 255, 0),
                thickness=2)

        return srcimg

    def detect_folder(self, img_fold, result_path):
        img_fold = Path(img_fold)
        result_path = Path(result_path)
        result_path.mkdir(parents=True, exist_ok=True)

        img_name_list = filter(
            lambda x: str(x).endswith(".png") or str(x).endswith(".jpg"),
            img_fold.iterdir(), )
        img_name_list = list(img_name_list)
        print(f"find {len(img_name_list)} images")

        for img_path in tqdm(img_name_list):
            img = cv2.imread(str(img_path))

            srcimg = net.detect(img)
            save_path = str(result_path / img_path.name.replace(".png", ".jpg"))
            cv2.imwrite(save_path, srcimg)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--modelpath',
        type=str,
        default='onnx_file/picodet_s_320_lcnet_postprocessed.onnx',
        help="onnx filepath")
    parser.add_argument(
        '--classfile',
        type=str,
        default='coco_label.txt',
        help="classname filepath")
    parser.add_argument(
        '--confThreshold', default=0.5, type=float, help='class confidence')
    parser.add_argument(
        '--nmsThreshold', default=0.6, type=float, help='nms iou thresh')
    parser.add_argument(
        "--img_fold", dest="img_fold", type=str, default="./imgs")
    parser.add_argument(
        "--result_fold", dest="result_fold", type=str, default="./results")
    args = parser.parse_args()

    net = PicoDet(
        args.modelpath,
        args.classfile,
        prob_threshold=args.confThreshold,
        iou_threshold=args.nmsThreshold)

    net.detect_folder(args.img_fold, args.result_fold)

max_stars_repo_name: Amanda-Barbara/PaddleDetection

max_issues_repo_name: Amanda-Barbara/PaddleDetection

max_forks_repo_name: Amanda-Barbara/PaddleDetection


text:
import pandas as pd
import numpy as np
from .utils import voc2coco, coco2voc, yolo2coco, coco2yolo, voc2yolo, yolo2voc
from tqdm import tqdm
import json
import argparse
import os
from loguru import logger

__all__ = ['Converter']

class NumpyEncoder(json.JSONEncoder):
    """
    https://stackoverflow.com/questions/26646362/numpy-array-is-not-json-serializable
    Special json encoder for numpy types
    """
    def default(self, obj):
        if isinstance(obj, np.integer):
            return int(obj)
        elif isinstance(obj, np.floating):
            return float(obj)
        elif isinstance(obj, np.ndarray):
            return obj.tolist()
        return json.JSONEncoder.default(self, obj)

class Converter(object):
    def __init__(self,
                 df,
                 categories = [{'id': 0, 'name': 'cots'}],
                 format = 'coco'):
        self.df = df
        self.categories = categories
        self.format = format

    def convert(self, df):
        annotion_id = 0
        images = []
        annotations = []

        for i, row in tqdm(df.iterrows(), total = len(df)):

            images.append({
                "id": i,
                "file_name": f"{row['image_id']}", # image_id must be including extension(.jpg, .png, etc)
                "height": row.height,
                "width": row.width,
            })

            bboxes = row['annotations'].copy()
            bboxes = np.array(bboxes).astype(np.float32)
            if self.format!='voc':
                bboxes = eval(f"{self.format}2voc")(bboxes, row.height, row.width)
            bboxes[..., 0::2] = np.clip(bboxes[..., 0::2], 0, row.width) # x must be in [0, width]
            bboxes[..., 1::2] = np.clip(bboxes[..., 1::2], 0, row.height) # y must be in [0, height]
            bboxes = eval(f"voc2coco")(bboxes, row.height, row.width)
            bboxes = bboxes.astype(int).tolist()

            category_ids = row['category_ids'].copy() # category_id must be for each bbox

            for j, bbox in enumerate(bboxes):
                category_id = int(category_ids[j])
                annotations.append({
                    "id": annotion_id,
                    "image_id": i,
                    "category_id": category_id,
                    "bbox": list(bbox),
                    "area": int(bbox[2]) * int(bbox[3]),
                    "segmentation": [],
                    "iscrowd": 0
                })
                annotion_id += 1

        self.json_file = {'categories':self.categories, 'images':images, 'annotations':annotations}
        return self.json_file

    def save(self, json_file, path):
        with open(path, 'w') as f:
            json.dump(json_file, f, indent=4, cls=NumpyEncoder)

    def convert_save(self, path):
        json_file = self.convert(self.df)
        self.save(json_file, path)

def parse_opt():
    parser = argparse.ArgumentParser()
    parser.add_argument('--train-csv', type=str, default='../data/train.csv')
    parser.add_argument('--valid-csv', type=str, default='../data/valid.csv')
    parser.add_argument('--test-csv', type=str, default=None)
    parser.add_argument('--format', type=str, default='coco')
    parser.add_argument('--output-dir', type=str, default='../output')
    return parser.parse_args()

@logger.catch
def main(opt):
    if not os.path.isdir(opt.output_dir):
        os.makedirs(opt.output_dir)

    train_df = pd.read_csv(opt.train_csv)
    train_converter = Converter(train_df, format=opt.format)
    train_converter.convert_save(f'{opt.output_dir}/annotations_train.json')

    if opt.valid_csv:
        valid_df = pd.read_csv(opt.valid_csv)
        valid_converter = Converter(valid_df, format=opt.format)
        valid_converter.convert_save(f'{opt.output_dir}/annotations_valid.json')

    if opt.test_csv:
        test_df = pd.read_csv(opt.test_csv)
        test_converter = Converter(test_df, format=opt.format)
        test_converter.convert_save(f'{opt.output_dir}/annotations_test.json')

if __name__=='__main__':
    opt = parse_opt()
    main(opt)


text:
import numpy as np
import scipy.linalg as linalg

def center(data):
    """Centers data by subtracting the mean

    Parameters
    ----------
    data : array-like
        data to be centered

    Returns
    -------
    data_centered : array-like
        centered-data

    """
    data_centered = data - np.mean(data)
    return data_centered

def contrast_normalize(data, centered=False):
    """Normalizes image data to have variance of 1

    Parameters
    ----------
    data : array-like
        data to be normalized

    centered : boolean
        When False (the default), centers the data first

    Returns
    -------
    data : array-like
        normalized data

    """
    if not centered:
        data = center(data)
    data = np.divide(data, np.sqrt(np.var(data)))
    return data

def whiten(data, window_size, step_size, centered=False, epsilon=1e-5, type="PCA"):
    """Performs PCA or ZCA whitening on an array. This preprocessing step is described
    in _[1].

    Parameters
    ----------
    img : array-like
        image to be vectorized

    window_size : array-like
        window size dictating the neighborhood to be vectorized, same number of
        dimensions as img, based on the top-left corner

    step_size : array-like
        step size in each of direction of window, same number of
        dimensions as img

    centered : boolean
        When False (the default), centers the data first

    epsilon : epsilon value for whitening

    type : string
        Determines the type of whitening. Can be either 'PCA' (default) or 'ZCA'

    Returns
    -------
    data-whitened : array-like
        whitened data

    S : 2D array
        Singular value array of covariance of vectorized image

    References
    ----------

    .. [1] http://ufldl.stanford.edu/tutorial/unsupervised/PCAWhitening/

    """
    if window_size.ndim > 1 or step_size.ndim > 1:
        raise ValueError("Invalid input")

    if len(window_size) != len(step_size):
        raise ValueError("Dimensions do not match")

    if data.ndim != len(window_size):
        raise ValueError("Dimensions do not match")

    if not centered:
        data = center(data)

    data_padded, pad_size = window_pad(data, window_size, step_size)
    data_vectorized = vectorize_img(data_padded, window_size, step_size)

    c = np.cov(data_vectorized)
    U, S, _ = linalg.svd(c)

    if type == "PCA":
        whiten_matrix = np.dot(np.diag(1.0 / np.sqrt(S + epsilon)), U.T)
    elif type == "ZCA":
        whiten_matrix = np.dot(U, np.dot(np.diag(1.0 / np.sqrt(S + epsilon)), U.T))
    else:
        raise ValueError("Invalid Whitening Type (must be either 'PCA' or 'ZCA'")
    whitened = np.dot(whiten_matrix, data_vectorized)

    data_whitened = imagize_vector(whitened, data_padded.shape, window_size, step_size)
    data_whitened = undo_pad(data_whitened, pad_size)

    return data_whitened, S

def window_pad(img, window_size, step_size):
    """Pad image at edges so the window can convolve evenly.
    Padding will be a copy of the edges.

    Parameters
    ----------
    img : array-like
        image to be padded

    window_size : array-like
        window size that will be convolved, same number of dimensions as img

    step_size : array-like
        step size in each of direction of window convolution, same number of
        dimensions as img

    Returns
    -------
    img_padded : array-like
        padded image

    pad_size : array-like
        amount of padding in every direction of the image

    """
    if window_size.ndim > 1 or step_size.ndim > 1:
        raise ValueError("Invalid input")

    if len(window_size) != len(step_size):
        raise ValueError("Dimensions do not match")

    if img.ndim != len(window_size):
        raise ValueError("Dimensions do not match")

    shp = img.shape
    d = len(shp)

    pad_size = np.zeros([d, 2])
    pad_size[:, 0] = np.floor(np.divide(window_size, 2))

    num_steps = np.floor(np.divide(shp, step_size))
    final_loc = np.multiply(num_steps, step_size) + np.floor(np.divide(window_size, 2))

    pad_size[:, 1] = final_loc - shp
    pad_width = [pad_size[dim, :].astype(int).tolist() for dim in range(d)]

    img_padded = np.pad(img, pad_width, mode="edge")
    # Why does the padding add so much to the edge?
    return img_padded, pad_size

def undo_pad(data, pad_size):
    """Remove padding fromt edges of images

    Parameters
    ----------
    data : array-like
        padded image

    pad_size : array-like
            amount of padding in every direction of the image

    Returns
    -------
    data : array-like
        unpadded image

    """
    if pad_size.ndim == 1 and data.ndim != 1:
        raise ValueError("Dimensions do not match")

    if data.ndim != pad_size.shape[0]:
        raise ValueError("Dimensions do not match")

    start = pad_size[:, 0].astype(int)
    end = (data.shape - pad_size[:, 1]).astype(int)
    coords = list(zip(start, end))
    slices = tuple(slice(coord[0], coord[1]) for coord in coords)
    data = data[slices]

    return data

def vectorize_img(img, window_size, step_size):
    """Reshapes an image by vectorizing different neighborhoods of the image.

    Parameters
    ----------
    img : array-like
        image to be vectorized

    window_size : array-like
        window size dictating the neighborhood to be vectorized, same number of
        dimensions as img, based on the top-left corner

    step_size : array-like
        step size in each of direction of window, same number of
        dimensions as img

    Returns
    -------
    vectorized : array-like
        vectorized image

    """
    if window_size.ndim > 1 or step_size.ndim > 1:
        raise ValueError("Invalid input")

    if len(window_size) != len(step_size):
        raise ValueError("Dimensions do not match")

    if img.ndim != len(window_size):
        raise ValueError("Dimensions do not match")

    shp = img.shape

    num_steps = (np.floor(np.divide(shp - window_size, step_size)) + 1).astype(int)
    vectorized = np.zeros([np.product(window_size), np.product(num_steps)])

    for step_num, step_coord in enumerate(np.ndindex(*num_steps)):
        start = np.multiply(step_coord, step_size)
        end = start + window_size

        coords = list(zip(start, end))
        slices = tuple(slice(coord[0], coord[1]) for coord in coords)
        vectorized[:, step_num] = img[slices].flatten()

    return vectorized

def imagize_vector(data, orig_shape, window_size, step_size):
    """Reshapes a vectorized image back to its original shape.

    Parameters
    ----------
    data : array-like
        vectorized image

    orig_shape : tuple
        dimensions of original image

    window_size : array-like
        window size dictating the neighborhood to be vectorized, same number of
        dimensions as img, based on the top-left corner

    step_size : array-like
        step size in each of direction of window, same number of
        dimensions as img

    Returns
    -------
    imagized : array-like
        original image

    """
    if window_size.ndim > 1 or step_size.ndim > 1:
        raise ValueError("Invalid input")

    if len(window_size) != len(step_size):
        raise ValueError("Dimensions do not match")

    if len(orig_shape) != len(window_size):
        raise ValueError("Dimensions do not match")

    imagized = np.zeros(orig_shape)
    d = len(orig_shape)

    shp = orig_shape

    num_steps = (np.floor(np.divide(shp - window_size, step_size)) + 1).astype(int)

    for step_num, step_coord in enumerate(np.ndindex(*num_steps)):
        start = np.multiply(step_coord, step_size)
        end = start + window_size

        coords = list(zip(start, end))
        slices = tuple(slice(coord[0], coord[1]) for coord in coords)

        imagized_temp = np.zeros(orig_shape)
        imagized_temp = data[:, step_num].reshape(window_size)
        stacked = np.stack((imagized[slices], imagized_temp), axis=-1)
        imagized[slices] = np.true_divide(stacked.sum(d), (stacked != 0).sum(d))
        imagized[slices] = np.nan_to_num(imagized[slices])

    return imagized

max_stars_repo_name: alishakodibagkar/brainlit

max_issues_repo_name: alishakodibagkar/brainlit

max_forks_repo_name: alishakodibagkar/brainlit

reason: import numpy,import scipy


text:
import numpy as np
from pandas import DataFrame

from .common_operations import *

def get_dataset_stats(X, column_types):
    number_of_instances = X.shape[0]
    number_of_features = X.shape[1]
    numeric_features = len(get_numeric_features(X, column_types))
    categorical_features = number_of_features - numeric_features
    ratio_of_numeric_features = numeric_features / number_of_features
    ratio_of_categorical_features = categorical_features / number_of_features
    return (number_of_instances, number_of_features, numeric_features, categorical_features, ratio_of_numeric_features, ratio_of_categorical_features)

def get_dimensionality(number_of_features, number_of_instances):
    dimensionality = number_of_features / number_of_instances
    return (dimensionality,)

def get_missing_values(X):
    missing_values_by_instance = X.shape[1] - X.count(axis=1)
    missing_values_by_feature = X.shape[0] - X.count(axis=0)
    number_missing = int(np.sum(missing_values_by_instance)) # int for json compatibility
    ratio_missing = number_missing / (X.shape[0] * X.shape[1])
    number_instances_with_missing = int(np.sum(missing_values_by_instance != 0)) # int for json compatibility
    ratio_instances_with_missing = number_instances_with_missing / X.shape[0]
    number_features_with_missing = int(np.sum(missing_values_by_feature != 0))
    ratio_features_with_missing = number_features_with_missing / X.shape[1]
    return (
        number_missing, ratio_missing, number_instances_with_missing,
        ratio_instances_with_missing, number_features_with_missing,
        ratio_features_with_missing
    )

def get_class_stats(Y):
    classes = Y.unique()
    number_of_classes = classes.shape[0]
    counts = [sum(Y == label) for label in classes]
    probs = [count/Y.shape[0] for count in counts]
    mean_class_probability, stdev_class_probability, min_class_probability, _, _, _, max_class_probability = profile_distribution(probs)
    majority_class_size = max(counts)
    minority_class_size = min(counts)
    return (number_of_classes, mean_class_probability, stdev_class_probability, min_class_probability, max_class_probability, minority_class_size, majority_class_size)

def get_categorical_cardinalities(X, column_types):
    cardinalities = [X[feature].unique().shape[0] for feature in get_categorical_features(X, column_types)]
    mean_cardinality_of_categorical_features, stdev_cardinality_of_categorical_features, min_cardinality_of_categorical_features, _, _, _, max_cardinality_of_categorical_features = profile_distribution(cardinalities)
    return (mean_cardinality_of_categorical_features, stdev_cardinality_of_categorical_features, min_cardinality_of_categorical_features, max_cardinality_of_categorical_features)

def get_numeric_cardinalities(X, column_types):
    cardinalities = [X[feature].unique().shape[0] for feature in get_numeric_features(X, column_types)]
    mean_cardinality_of_numeric_features, stdev_cardinality_of_numeric_features, min_cardinality_of_numeric_features, _, _, _, max_cardinality_of_numeric_features = profile_distribution(cardinalities)
    return (mean_cardinality_of_numeric_features, stdev_cardinality_of_numeric_features, min_cardinality_of_numeric_features, max_cardinality_of_numeric_features)


text:
import sys
import os
import cv2
import numpy as np
import numpy as np
import asyncio

## Convert COCO keypoints to PoseTrack keypoints
def convert_coco_poseTrack(kps):
    """
        input:  N, 17, 3
            kp_names = [0:'nose', 1:'l_eye', 2:'r_eye', 3:'l_ear', 4:'r_ear', 5:'l_shoulder',
                        6:'r_shoulder', 7:'l_elbow', 8:'r_elbow', 9:'l_wrist', 10:'r_wrist',
                        11:'l_hip', 12:'r_hip', 13:'l_knee', 14:'r_knee', 15:'l_ankle', 16:'r_ankle']
        output: N, 15, 3
            {0: 'right_ankle', 1: 'right_knee', 2: 'right_hip', 3: 'left_hip', 4:'left_knee', 5:'left_ankle',  6:'right_wrist', 7:'right_elbow', 8:'right_shoulder',9:'left_shoulder', 10:'left_elbow', 11:'left_wrist',12:'head_bottom', 13:'nose', 14'head_top']

    """
    MAP ={16:0, 14:1, 12:2, 11:3, 13:4, 15:5, 10:6, 8:7, 6:8, 5:9, 7:10, 9:11, 0:13}

    pose_track_kps = np.zeros((kps.shape[0], 15, 3))

    for k in range(kps.shape[0]):
        pivel = (kps[k][5] + kps[k][6])/2
        head_bottom = (pivel+kps[k][0])/2 # 12
        head_top = 2*kps[k][0] - head_bottom # 14
        head_top[0] = (head_bottom[0] + kps[k][0][0])/2 #改变x

        for coco_i, pt_j in MAP.items():
            pose_track_kps[k][pt_j] = kps[k][coco_i]
        pose_track_kps[k][12] = head_bottom
        pose_track_kps[k][14] = head_top

    return pose_track_kps

async def async_get_keypoints_from_api(img, bbox, fast=None):
    if fast:
        #  url_pose='http://127.0.0.1:9094/predictions/simple_baseline'
        url_pose='http://127.0.0.1:9294/predictions/simple_baseline'
    else:
        url_pose='http://127.0.0.1:9092/predictions/hrnet'
    _, jepg = cv2.imencode('.jpg', img)
    file_pose = {'data': jepg.tobytes(), 'boxes': str([bbox])}
    re = await requests.post(url_pose, files=file_pose)
    kpts = eval(re.text)['result']
    return np.array(kpts)

async def async_inference_keypoints_api(test_data, raw_img, fast=0):
    image_id = test_data['img_id']
    category_id = 1
    bbox = test_data['bbox']
    coco_kps = await async_get_keypoints_from_api(raw_img, bbox, fast)
    keypoints = convert_coco_poseTrack(coco_kps)[0]
    keypoints[..., 2] = keypoints[..., 2] + 0.5
    score = np.mean(coco_kps[0][...,2], 0) + 0.5
    keypoints = keypoints.reshape(-1).tolist()
    keypoints = [round(i, 3) for i in keypoints]
    return keypoints, coco_kps[0].tolist()

def inference_keypoints_api(test_data, raw_img, fast):
    image_id = test_data['img_id']
    category_id = 1
    bbox = test_data['bbox']
    coco_kps = get_keypoints_from_api(raw_img, bbox, fast=1)
    keypoints = convert_coco_poseTrack(coco_kps)[0]
    keypoints[..., 2] = keypoints[..., 2] + 0.5
    score = np.mean(coco_kps[0][...,2], 0) + 0.5
    keypoints = keypoints.reshape(-1).tolist()
    keypoints = [round(i, 3) for i in keypoints]

    return keypoints, coco_kps[0].tolist()

def get_keypoints_from_api(img, bbox, fast=None):
    import requests
    if fast:
        url_pose='http://127.0.0.1:9294/predictions/simple_baseline'
    else:
        url_pose='http://127.0.0.1:9092/predictions/hrnet'
    _, jepg = cv2.imencode('.jpg', img)
    file_pose = {'data': jepg.tobytes(), 'boxes': str([bbox])}
    re = requests.post(url_pose, files=file_pose)
    kpts = eval(re.text)['result']
    return np.array(kpts)

def inference_keypoints_api_coco(test_data, raw_img):
    # 动作识别使用的,多返回coco本身关节点
    image_id = test_data['img_id']
    category_id = 1
    bbox = test_data['bbox']
    coco_kps = get_keypoints_from_api(raw_img, bbox, fast=1)
    keypoints = convert_coco_poseTrack(coco_kps)[0]
    keypoints[..., 2] = keypoints[..., 2] + 0.5
    score = np.mean(coco_kps[0][...,2], 0) + 0.5
    keypoints = keypoints.reshape(-1).tolist()
    keypoints = [round(i, 3) for i in keypoints]
    return keypoints, coco_kps[0].tolist()

# convert coco(17) to openpose_coco(18)
def convert17_18(coco_kps):
    # map:0, neck, 5, 7, 9, 6, 8, 10, 11, 13, 15, 12, 14, 16, 1, 2, 3, 4
    kpts = np.zeros(shape=(len(coco_kps), 18, 3))
    for i in range(len(coco_kps)):
        # neck point
        neck =  (coco_kps[i][5] + coco_kps[i][6])/2
        mapping = (coco_kps[i][0], neck, coco_kps[i][5], coco_kps[i][7], coco_kps[i][9], coco_kps[i][6], coco_kps[i][8], coco_kps[i][10], coco_kps[i][11], coco_kps[i][13], coco_kps[i][15], coco_kps[i][12], coco_kps[i][14], coco_kps[i][16], coco_kps[i][1], coco_kps[i][2], coco_kps[i][3], coco_kps[i][4])
        kpts[i] = np.concatenate(mapping, 0).reshape(-1, 3) #(18, 3)
    return kpts

max_stars_repo_name: abrichr/cvToolkit

max_issues_repo_name: abrichr/cvToolkit

max_forks_repo_name: abrichr/cvToolkit