text:
#!/usr/bin/env python3
"""
Understanding SGD vs GD.

"""
from autograd import numpy as np, grad
from matplotlib import pyplot

np.set_printoptions(suppress=True)  # never show scientific-notation
pyplot.rcParams["axes.grid"] = True  # grid on plots by default
pyplot.rcParams["font.size"] = 16  # default fontsize for plot labels

##################################################

# Noisy scalar-polynomial data
X = np.arange(-3, 3, 0.1).reshape(-1, 1)  # n-by-1 matrix (n points, 1 feature)
Y = 1*X**3 + 2*X**2 + 3*X + 4 + np.random.normal(0.0, 2.0, X.shape)  # n-by-1 matrix
n = len(X)

# Scalar-polynomial model with parameters c
def f(c, X):
    return c[0]*X**3 + c[1]*X**2 + c[2]*X + c[3]

##################################################

# L2 loss
def l(c, X, Y):
    return np.mean(np.sum((Y - f(c, X))**2, axis=1))
dldc = grad(l)  # this gradient would not be hard to write by hand, but autodiff is even easier!

# Closed-form optimum for this linear regression (assuming L2 loss)
print("(solving pseudoinverse)")
A = np.concatenate((X**3, X**2, X, np.ones_like(X)), axis=1)  # A(X)*c = Y
c_opt = np.linalg.pinv(A).dot(Y)  # c = (A(X)^~1)*Y  where pseudoinverse projection minimizes L2 error

##################################################

# Gradient descent assuming model f and loss l
def gd(X, Y, c0, step, iters):
    print("(running gd)")
    c = np.array(c0, float)
    L = []
    for i in range(iters):
        c -= step*dldc(c, X, Y)
        L.append(l(c, X, Y))  # record loss at each iter for plotting
    return c, L

# Stochastic gradient descent assuming model f and loss l
def sgd(X, Y, c0, step, iters, batch_size, replace=False):
    print("(running sgd)")
    c = np.array(c0, float)
    L = []
    for i in range(iters * n//batch_size):
        # Randomly select batch_size integers from 0 to n-1 with or without replacement
        # If you want to guarantee that every data-point gets used the same amount,
        # then do what is done in practice: iterate over batches of a shuffle.
        samples = np.random.choice(n, batch_size, replace=replace)
        c -= step*dldc(c, X[samples], Y[samples])
        L.append(l(c, X, Y))  # record TOTAL loss at each epoch for plotting
    return c, L

# On convex problems (like linear regression), SGD is a
# BAD idea because there are no local optima to "shake" out of.
# SGD is useful for nonconvex optimizations where the effective
# added noise might get us out of local optima, and for huge
# datasets where batch-processing (processing in small chunks)
# may be the only feasible option (you can't just load an entire 5TB
# SQL database into Python's working memory, so query batches).

##################################################

# Select hyperparameters
c0 = [0, 0, 0, 0]  # selecting a good random initial condition is not as important for convex problems
step = 0.008
iters = 200
batch_size = 15

# Run iterative optimizers
c_gd, L_gd = gd(X, Y, c0, step, iters)  # same as SGD with batch_size=n (and replace=False)
c_sgd, L_sgd = sgd(X, Y, c0, step, iters, batch_size)

# Compare results
print("c_opt:", c_opt.flatten())
print(" c_gd:", c_gd)
print("c_sgd:", c_sgd)
fig, axes = pyplot.subplots(1, 2)
axes[0].set_title("Fits")
axes[0].set_ylabel("Y")
axes[0].set_xlabel("X")
axes[0].scatter(X, Y, c='k', alpha=0.4, s=16, label="Data")
axes[0].plot(X, f(c_opt, X), c='g', ls='--', lw=4, label="Opt")
axes[0].plot(X, f(c_gd, X), c='b', label="GD")
axes[0].plot(X, f(c_sgd, X), c='r', label="SGD")
axes[0].legend()
axes[1].set_title("Convergence")
axes[1].set_ylabel("Loss")
axes[1].set_xlabel("Iteration")
axes[1].plot(L_gd, c='b', alpha=0.6, lw=3, label="GD")
axes[1].plot(L_sgd, c='r', label="SGD")
axes[1].set_ylim([0, l(c0, X, Y)])
axes[1].set_xlim([0, iters])
axes[1].legend()
pyplot.show()


text:
from __future__ import print_function
from sage.sets.family import Family
from sage.combinat.integer_lists import IntegerListsLex
from sage.rings.semirings.non_negative_integer_semiring import NN
from sage.sets.disjoint_union_enumerated_sets import DisjointUnionEnumeratedSets

def IntegerListsNN(**kwds):
    """
    Lists of nonnegative integers with constraints.

    This function returns the union of ``IntegerListsLex(n, **kwds)``
    where `n` ranges over all nonnegative integers.

    EXAMPLES::

        sage: from sage.combinat.integer_lists.nn import IntegerListsNN
        sage: L = IntegerListsNN(max_length=3, max_slope=-1)
        sage: L
        Disjoint union of Lazy family (<lambda>(i))_{i in Non negative integer semiring}
        sage: it = iter(L)
        sage: for _ in range(20):
        ....:     print(next(it))
        []
        [1]
        [2]
        [3]
        [2, 1]
        [4]
        [3, 1]
        [5]
        [4, 1]
        [3, 2]
        [6]
        [5, 1]
        [4, 2]
        [3, 2, 1]
        [7]
        [6, 1]
        [5, 2]
        [4, 3]
        [4, 2, 1]
        [8]
    """
    return DisjointUnionEnumeratedSets(Family(NN, lambda i: IntegerListsLex(i, **kwds)))

ext: py

max_stars_repo_name: bopopescu/sage

max_issues_repo_name: Ivo-Maffei/sage

max_forks_repo_name: dimpase/sage

reason: from sage


text:
from builtins import print
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import accuracy_score
from sklearn.metrics import precision_score
from sklearn.metrics import recall_score
from sklearn.metrics import confusion_matrix
from sklearn.metrics import f1_score
from scipy.io import arff
import scipy.io as sio
import numpy as np
import pandas as pd
import matplotlib

matplotlib.use('agg')
import matplotlib.pyplot as plt
import seaborn as sns
import os
import operator
import h5py
import glob
import scipy
from collections import Counter

def readucr(filename):
    data = np.loadtxt(filename)
    # 混洗 shuffle 默认打乱第一维度
    np.random.shuffle(data)
    Y = data[:, 0]
    X = data[:, 1:]
    return X, Y

def readmts(filename):
    data = arff.loadarff(filename + '_TRAIN.arff')
    df = pd.DataFrame(data[0])
    values = df.values

    TrainSize = values.shape[0]
    SeriesLength = len(values[0][0][0])
    NumDimensions = values[0][0].size

    train_data = np.zeros((TrainSize, SeriesLength, NumDimensions))
    train_label = []
    for i in range(TrainSize):
        train_label.append(values[i][1].decode())
        for j in range(NumDimensions):
            a = list(values[i][0][j])
            train_data[i, :, j] = np.array(a, dtype=float)

    data = arff.loadarff(filename + '_TEST.arff')
    df = pd.DataFrame(data[0])
    values = df.values

    TestSize = values.shape[0]

    test_data = np.zeros((TestSize, SeriesLength, NumDimensions))
    test_label = []
    for i in range(TestSize):
        test_label.append(values[i][1].decode())
        for j in range(NumDimensions):
            a = list(values[i][0][j])
            test_data[i, :, j] = np.array(a, dtype=float)

    nb_classes = len(np.unique(np.concatenate((train_label, test_label), axis=0)))
    labels = train_label + test_label
    labels = list(set(labels))
    le = LabelEncoder().fit(labels)
    new_train_label = le.transform(train_label)
    new_test_label = le.transform(test_label)

    permutation = np.random.permutation(TrainSize)
    X_train = train_data[permutation, :, :]
    y_train = new_train_label[permutation]

    permutation = np.random.permutation(TestSize)
    X_test = test_data[permutation, :, :]
    y_test = new_test_label[permutation]

    return X_train, y_train, X_test, y_test, nb_classes

def readmts_uci_har(filename):  # file_name = 'datasets/uts_data/' + config.dataset.name

    path_train = filename + '/train/Inertial Signals/'
    file_train = ['body_acc_x_train.txt', 'body_acc_y_train.txt', 'body_acc_z_train.txt', 'body_gyro_x_train.txt',
                  'body_gyro_y_train.txt', 'body_gyro_z_train.txt', 'total_acc_x_train.txt', 'total_acc_y_train.txt',
                  'total_acc_z_train.txt']
    list_train = []
    for i in range(9):
        data = np.loadtxt(path_train + file_train[i])
        list_train.append(data)

    length = list_train[0].shape[1]
    train_size = list_train[0].shape[0]

    train_data = np.zeros((train_size, length, 9))
    for i in range(train_size):
        for j in range(9):
            train_data[i, :, j] = list_train[j][i]

    train_label = np.loadtxt(filename + '/train/y_train.txt')

    path_test = filename + '/test/Inertial Signals/'
    file_test = ['body_acc_x_test.txt', 'body_acc_y_test.txt', 'body_acc_z_test.txt', 'body_gyro_x_test.txt',
                 'body_gyro_y_test.txt', 'body_gyro_z_test.txt', 'total_acc_x_test.txt', 'total_acc_y_test.txt',
                 'total_acc_z_test.txt']
    list_test = []
    for i in range(9):
        data = np.loadtxt(path_test + file_test[i])
        list_test.append(data)

    length = list_test[0].shape[1]
    test_size = list_test[0].shape[0]

    test_data = np.zeros((test_size, length, 9))
    for i in range(test_size):
        for j in range(9):
            test_data[i, :, j] = list_test[j][i]

    test_label = np.loadtxt(filename + '/test/y_test.txt')

    permutation = np.random.permutation(train_size)
    X_train = train_data[permutation, :, :]
    y_train = train_label[permutation]

    permutation = np.random.permutation(test_size)
    X_test = test_data[permutation, :, :]
    y_test = test_label[permutation]

    # X_train = train_data
    # y_train = train_label
    # X_test = test_data
    # y_test = test_label

    return X_train, y_train, X_test, y_test

def readmts_ptb(filename):
    ## Loading time serie signals
    files = sorted(glob.glob(filename + "*.mat"))
    count = 0
    maxlen = -1
    minlen = 999

    # all_data = np.zeros((len(files), 200000))
    all_data = []
    for i in range(len(files)):
        # if i in index2del:
        #     continue
        f = files[i]
        record = f[:-4]
        record = record[-6:]
        # Loading
        mat_data = scipy.io.loadmat(f[:-4] + ".mat")
        print('Loading record {}'.format(record))
        data = mat_data['val'].squeeze()
        all_data.append(data)
        data = np.array(data)
        if len(data) > maxlen:
            maxlen = len(data)
        if len(data) < minlen:
            minlen = len(data)
    all_data = np.array(all_data)
    # normalize
    for i in range(len(all_data)):
        tmp_data = all_data[i]
        tmp_std = np.std(tmp_data)
        tmp_mean = np.mean(tmp_data)
        data_tmp = (tmp_data - tmp_mean) / tmp_std
        data_tmp = data_tmp[:minlen]
        all_data[i] = data_tmp
    all_data = np.array(all_data)

    # WINDOW_SIZE = maxlen
    # ## 短数据处理
    #
    # for i in range(len(files)):
    #     offset = 0
    #     while offset == 0 or offset + WINDOW_SIZE < len(all_data[i]):
    #         slice = np.zeros(WINDOW_SIZE)
    #         if offset + WINDOW_SIZE < len(all_data[i]):
    #             slice[offset: offset + WINDOW_SIZE] = x_train[i][offset: offset + WINDOW_SIZE]
    #         else:
    #             slice[offset: offset + len(x_train[i])] = x_train[i][offset: len(x_train[i])]

    ## read label
    label_df = pd.read_csv(os.path.join(filename, 'REFERENCE-v3.csv'), header=None)
    label = label_df.iloc[:, 1].values
    all_label = []
    for i in label:

        # Myocardial infarction (368)
        if i == 'Myocardial infarction':
            all_label.append(0)

        # 'Healthy control': 80
        elif i == 'Healthy control':
            all_label.append(1)

        # 'Valvular heart disease': 6
        # elif i == 'Valvular heart disease':
        #     all_label.append(2)

        # 'Dysrhythmia': 16
        elif i == 'Dysrhythmia':
            all_label.append(2)

        # 'Heart failure (NYHA 2)': 1
        # elif i == 'Heart failure (NYHA 2)':
        #     all_label.append(4)

        # 'Heart failure (NYHA 3)': 1
        # elif i == 'Heart failure (NYHA 3)':
        #     all_label.append(5)

        # 'Heart failure (NYHA 4)': 1
        # elif i == 'Heart failure (NYHA 4)':
        #     all_label.append(6)

        # 'Palpitation': 1
        # elif i == 'Palpitation':
        #     all_label.append(7)

        # 'Cardiomyopathy': 17
        elif i == 'Cardiomyopathy':
            all_label.append(3)

        # 'Stable angina': 2
        # elif i == 'Stable angina':
        #     all_label.append(9)

        # 'Hypertrophy': 7
        # elif i == 'Hypertrophy':
        #     all_label.append(10)

        # 'Bundle branch block': 17
        elif i == 'Bundle branch block':
            all_label.append(4)

        # 'Unstable angina': 1
        # elif i == 'Unstable angina':
        #     all_label.append(12)

        # 'Myocarditis': 4
        # elif i == 'Myocarditis':
        #     all_label.append(13)

        # 'n/a': 27'
        # elif i == 'n/a':
        #     all_label.append(5)

        else:
            all_label.append(-1)

    all_label = np.array(all_label)

    data = []
    label = []

    for i in range(all_data.shape[0]):
        if all_label[i] != -1:
            data.append(all_data[i])
            label.append(all_label[i])

    data = np.array(data)
    label = np.array(label)

    n_sample = data.shape[0]

    split_idx_1 = int(0.75 * n_sample)
    split_idx_2 = int(0.85 * n_sample)

    shuffle_idx = np.random.permutation(n_sample)
    data = data[shuffle_idx]
    label = label[shuffle_idx]

    X_train = data[:split_idx_1]
    X_val = data[split_idx_1:split_idx_2]
    X_test = data[split_idx_2:]
    Y_train = label[:split_idx_1]
    Y_val = label[split_idx_1:split_idx_2]
    Y_test = label[split_idx_2:]

    train_data = []
    train_label = []
    val_data = []
    val_label = []
    test_data = []
    test_label = []

    for i in range(X_train.shape[0]):
        tmp_ts = X_train[i]
        tmp_Y = Y_train[i]
        ###############################################
        # 0: Myocardial infarction (368)

        # 1: 'Healthy control': 80

        # 2: 'Dysrhythmia': 16

        # 3: 'Cardiomyopathy': 17

        # 4: 'Bundle branch block': 17

        # 5: 'n/a': 27'
        ####################################
        if tmp_Y == 0:
            i_stride = 1
        elif tmp_Y == 1:
            i_stride = 4
        elif tmp_Y == 2:
            i_stride = 30
        elif tmp_Y == 3:
            i_stride = 18
        elif tmp_Y == 4:
            i_stride = 17
        else:
            i_stride = 21
        # if tmp_Y == 0:
        #     i_stride = 1
        # else:
        #     i_stride = 4

        for i in range(i_stride):
            train_data.append(tmp_ts)
            train_label.append(tmp_Y)

    for i in range(X_val.shape[0]):
        tmp_ts = X_val[i]
        tmp_Y = Y_val[i]
        # ###############################################
        # # 0: Myocardial infarction (368)
        #
        # # 1: 'Healthy control': 80
        #
        # # 2: 'Dysrhythmia': 16
        #
        # # 3: 'Cardiomyopathy': 17
        #
        # # 4: 'Bundle branch block': 17
        #
        # # 5: 'n/a': 27'
        # ####################################
        # if tmp_Y == 0:
        #     i_stride = 1
        # elif tmp_Y == 1:
        #     i_stride = 4
        # elif tmp_Y == 2:
        #     i_stride = 30
        # elif tmp_Y == 3:
        #     i_stride = 18
        # elif tmp_Y == 4:
        #     i_stride = 17
        # else:
        #     i_stride = 21
        # # if tmp_Y == 0:
        # #     i_stride = 1
        # # else:
        # #     i_stride = 4
        i_stride = 1
        for i in range(i_stride):
            val_data.append(tmp_ts)
            val_label.append(tmp_Y)

    for i in range(X_test.shape[0]):
        tmp_ts = X_test[i]
        tmp_Y = Y_test[i]
        # ###############################################
        # # 0: Myocardial infarction (368)
        #
        # # 1: 'Healthy control': 80
        #
        # # 2: 'Dysrhythmia': 16
        #
        # # 3: 'Cardiomyopathy': 17
        #
        # # 4: 'Bundle branch block': 17
        #
        # # 5: 'n/a': 27'
        # ####################################
        # if tmp_Y == 0:
        #     i_stride = 1
        # elif tmp_Y == 1:
        #     i_stride = 4
        # elif tmp_Y == 2:
        #     i_stride = 30
        # elif tmp_Y == 3:
        #     i_stride = 18
        # elif tmp_Y == 4:
        #     i_stride = 17
        # else:
        #     i_stride = 21
        # # if tmp_Y == 0:
        # #     i_stride = 1
        # # else:
        # #     i_stride = 4
        i_stride = 1
        for i in range(i_stride):
            test_data.append(tmp_ts)
            test_label.append(tmp_Y)

    X_train = np.array(train_data)
    Y_train = np.array(train_label)
    X_val = np.array(val_data)
    Y_val = np.array(val_label)
    X_test = np.array(test_data)
    Y_test = np.array(test_label)

    # shuffle train
    shuffle_pid = np.random.permutation(X_train.shape[0])
    X_train = X_train[shuffle_pid]
    Y_train = Y_train[shuffle_pid]

    print(Counter(Y_train), Counter(Y_val), Counter(Y_test))

    return X_train, Y_train, X_val, Y_val, X_test, Y_test

def readmts_ptb_aug(filename):
    ## Loading time serie signals
    files = sorted(glob.glob(filename + "*.mat"))
    count = 0
    maxlen = 0

    all_data = []
    for i in range(len(files)):
        # if i in index2del:
        #     continue
        f = files[i]
        record = f[:-4]
        record = record[-6:]
        # Loading
        mat_data = scipy.io.loadmat(f[:-4] + ".mat")
        print('Loading record {}'.format(record))
        data = mat_data['val'].squeeze()
        all_data.append(data)
        data = np.array(data)
        if len(data) > maxlen:
            maxlen = len(data)
    all_data = np.array(all_data)

    # # normalize
    # for i in range(len(all_data)):
    #     tmp_data = all_data[i]
    #     tmp_std = np.std(tmp_data)
    #     tmp_mean = np.mean(tmp_data)
    #     all_data[i] = (tmp_data - tmp_mean) / tmp_std

    ## read label
    label_df = pd.read_csv(os.path.join(filename, 'REFERENCE-v3.csv'), header=None)
    label = label_df.iloc[:, 1].values
    all_label = []
    for i in label:

        # Myocardial infarction (368)
        if i == 'Myocardial infarction':
            all_label.append(0)

        # 'Healthy control': 80
        elif i == 'Healthy control':
            all_label.append(1)

        # 'Valvular heart disease': 6
        # elif i == 'Valvular heart disease':
        #     all_label.append(2)

        # 'Dysrhythmia': 16
        elif i == 'Dysrhythmia':
            all_label.append(2)

        # 'Heart failure (NYHA 2)': 1
        # elif i == 'Heart failure (NYHA 2)':
        #     all_label.append(4)

        # 'Heart failure (NYHA 3)': 1
        # elif i == 'Heart failure (NYHA 3)':
        #     all_label.append(5)

        # 'Heart failure (NYHA 4)': 1
        # elif i == 'Heart failure (NYHA 4)':
        #     all_label.append(6)

        # 'Palpitation': 1
        # elif i == 'Palpitation':
        #     all_label.append(7)

        # 'Cardiomyopathy': 17
        elif i == 'Cardiomyopathy':
            all_label.append(3)

        # 'Stable angina': 2
        # elif i == 'Stable angina':
        #     all_label.append(9)

        # 'Hypertrophy': 7
        # elif i == 'Hypertrophy':
        #     all_label.append(10)

        # 'Bundle branch block': 17
        elif i == 'Bundle branch block':
            all_label.append(4)

        # 'Unstable angina': 1
        # elif i == 'Unstable angina':
        #     all_label.append(12)

        # 'Myocarditis': 4
        # elif i == 'Myocarditis':
        #     all_label.append(13)

        # 'n/a': 27'
        # elif i == 'n/a':
        #     all_label.append(5)
        else:
            all_label.append(-1)

    all_label = np.array(all_label)

    data = []
    label = []

    for i in range(len(all_label)):
        if all_label[i] != -1:
            data.append(all_data[i])
            label.append(all_label[i])

    data = np.array(data)
    label = np.array(label)

    n_sample = len(label)

    split_idx_1 = int(0.75 * n_sample)
    split_idx_2 = int(0.85 * n_sample)

    flag = 1
    while flag == 1:
        shuffle_idx = np.random.permutation(n_sample)
        data = data[shuffle_idx]
        label = label[shuffle_idx]

        X_train = data[:split_idx_1]
        X_val = data[split_idx_1:split_idx_2]
        X_test = data[split_idx_2:]
        Y_train = label[:split_idx_1]
        Y_val = label[split_idx_1:split_idx_2]
        Y_test = label[split_idx_2:]

        train_labels = len(np.unique(Y_train, axis=0))
        val_labels = len(np.unique(Y_val, axis=0))
        test_labels = len(np.unique(Y_test, axis=0))
        if train_labels == val_labels and train_labels == test_labels:
            flag = 0

    # slide and cut
    print(Counter(Y_train), Counter(Y_val), Counter(Y_test))

    s1 = Counter(Y_train)[0] / Counter(Y_train)[1]
    s2 = Counter(Y_train)[0] / Counter(Y_train)[2] + 10
    s3 = Counter(Y_train)[0] / Counter(Y_train)[3]
    s4 = Counter(Y_train)[0] / Counter(Y_train)[4]
    X_train, Y_train = slide_and_cut(X_train, Y_train, window_size=5000, stride=500, s1=s1, s2=s2, s3=s3, s4=s4,
                                     output_pid=False)

    s1 = Counter(Y_val)[0] / Counter(Y_val)[1]
    s2 = Counter(Y_val)[0] / Counter(Y_val)[2] + 5
    s3 = Counter(Y_val)[0] / Counter(Y_val)[3]
    s4 = Counter(Y_val)[0] / Counter(Y_val)[4]
    X_val, Y_val = slide_and_cut(X_val, Y_val, window_size=5000, stride=500, s1=s1, s2=s2, s3=s3, s4=s4,
                                 output_pid=False)

    s1 = Counter(Y_test)[0] / Counter(Y_test)[1]
    s2 = Counter(Y_test)[0] / Counter(Y_test)[2] + 10
    s3 = Counter(Y_test)[0] / Counter(Y_test)[3]
    s4 = Counter(Y_test)[0] / Counter(Y_test)[4]
    X_test, Y_test = slide_and_cut(X_test, Y_test, window_size=5000, stride=500, s1=s1, s2=s2, s3=s3, s4=s4,
                                   output_pid=False)

    print(Counter(Y_train), Counter(Y_val), Counter(Y_test))
    print(X_train.shape, X_val.shape, X_test.shape)

    # shuffle train
    shuffle_pid = np.random.permutation(X_train.shape[0])
    X_train = X_train[shuffle_pid]
    Y_train = Y_train[shuffle_pid]

    print(Counter(Y_train), Counter(Y_val), Counter(Y_test))

    return X_train, Y_train, X_val, Y_val, X_test, Y_test

def slide_and_cut(X, Y, window_size, stride, s1, s2, s3, s4, output_pid=False):
    out_X = []
    out_Y = []
    out_pid = []
    n_sample = X.shape[0]
    mode = 0
    for i in range(n_sample):
        tmp_ts = X[i]
        tmp_Y = Y[i]

        if tmp_Y == 0:
            i_stride = stride
        elif tmp_Y == 1:
            i_stride = int(stride // s1)  # 4.6
        elif tmp_Y == 2:
            i_stride = int(stride // s2)  # 23
        elif tmp_Y == 3:
            i_stride = int(stride // s3)  # 21
        elif tmp_Y == 4:
            i_stride = int(stride // s4)  # 21

        # for j in range(0, len(tmp_ts)-window_size, i_stride):
        #     out_X.append(tmp_ts[j:j+window_size])
        #     out_Y.append(tmp_Y)
        #     out_pid.append(i)
        for j in range(0, len(tmp_ts) - window_size, i_stride):
            X_tmp = tmp_ts[j:j + window_size]
            out_X.append(X_tmp)
            out_Y.append(tmp_Y)
            out_pid.append(i)
    if output_pid:
        return np.array(out_X), np.array(out_Y), np.array(out_pid)
    else:
        return np.array(out_X), np.array(out_Y)

def transform_labels(y_train, y_test, y_val=None):
    """
    Transform label to min equal zero and continuous
    For example if we have [1,3,4] --->  [0,1,2]
    """
    if not y_val is None:
        # index for when resplitting the concatenation
        idx_y_val = len(y_train)
        idx_y_test = idx_y_val + len(y_val)
        # init the encoder
        encoder = LabelEncoder()
        # concat train and test to fit
        y_train_val_test = np.concatenate((y_train, y_val, y_test), axis=0)
        # fit the encoder
        encoder.fit(y_train_val_test)
        # transform to min zero and continuous labels
        new_y_train_val_test = encoder.transform(y_train_val_test)
        # resplit the train and test
        new_y_train = new_y_train_val_test[0:idx_y_val]
        new_y_val = new_y_train_val_test[idx_y_val:idx_y_test]
        new_y_test = new_y_train_val_test[idx_y_test:]
        return new_y_train, new_y_val, new_y_test
    else:
        # no validation split
        # init the encoder
        encoder = LabelEncoder()
        # concat train and test to fit
        y_train_test = np.concatenate((y_train, y_test), axis=0)
        # fit the encoder
        encoder.fit(y_train_test)
        # transform to min zero and continuous labels
        new_y_train_test = encoder.transform(y_train_test)
        # resplit the train and test
        new_y_train = new_y_train_test[0:len(y_train)]
        new_y_test = new_y_train_test[len(y_train):]
        return new_y_train, new_y_test

def calculate_metrics(y_true, y_pred):
    res = pd.DataFrame(data=np.zeros((1, 4), dtype=np.float), index=[0],
                       columns=['Accuracy', 'Precision(macro)', 'Recall(macro)', 'Duration'])
    res['Accuracy'] = accuracy_score(y_true, y_pred)
    res['Precision(macro)'] = precision_score(y_true, y_pred, average='macro')  # sklearn.metrics 中直接计算precision
    res['Recall(macro)'] = recall_score(y_true, y_pred, average='macro')  # sklearn.metrics 中直接计算recall
    res['F1(macro)'] = f1_score(y_true, y_pred, average='macro')
    res['Precision(micro)'] = precision_score(y_true, y_pred, average='micro')
    res['Recall(micro)'] = recall_score(y_true, y_pred, average='micro')
    res['F1(micro)'] = f1_score(y_true, y_pred, average='micro')
    return res

def plot_epochs_metric(hist, file_name, metric='loss'):
    plt.figure()
    plt.plot(hist.history[metric])
    plt.plot(hist.history['val_' + metric])
    plt.title('model ' + metric)
    plt.ylabel(metric, fontsize='large')
    plt.xlabel('epoch', fontsize='large')
    plt.legend(['train', 'val'], loc='upper left')
    plt.savefig(file_name, bbox_inches='tight')
    plt.close('all')

def plot_trainingsize_metric(data, file_name):
    plt.figure()
    plt.plot(data["training_size"], data["accuracy"])
    plt.plot(data["training_size"], data["f1"])
    plt.title('model ')
    plt.ylabel('metric', fontsize='large')
    plt.xlabel('training_size', fontsize='large')
    plt.legend(['accuracy', 'f1'], loc='upper left')
    plt.savefig(file_name, bbox_inches='tight')
    plt.close('all')

def plot_confusion_matrix(cm, nb_classes, output_directory, title='Confusion Matrix', cmap=plt.cm.binary):
    tick_marks = np.array(range(nb_classes))

    labels = []
    for i in range(nb_classes):
        labels.append("Cls." + str(i))

    np.set_printoptions(precision=2)
    cm_normalized = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]

    plt.figure(figsize=(12, 12), dpi=120)

    ind_array = np.arange(nb_classes)
    x, y = np.meshgrid(ind_array, ind_array)

    for x_val, y_val in zip(x.flatten(), y.flatten()):
        c = cm_normalized[y_val][x_val]
        plt.text(x_val, y_val, "%0.3f" % (c,), color='red', fontsize=100 / nb_classes, va='center', ha='center')
    # offset the tick
    plt.xticks(tick_marks, labels)
    plt.yticks(tick_marks, labels)
    plt.gca().xaxis.set_ticks_position('top')
    plt.gca().yaxis.set_ticks_position('left')
    plt.grid(True, which='minor', linestyle='-')
    plt.gcf().subplots_adjust(bottom=0.15)

    # show confusion matrix

    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    # xlocations = np.array(range(nb_class))
    # plt.xticks(xlocations, xlocations, rotation=90)
    # plt.yticks(xlocations, xlocations)

    plt.ylabel('True label')
    plt.xlabel('Predicted label')
    plt.savefig(output_directory, format='png')
    plt.close('all')

def plot_confusion_matrix_seaborn(cm, nb_class, output_directory, title='Confusion Matrix'):
    labels = []
    for i in range(nb_class):
        labels.append(("Cls." + str(i)))
    sns.set(font_scale=1.5)
    f, hm = plt.subplots(figsize=(25, 25))
    hm = sns.heatmap(cm,
                     xticklabels=True,
                     yticklabels=True,
                     cbar=True,
                     annot=True,
                     square=True,
                     fmt='d',
                     annot_kws={'size': 20})
    hm.set_xticklabels(labels, fontsize=18, horizontalalignment='right')
    hm.set_yticklabels(labels, fontsize=18, horizontalalignment='right')
    plt.title(title)
    plt.ylabel('True label')
    plt.xlabel('Predicted label')
    plt.savefig('cc.png', format='png')
    plt.savefig(output_directory, format='png')
    plt.close('all')

def plot_metrics_matrix(Precision, Recall, F1, nb_classes, output_directory):
    metrics = np.zeros((3, nb_classes))
    for i in range(3):
        for j in range(nb_classes):
            if i == 0:
                metrics[i, j] = Precision[j, 0]
            elif i == 1:
                metrics[i, j] = Recall[j, 0]
            else:
                metrics[i, j] = F1[j, 0]
    col_labels = []
    for i in range(nb_classes):
        col_labels.append("Cls." + str(i))
    row_labels = ['Precision', 'Recall', 'F1-score']
    sns.set(font_scale=2.5)
    f, hm = plt.subplots(figsize=(25, 25))
    hm = sns.heatmap(metrics,
                     xticklabels=True,
                     yticklabels=True,
                     cmap="YlGnBu",
                     cbar=True,
                     annot=True,
                     square=True,
                     fmt='.2f',
                     annot_kws={'size': 20})
    hm.set_xticklabels(col_labels, fontsize=18, horizontalalignment='right')
    hm.set_yticklabels(row_labels, fontsize=18, horizontalalignment='right')
    plt.title('Metrics')
    plt.savefig(output_directory, format='png')
    plt.close('all')

def save_training_logs(output_directory, hist, lr=True):
    hist_df = pd.DataFrame(hist.history)
    hist_df.to_csv(output_directory + 'history.csv', index=False)
    index_best_model = hist_df['val_loss'].idxmin()
    row_best_model = hist_df.loc[index_best_model]

    df_best_model = pd.DataFrame(data=np.zeros((1, 12), dtype=np.float), index=[0],
                                 columns=['best_model_train_loss', 'best_model_val_loss',
                                          'best_model_train_acc', 'best_model_val_acc',
                                          'best_model_train_precision', 'best_model_val_precision',
                                          'best_model_train_recall', 'best_model_val_recall',
                                          'best_model_train_f1', 'best_model_val_f1',
                                          'best_model_learning_rate', 'best_model_nb_epoch'])

    df_best_model['best_model_train_loss'] = row_best_model['loss']
    df_best_model['best_model_val_loss'] = row_best_model['val_loss']
    df_best_model['best_model_train_acc'] = row_best_model['accuracy']
    df_best_model['best_model_val_acc'] = row_best_model['val_accuracy']
    df_best_model['best_model_train_precision'] = row_best_model['precision']
    df_best_model['best_model_val_precision'] = row_best_model['val_precision']
    df_best_model['best_model_train_recall'] = row_best_model['recall']
    df_best_model['best_model_val_recall'] = row_best_model['val_recall']
    df_best_model['best_model_train_f1'] = row_best_model['f1']
    df_best_model['best_model_val_f1'] = row_best_model['val_f1']

    if lr == True:
        df_best_model['best_model_learning_rate'] = row_best_model['lr']
    df_best_model['best_model_nb_epoch'] = index_best_model

    df_best_model.to_csv(output_directory + 'df_best_model.csv', index=False)

    # plot losses
    plot_epochs_metric(hist, output_directory + 'epochs_loss.png')
    plot_epochs_metric(hist, output_directory + 'epochs_accuracy.png', metric='accuracy')
    plot_epochs_metric(hist, output_directory + 'epochs_precision.png', metric='precision')
    plot_epochs_metric(hist, output_directory + 'epochs_recall.png', metric='recall')
    plot_epochs_metric(hist, output_directory + 'epochs_f1.png', metric='f1')

    return df_best_model

def save_evaluating_result(output_directory, y_pred, y_true, nb_classes):
    df_metrics = calculate_metrics(y_true, y_pred)

    cvconfusion = confusion_matrix(y_true, y_pred)

    # cvconfusion = np.zeros((nb_classes,nb_classes))
    # cvconfusion = confusion_matrix(y_true,y_pred)
    # cvconfusion_val = np.zeros((nb_classes,nb_classes))
    # cvconfusion_val = confusion_matrix(y_true,y_pred)

    classes = list(np.array(range(nb_classes)))

    df_confusion = pd.DataFrame(data=cvconfusion, index=classes, columns=classes)

    df_confusion.to_csv(output_directory + 'confusion_matrix.csv')

    plot_confusion_matrix(cvconfusion, nb_classes, output_directory + 'confusion_matrix.png')

    # print("hhhh")

    plot_confusion_matrix_seaborn(cvconfusion, nb_classes, output_directory + 'confusion_matrix(2).png')

    F1 = np.zeros((nb_classes, 1))
    Precision = np.zeros((nb_classes, 1))
    Recall = np.zeros((nb_classes, 1))

    for i in range(nb_classes):
        F1[i, 0] = 2 * cvconfusion[i, i] / (
                np.sum(cvconfusion[i, :]) + np.sum(cvconfusion[:, i]))
        Precision[i, 0] = cvconfusion[i, i] / np.sum(cvconfusion[:, i])
        Recall[i, 0] = cvconfusion[i, i] / np.sum(cvconfusion[i, :])

    df_metrics['F1(all)'] = np.mean(F1[0:nb_classes])

    for i in range(nb_classes):
        df_metrics['Precison(Cla.' + str(i) + ')'] = Precision[i, 0]
        df_metrics['Recall(Cla.' + str(i) + ')'] = Recall[i, 0]
        df_metrics['F1(Cla.' + str(i) + ')'] = F1[i, 0]

    plot_metrics_matrix(Precision, Recall, F1, nb_classes, output_directory + 'metrics.png')

    df_metrics.to_csv(output_directory + 'df_metrics.csv', index=False)

    return cvconfusion, df_metrics

reason: import numpy,import scipy,from scipy


text:
from numpy.distutils.fcompiler import FCompiler

compilers = ['HPUXFCompiler']

class HPUXFCompiler(FCompiler):

    compiler_type = 'hpux'
    description = 'HP Fortran 90 Compiler'
    version_pattern =  r'HP F90 (?P<version>[^\s*,]*)'

    executables = {
        'version_cmd'  : ["f90", "+version"],
        'compiler_f77' : ["f90"],
        'compiler_fix' : ["f90"],
        'compiler_f90' : ["f90"],
        'linker_so'    : ["ld", "-b"],
        'archiver'     : ["ar", "-cr"],
        'ranlib'       : ["ranlib"]
        }
    module_dir_switch = None #XXX: fix me
    module_include_switch = None #XXX: fix me
    pic_flags = ['+Z']
    def get_flags(self):
        return self.pic_flags + ['+ppu', '+DD64']
    def get_flags_opt(self):
        return ['-O3']
    def get_libraries(self):
        return ['m']
    def get_library_dirs(self):
        opt = ['/usr/lib/hpux64']
        return opt
    def get_version(self, force=0, ok_status=[256,0,1]):
        # XXX status==256 may indicate 'unrecognized option' or
        #     'no input file'. So, version_cmd needs more work.
        return FCompiler.get_version(self,force,ok_status)

if __name__ == '__main__':
    from distutils import log
    log.set_verbosity(10)
    from numpy.distutils.fcompiler import new_fcompiler
    compiler = new_fcompiler(compiler='hpux')
    compiler.customize()
    print(compiler.get_version())

max_stars_repo_name: walshb/numpy

max_issues_repo_name: walshb/numpy

max_forks_repo_name: walshb/numpy

reason: from numpy


text:
import argparse
import multiprocessing
import numpy as np
from shutil import copyfile
import os

from franka_panda.pybullet_simulation.push_cube_demo import cube_demo
from franka_panda.pybullet_simulation.push_rope_demo import rope_demo

class SimulationWorker(multiprocessing.Process):
    def __init__(self, id_range, run_function, output_dir, texture, n_frames, config_path: str):
        multiprocessing.Process.__init__(self)
        self.id_range = id_range
        self.run_function = run_function
        self.output_dir = output_dir
        self.texture = texture
        self.n_frames = n_frames
        self.config_path=config_path

    def run(self):
        self.run_function(output_dir=self.output_dir,
                          id=list(range(self.id_range[0], self.id_range[1])),
                          use_ui=False,
                          generate_alternative=self.texture,
                          random_size=True,
                          random_light=True,
                          random_camera=False,
                          n_frames=self.n_frames,
                          config_path=self.config_path)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-d', '--output_dir', required=True, type=str, help='path to store generated data.')
    parser.add_argument('-n', '--n_workers', type=int, default=1, help='number of workers')
    parser.add_argument('--n_start', default=1, type=int, help='init video.')
    parser.add_argument('--n_end', default=11, type=int, help='end video.')
    parser.add_argument('-t', '--texture', action='store_true', required=False, help='2 textures')
    parser.add_argument('-l', '--video_length', type=int, default=10, required=False, help='video length')
    parser.add_argument('-e', '--env', type=str, default="cube", required=False, help='either cube or rope')
    parser.add_argument('-c', '--config_path', type=str, default="default_push.yml",
                        required=False, help='path to config file')

    args = parser.parse_args()

    os.makedirs(args.output_dir, exist_ok=True)
    config_path = args.config_path
    if not os.path.exists(config_path):
        config_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "pybullet_simulation", "configs", config_path)

    copyfile(config_path, os.path.join(args.output_dir, "simulation_config.yml"))
    n_per_worker = int((args.n_end - args.n_start) / args.n_workers)
    id_range = [[i, np.minimum(i + n_per_worker, args.n_end)] for i in range(args.n_start, args.n_end, n_per_worker)]

    env_func = rope_demo if args.env == "rope" else cube_demo
    workers = [SimulationWorker(output_dir=args.output_dir,
                                id_range=id_range[i],
                                run_function=env_func,
                                texture=args.texture,
                                n_frames=args.video_length,
                                config_path=config_path)
               for i in range(args.n_workers)]

    if args.n_workers == 1:
        workers[0].run()
    else:
        for w in workers:
            w.start()

reason: import numpy


text:
# -*- coding: utf-8 -*-
#
# @Author   : Yuxiang WU
# @Email    : elephantameler@gmail.com

import random
import numpy as np
from typing import NewType, List, Tuple, Dict

DatasetIndexTuple = Tuple[int, int]

def split_processing_streams(paths, max_mats_one_stream = 8):
    streams = [paths[i * max_mats_one_stream: (i + 1) * max_mats_one_stream] for i in range(int(np.ceil(len(paths) / max_mats_one_stream)))]
    return streams

def split_one_individual_dataset_name(label: Dict, train_index: DatasetIndexTuple, val_index: DatasetIndexTuple, test_index: DatasetIndexTuple, shuffle: bool = False):
    names = sorted(label.keys())
    dataset_name = [names[i0: i1] for (i0, i1) in [train_index, val_index, test_index]]
    dataset_name = [random.sample(p, len(p)) for p in dataset_name] if shuffle else dataset_name
    return dataset_name

def split_multi_individuals_datasets(labels: List[Dict], indexes: List[List[DatasetIndexTuple]], shuffle_type: int = 0):
    """

    :param labels:
    :param indexes:
    :param shuffle_type: 0 - no shuffle, 1 - shuffle within a individual, 2 - shuffle all
    :return:
    """
    merged_label = {k: v for label in labels for k, v in label.items()}

    train_names, val_names, test_names = list(), list(), list()
    for label, index_tuple in zip(labels, indexes):
        train_name, val_name, test_name = split_one_individual_dataset_name(label, *index_tuple, shuffle = (shuffle_type == 1))
        train_names.extend(train_name)
        val_names.extend(val_name)
        test_names.extend(test_name)
    if shuffle_type == 2:
        train_len, val_len, test_len = sum([idx[0][1] - idx[0][0] for idx in indexes]), sum([idx[1][1] - idx[1][0] for idx in indexes]), sum([idx[2][1] - idx[2][0] for idx in indexes])
        names = random.sample(train_names + val_names + test_names, train_len + val_len + test_len)
        train_names, val_names, test_names = names[: train_len], names[train_len: train_len + val_len], names[train_len + val_len:]
    train_names, val_names, test_names = sorted(train_names), sorted(val_names), sorted(test_names)

    selected_label = {k: merged_label[k] for k in sorted(train_names + val_names + test_names)}
    return selected_label, (train_names, val_names, test_names)

max_stars_repo_name: Wenlab/cendr

max_issues_repo_name: Wenlab/cendr

max_forks_repo_name: Wenlab/cendr


text:
from collections import defaultdict
from pathlib import Path
from time import time

import h5py
import numpy as np
from tqdm import tqdm

class MatDataExtractor:
    def __init__(self, file_name):
        self.file_name = Path(file_name)
        assert self.file_name.suffix == ".mat"
        self._open_file = h5py.File(self.file_name, "r")
        self.trials = self._open_file["trials"]
        self.npix_meta = self._open_file["npix_meta"]
        self.trial_colnames = list(self.trials.keys())
        self._no_trials = max(self.trials[self.trial_colnames[0]].shape)
        self._no_units = max(self.npix_meta["cluster_ids"].shape)
        subject_array = self._open_file[self.trials["subject"][0, 0]]
        self.subject_name = "".join(
            [chr(subject_array[i, 0]) for i in range(subject_array.shape[0])]
        )

    def get_trial_times(self):
        """
        Trial start and end times for given trial nos.
        """
        start_time_list = []
        stop_time_list = []
        for trial_no in range(self._no_trials):
            start_time_list.append(
                self._open_file[self.trials["npix_start_idx"][0, trial_no]][0, 0]/3e4
            )
            stop_time_list.append(
                self._open_file[self.trials["npix_stop_idx"][0, trial_no]][0, 0]/3e4
            )
        return np.array(start_time_list), np.array(stop_time_list)

    def get_trial_epochs(self):
        """
        Trial start and end times for given trial nos.
        """
        trial_start, trial_end = self.get_trial_times()

        def get_val(name, trial_no):
            return (
                    np.array(self._return_trial_value(name, trial_no=trial_no)).flatten()[0]
                    *1e-3
                    + trial_start[trial_no]*1e-3
            )

        events_list = {
            "target_onset_time": "TargetOnset",
            "go_cue_time": "GoCue",
            "move_start_time": "Move",
            "move_end_time": "MoveEnd",
            "target_acquired_time": "TargetAcquired",
            "target_held_time": "TargetHeld",
            "delay_period": "delay",
            "reaction_time": "rt",
        }

        events_dict = defaultdict(dict)
        for event_name in events_list:
            events_dict[event_name].update(
                data=[
                    get_val(events_list[event_name], i) for i in range(self._no_trials)
                ],
                description=f"{events_list[event_name]} time in s",
            )

        return events_dict

    def _return_trial_value(self, field, trial_no=0):
        return self._open_file[self.trials[field][0, trial_no]]

    def get_behavior_movement(self):
        trial_start, trial_end = self.get_trial_times()
        beh_fields = {"hand_position": "handPosition", "hand_speed": "handSpeed"}
        beh_dict = defaultdict(dict)
        for field in beh_fields:
            beh_dict[field].update(
                data=np.concatenate(
                    [
                        np.array(self._return_trial_value(beh_fields[field], i)).T
                        *1e-3
                        for i in range(self._no_trials)
                    ],
                    axis=0,
                ).squeeze(),
                description=f"{field} x,y,z in m",
            )
        beh_dict["times"].update(
            data=np.concatenate(
                [
                    np.array(self._return_trial_value("hand_time", i)).T*1e-3
                    + trial_start[i]
                    for i in range(self._no_trials)
                ],
                axis=0,
            ).squeeze(),
            description="time vector in s",
        )
        return beh_dict

    def get_trial_ids(self):
        return np.array(
            [
                self._return_trial_value("trialId", i)[0, 0]
                for i in range(self._no_trials)
            ]
        )

    def get_task_details(self):
        task_fields = {
            "centerX": "center hold location x: screen center origin, direction: right",
            "centerY": "center hold location y: screen center origin, direction: top",
            "targetX": "x position of target",
            "targetY": "y position of target",
            "targetSize": "target dia",
            "targetDirection": "reach direction in radians, measured clockwise from +X axis at 3:00",
            "targetDistance": "distance to reaching target",
            "saveTag": "internal trial grouping indicator",
            "success": "boolean indicator of trial success, these will be all be true by filtering",
        }
        task_dict = defaultdict(dict)
        for field, desc in task_fields.items():
            task_dict[field].update(
                data=[
                    self._return_trial_value(field, i)[0, 0]
                    for i in range(self._no_trials)
                ],
                description=desc,
            )
        return task_dict

    def extract_unit_spike_times(self, spike_ids: list = None):
        no_neurons = len(self._open_file[self.trials["npix"][0, 0]])
        if spike_ids is None:
            spike_ids = np.arange(no_neurons)
        trial_nos = np.arange(self._no_trials)
        spike_times_all_list = []
        trial_start, _ = self.get_trial_times()
        for _ in spike_ids:
            spike_times_all_list.append([])
        start = time()
        for trl in tqdm(trial_nos):
            sptimes = self._return_trial_value("npix", trl)
            for id in spike_ids:
                spike_times_all_list[id].extend(
                    (
                            self._open_file[sptimes[id, 0]][:].flatten()*1e-3
                            + trial_start[trl]
                    ).tolist()
                )
        print(time() - start)
        return spike_times_all_list

    def extract_unit_details(self, selfspike_ids: list = None):
        default_unit_args = {
            "cluster_ids": "id",
            "cluster_waveform_ch": "electrodes",
            "cluster_waveform_template": "waveform_mean",
        }
        custom_unit_args = {
            "cluster_amplitude": "peak to peak amplitude in uV",
            "cluster_is_localized": "boolean indicator of whether or not center of mass of "
                                    "waveform is well defined",
            # 'cluster_layer':'one of "Superficial", "Deep", "White" (white matter), or '
            #                 '"Unknown", cluster layer identified via current source density '
            #                 'and peak first waveforms analysis',
            "cluster_position_aligned": "spatial positions of neurons on probe, "
                                        "aligned using CSD analysis. for Y, 0 corresponds"
                                        " to the nominal cortical surface, and increasingly "
                                        "negative values indicate deeper in cortex towards white matter",
            "cluster_positions": "spatial positions of neurons on probe, using raw probe coordinates "
                                 "from the channel map. 0 corresponds to the tip of the probe, "
                                 "and increasingly positive values indicate shallower in cortex "
                                 "towards the surface (note: same orientation as aligned, "
                                 "different offset)",
            # 'cluster_ratings':'one of "good" or "unstable", individually hand-reviewed indicator '
            #                   'of neuron stability over trials during the session'
        }
        default_dict = defaultdict(dict)
        custom_dict = defaultdict(dict)
        for arg in default_unit_args:
            default_dict[default_unit_args[arg]].update(
                data=np.concatenate(
                    [
                        self.npix_meta[arg][:, i].flatten()[:, np.newaxis]
                        for i in range(self._no_units)
                    ],
                    axis=1,
                ).T
            )
        for arg in custom_unit_args:
            custom_dict[arg].update(
                data=np.concatenate(
                    [
                        self.npix_meta[arg][:, i].flatten()[:, np.newaxis]
                        for i in range(self._no_units)
                    ],
                    axis=1,
                ).T,
                description=custom_unit_args[arg],
            )
        return default_dict, custom_dict

max_stars_repo_name: catalystneuro/shenoy-lab-to-nwb

max_issues_repo_name: catalystneuro/shenoy-lab-to-nwb

max_forks_repo_name: catalystneuro/shenoy-lab-to-nwb


text:
import numpy as np
import matplotlib.pyplot as plt
import sys
try:
    fn=sys.argv[1]
except:
    fn='log.dat'
d=np.loadtxt(fn)
##index, dx, fy, dfydx

t=np.argmin(d[:,2])
print(d[t,1])

t=np.argmin(np.abs(d[:,3]))
print(d[t,1])

#-0.122

dx = d[1,1]-d[0,1]

x1 = d[1:-1,1]
f1 = (d[2:,2]-d[:-2,2])/dx/2.0
plt.figure(0)
plt.plot(d[:,1],d[:,3],'r',label='from code')
plt.plot(x1,f1,'g--', label='numerical diff')
plt.plot(d[:,1], np.zeros_like(d[:,1]))
plt.legend()
plt.ylabel('dfy_dx (eV/Angstrom^2)')
#plt.xlim([0.0,10])

plt.figure(1)
df = d[1:-1,3]-f1
plt.plot(x1[:],df[:])
plt.ylabel('dfy_dx difference (eV/Angstrom)')
#plt.xlim([0.0,10])

plt.figure(2)
plt.plot(d[:,1], d[:,2])
plt.ylabel('fy (eV/Angstrom)')
#plt.xlim([0.0,10])

plt.show()


text:
import pyclesperanto_prototype as cle
import numpy as np

def test_mean_x_projection():

    test1 = cle.push(np.asarray([
        [
            [1, 0, 0, 0, 9],
            [0, 2, 0, 8, 0],
            [3, 0, 1, 0, 10],
            [0, 4, 0, 7, 0],
            [5, 0, 6, 0, 10]
        ],[
            [0, 2, 0, 8, 0],
            [1, 0, 0, 0, 9],
            [3, 0, 1, 0, 10],
            [0, 4, 0, 7, 0],
            [5, 0, 6, 0, 10]
        ],[
            [0, 2, 0, 8, 0],
            [3, 0, 1, 0, 10],
            [0, 4, 0, 7, 0],
            [1, 0, 0, 0, 9],
            [5, 0, 6, 0, 10]
        ],[
            [0, 2, 0, 8, 0],
            [1, 0, 0, 0, 9],
            [0, 4, 0, 7, 0],
            [3, 0, 1, 0, 10],
            [5, 0, 6, 0, 10]
        ],[
            [1, 0, 0, 0, 9],
            [0, 4, 0, 7, 0],
            [3, 0, 1, 0, 10],
            [0, 2, 0, 8, 0],
            [5, 0, 6, 0, 10]
        ]
    ]).T)

    reference = cle.push(np.asarray([
        [0.4, 1.0, 1.8, 0.8, 5.],
        [1.2, 1.2, 1.6, 2.0, 0.],
        [0.,  0.2, 0.6, 0.2, 6.],
        [4.8, 3.0, 2.8, 4.4, 0.],
        [3.6, 5.6, 6.0, 3.8, 10.]
    ]).T)

    result = cle.create(reference)
    cle.mean_x_projection(test1, result)

    a = cle.pull(result)
    b = cle.pull(reference)

    print(a)
    print(b)

    assert (np.allclose(a, b, 0.001))


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri May 18 22:38:30 2018

@author: jack.lingheng.meng
"""

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue May 15 09:43:11 2018

@author: jack.lingheng.meng
"""

import tensorflow as tf
import numpy as np
import gym
from gym import wrappers
import tflearn
import argparse
import pprint as pp
from collections import deque
import time

from IPython.core.debugger import Tracer

from Environment.LASEnv import LASEnv
from LASAgent.LASAgent_Actor_Critic import LASAgent_Actor_Critic

import matplotlib.pyplot as plt
def plot_cumulative_reward(cumulativeReward):
    line, = plt.plot(cumulativeReward)
    plt.ion()
    plt.show()
    plt.pause(0.0001)

if __name__ == '__main__':

    with tf.Session() as sess:

        #env = gym.make('Pendulum-v0')
        #env = gym.make('MountainCarContinuous-v0')
        env = LASEnv(IP = '127.0.0.1',
                     Port = 19997,
                     reward_function_type = 'red_light_dense')

        LASAgent = LASAgent_Actor_Critic(sess, env,
                                         actor_lr = 0.0001, actor_tau = 0.001,
                                         critic_lr = 0.0001, critic_tau = 0.001, gamma = 0.99,
                                         minibatch_size = 64,
                                         max_episodes = 50000, max_episode_len = 1000,
                                         # Exploration Strategies
                                         exploration_action_noise_type = 'ou_0.2',
                                         exploration_epsilon_greedy_type = 'none',
                                         # Save Summaries
                                         save_dir = '../LAS_gym_results/LASAgentActorCritic/',
                                         experiment_runs = 'run1',
                                         # Save and Restore Actor-Critic Model
                                         restore_actor_model_flag = False,
                                         restore_critic_model_flag = False)

        # Learning records
        episod_reward_memory = deque(maxlen = 10000)

        # Train parameters
        max_episodes = 50000
        max_episode_len = 1000
        render_env = False
        reward = 0
        done = False
        start_time = time.time()

        try:
            for i in range(max_episodes):
                observation = env.reset()
                ep_reward = 0
                for j in range(max_episode_len):

                    if render_env == True:
                        env.render()

                    # Added exploration noise
                    action = LASAgent.perceive_and_act(observation,reward,done)

                    observation, reward, done, info = env.step(action[0])
                    ep_reward += reward
                    if done or j == (max_episode_len-1):
                        print('| Reward: {:d} | Episode: {:d} '.format(int(ep_reward),i))
                        episod_reward_memory.append(ep_reward)
                        plot_cumulative_reward(episod_reward_memory)
                        break
                    #time.sleep(0.5)
                print("Time elapsed:{}".format(time.time()-start_time))
        except KeyboardInterrupt:
            sess.close()
            env.destroy()
            print("Shut Down.")


text:
import sys

import numpy as np
import tensorflow as tf
from keras import backend as K
from keras.layers import Lambda
from keras.layers.merge import concatenate
from keras.models import Model

from ..utils import compose
from .keras_darknet19 import (DarknetConv2D, DarknetConv2D_BN_Leaky, darknet_body)

sys.path.append('..')

voc_anchors = np.array(
    [[1.08, 1.19], [3.42, 4.41], [6.63, 11.38], [9.42, 5.11], [16.62, 10.52]])

voc_classes = [
    "aeroplane", "bicycle", "bird", "boat", "bottle", "bus", "car", "cat",
    "chair", "cow", "diningtable", "dog", "horse", "motorbike", "person",
    "pottedplant", "sheep", "sofa", "train", "tvmonitor"
]

def space_to_depth_x2(x):

    import tensorflow as tf
    return tf.space_to_depth(x, block_size=2)

def space_to_depth_x2_output_shape(input_shape):

    return (input_shape[0], input_shape[1] // 2, input_shape[2] // 2, 4 *
            input_shape[3]) if input_shape[1] else (input_shape[0], None, None,
                                                    4 * input_shape[3])

def yolo_body(inputs, num_anchors, num_classes):

    darknet = Model(inputs, darknet_body()(inputs))
    conv20 = compose(
        DarknetConv2D_BN_Leaky(1024, (3, 3)),
        DarknetConv2D_BN_Leaky(1024, (3, 3)))(darknet.output)

    conv13 = darknet.layers[43].output
    conv21 = DarknetConv2D_BN_Leaky(64, (1, 1))(conv13)

    conv21_reshaped = Lambda(
        space_to_depth_x2,
        output_shape=space_to_depth_x2_output_shape,
        name='space_to_depth')(conv21)

    x = concatenate([conv21_reshaped, conv20])
    x = DarknetConv2D_BN_Leaky(1024, (3, 3))(x)
    x = DarknetConv2D(num_anchors * (num_classes + 5), (1, 1))(x)
    return Model(inputs, x)

def yolo_head(feats, anchors, num_classes):

    num_anchors = len(anchors)

    anchors_tensor = K.reshape(K.variable(anchors), [1, 1, 1, num_anchors, 2])

    conv_dims = K.shape(feats)[1:3]

    conv_height_index = K.arange(0, stop=conv_dims[0])
    conv_width_index = K.arange(0, stop=conv_dims[1])
    conv_height_index = K.tile(conv_height_index, [conv_dims[1]])

    conv_width_index = K.tile(K.expand_dims(conv_width_index, 0), [conv_dims[0], 1])
    conv_width_index = K.flatten(K.transpose(conv_width_index))
    conv_index = K.transpose(K.stack([conv_height_index, conv_width_index]))
    conv_index = K.reshape(conv_index, [1, conv_dims[0], conv_dims[1], 1, 2])
    conv_index = K.cast(conv_index, K.dtype(feats))

    feats = K.reshape(feats, [-1, conv_dims[0], conv_dims[1], num_anchors, num_classes + 5])
    conv_dims = K.cast(K.reshape(conv_dims, [1, 1, 1, 1, 2]), K.dtype(feats))

    box_confidence = K.sigmoid(feats[..., 4:5])
    box_xy = K.sigmoid(feats[..., :2])
    box_wh = K.exp(feats[..., 2:4])
    box_class_probs = K.softmax(feats[..., 5:])

    box_xy = (box_xy + conv_index) / conv_dims
    box_wh = box_wh * anchors_tensor / conv_dims

    return box_confidence, box_xy, box_wh, box_class_probs

def yolo_boxes_to_corners(box_xy, box_wh):

    box_mins = box_xy - (box_wh / 2.)
    box_maxes = box_xy + (box_wh / 2.)

    return K.concatenate([
        box_mins[..., 1:2],
        box_mins[..., 0:1],
        box_maxes[..., 1:2],
        box_maxes[..., 0:1]
    ])

def yolo_loss(args,
              anchors,
              num_classes,
              rescore_confidence=False,
              print_loss=False):

    (yolo_output, true_boxes, detectors_mask, matching_true_boxes) = args
    num_anchors = len(anchors)
    object_scale = 5
    no_object_scale = 1
    class_scale = 1
    coordinates_scale = 1
    pred_xy, pred_wh, pred_confidence, pred_class_prob = yolo_head(
        yolo_output, anchors, num_classes)

    yolo_output_shape = K.shape(yolo_output)
    feats = K.reshape(yolo_output, [
        -1, yolo_output_shape[1], yolo_output_shape[2], num_anchors,
        num_classes + 5
    ])
    pred_boxes = K.concatenate(
        (K.sigmoid(feats[..., 0:2]), feats[..., 2:4]), axis=-1)

    pred_xy = K.expand_dims(pred_xy, 4)
    pred_wh = K.expand_dims(pred_wh, 4)

    pred_wh_half = pred_wh / 2.
    pred_mins = pred_xy - pred_wh_half
    pred_maxes = pred_xy + pred_wh_half

    true_boxes_shape = K.shape(true_boxes)

    true_boxes = K.reshape(true_boxes, [
        true_boxes_shape[0], 1, 1, 1, true_boxes_shape[1], true_boxes_shape[2]
    ])
    true_xy = true_boxes[..., 0:2]
    true_wh = true_boxes[..., 2:4]

    true_wh_half = true_wh / 2.
    true_mins = true_xy - true_wh_half
    true_maxes = true_xy + true_wh_half

    intersect_mins = K.maximum(pred_mins, true_mins)
    intersect_maxes = K.minimum(pred_maxes, true_maxes)
    intersect_wh = K.maximum(intersect_maxes - intersect_mins, 0.)
    intersect_areas = intersect_wh[..., 0] * intersect_wh[..., 1]

    pred_areas = pred_wh[..., 0] * pred_wh[..., 1]
    true_areas = true_wh[..., 0] * true_wh[..., 1]

    union_areas = pred_areas + true_areas - intersect_areas
    iou_scores = intersect_areas / union_areas

    best_ious = K.max(iou_scores, axis=4)  # Best IOU scores.
    best_ious = K.expand_dims(best_ious)

    object_detections = K.cast(best_ious > 0.6, K.dtype(best_ious))

    no_object_weights = (no_object_scale * (1 - object_detections) *
                         (1 - detectors_mask))
    no_objects_loss = no_object_weights * K.square(-pred_confidence)

    if rescore_confidence:
        objects_loss = (object_scale * detectors_mask *
                        K.square(best_ious - pred_confidence))
    else:
        objects_loss = (object_scale * detectors_mask *
                        K.square(1 - pred_confidence))
    confidence_loss = objects_loss + no_objects_loss

    matching_classes = K.cast(matching_true_boxes[..., 4], 'int32')
    matching_classes = K.one_hot(matching_classes, num_classes)
    classification_loss = (class_scale * detectors_mask *
                           K.square(matching_classes - pred_class_prob))

    matching_boxes = matching_true_boxes[..., 0:4]
    coordinates_loss = (coordinates_scale * detectors_mask *
                        K.square(matching_boxes - pred_boxes))

    confidence_loss_sum = K.sum(confidence_loss)
    classification_loss_sum = K.sum(classification_loss)
    coordinates_loss_sum = K.sum(coordinates_loss)
    total_loss = 0.5 * (
        confidence_loss_sum + classification_loss_sum + coordinates_loss_sum)
    if print_loss:
        total_loss = tf.Print(
            total_loss, [
                total_loss, confidence_loss_sum, classification_loss_sum,
                coordinates_loss_sum
            ],
            message='yolo_loss, conf_loss, class_loss, box_coord_loss:')

    return total_loss

def yolo(inputs, anchors, num_classes):

    num_anchors = len(anchors)
    body = yolo_body(inputs, num_anchors, num_classes)
    outputs = yolo_head(body.output, anchors, num_classes)
    return outputs

def yolo_filter_boxes(box_confidence, boxes, box_class_probs, threshold=.6):

    box_scores = box_confidence * box_class_probs
    box_classes = K.argmax(box_scores, axis=-1)
    box_class_scores = K.max(box_scores, axis=-1)
    prediction_mask = box_class_scores >= threshold

    # TODO: Expose tf.boolean_mask to Keras backend?
    boxes = tf.boolean_mask(boxes, prediction_mask)
    scores = tf.boolean_mask(box_class_scores, prediction_mask)
    classes = tf.boolean_mask(box_classes, prediction_mask)

    return boxes, scores, classes

def yolo_eval(yolo_outputs,
              image_shape,
              max_boxes=10,
              score_threshold=.6,
              iou_threshold=.5):

    box_confidence, box_xy, box_wh, box_class_probs = yolo_outputs
    boxes = yolo_boxes_to_corners(box_xy, box_wh)
    boxes, scores, classes = yolo_filter_boxes(
        box_confidence, boxes, box_class_probs, threshold=score_threshold)

    height = image_shape[0]
    width = image_shape[1]
    image_dims = K.stack([height, width, height, width])
    image_dims = K.reshape(image_dims, [1, 4])
    boxes = boxes * image_dims

    max_boxes_tensor = K.variable(max_boxes, dtype='int32')
    K.get_session().run(tf.variables_initializer([max_boxes_tensor]))
    nms_index = tf.image.non_max_suppression(
        boxes, scores, max_boxes_tensor, iou_threshold=iou_threshold)
    boxes = K.gather(boxes, nms_index)
    scores = K.gather(scores, nms_index)
    classes = K.gather(classes, nms_index)

    return boxes, scores, classes

def preprocess_true_boxes(true_boxes, anchors, image_size):

    height, width = image_size
    num_anchors = len(anchors)

    assert height % 32 == 0,
    assert width % 32 == 0,
    conv_height = height // 32
    conv_width = width // 32
    num_box_params = true_boxes.shape[1]
    detectors_mask = np.zeros(
        (conv_height, conv_width, num_anchors, 1), dtype=np.float32)
    matching_true_boxes = np.zeros(
        (conv_height, conv_width, num_anchors, num_box_params),
        dtype=np.float32)

    for box in true_boxes:

        box_class = box[4:5]
        box = box[0:4] * np.array(
            [conv_width, conv_height, conv_width, conv_height])
        i = np.floor(box[1]).astype('int')
        j = min(np.floor(box[0]).astype('int'),1)
        best_iou = 0
        best_anchor = 0

        for k, anchor in enumerate(anchors):

            box_maxes = box[2:4] / 2.
            box_mins = -box_maxes
            anchor_maxes = (anchor / 2.)
            anchor_mins = -anchor_maxes

            intersect_mins = np.maximum(box_mins, anchor_mins)
            intersect_maxes = np.minimum(box_maxes, anchor_maxes)
            intersect_wh = np.maximum(intersect_maxes - intersect_mins, 0.)
            intersect_area = intersect_wh[0] * intersect_wh[1]
            box_area = box[2] * box[3]
            anchor_area = anchor[0] * anchor[1]
            iou = intersect_area / (box_area + anchor_area - intersect_area)
            if iou > best_iou:
                best_iou = iou
                best_anchor = k

        if best_iou > 0:
            detectors_mask[i, j, best_anchor] = 1
            adjusted_box = np.array(
                [
                    box[0] - j, box[1] - i,
                    np.log(box[2] / anchors[best_anchor][0]),
                    np.log(box[3] / anchors[best_anchor][1]), box_class
                ],
                dtype=np.float32)
            matching_true_boxes[i, j, best_anchor] = adjusted_box
    return detectors_mask, matching_true_boxes


text:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import json
import json_lines
from PIL import Image
from collections import Counter
from wordcloud import WordCloud, STOPWORDS

#Read the file and set artist list
df = pd.read_json (,lines=True)
list_artists = ['Ed Sheeran','Camila Cabello','Queen','Eminem','Drake','Post Malone','Khalid','Rihanna']

#Feature Engineering
df = df.drop(axis=0,labels=range(8))
df = df.reset_index()
df= df.drop(['index'],axis=1)
df = df.drop_duplicates(subset='lyrics')
df['artist'] = df[df['artist'].isin(list_artists)]
df = df.dropna()

#List of Stopwords
SW = ["i","I'm","m","am", "me", "my", "myself", "we", "our", "ours", "ourselves", "you", "your", "yours", "yourself", "yourselves", "he", "him", "his", "himself", "she", "her", "hers", "herself", "it", "its", "itself", "they", "them", "their", "theirs", "themselves", "what", "which", "who", "whom", "this", "that", "these", "those", "am", "is", "are", "was", "were", "be", "been", "being", "have", "has", "had", "having", "do", "does", "did", "doing", "a", "an", "the", "and", "but", "if", "or", "because", "as", "until", "while", "of", "at", "by", "for", "with", "about", "against", "between", "into", "through", "during", "before", "after", "above", "below", "to", "from", "up", "down", "in", "out", "on", "off", "over", "under", "again", "further", "then", "once", "here", "there", "when", "where", "why", "how", "all", "any", "both", "each", "few", "more", "most", "other", "some", "such", "no", "nor","oh", "not", "only", "own", "same", "so", "than", "too", "very", "s", "t", "can", "will", "just", "don", "should", "now"]

#Takes all songs from an artist (as string) and put them as a string
def raw (artist):
    df_a = df.loc[df['artist']==f'{artist}']
    a_lyrics = df_a['lyrics']
    a_lyrics = a_lyrics.tolist()
    a_lyrics = [x.replace('\n', '') for x in a_lyrics]
    a_lyrics_raw = (" ").join(a_lyrics)
    lyrics_raw = a_lyrics_raw
    return lyrics_raw

def face_cloud(mask,lyrics_raw):
    def transform_format(val):
        if val == 0:
            return 255
        else:
            return 1
    transformed_mask = np.ndarray((mask.shape[0],mask.shape[1]), np.int32)

    for i in range(len(mask)):
        transformed_mask[i] = list(map(transform_format, mask[i]))

    wc = WordCloud(background_color="black", max_words=100, mask=transformed_mask,
        contour_width=0.1,contour_color='grey',colormap='Greys',stopwords=SW)

    wc.generate(lyrics_raw)
    plt.imshow(wc)
    plt.axis("off")
    plt.figure(figsize=(10,10))
    plt.show()

def cloud(artist,mask):
    face_cloud(mask,raw(artist))

# cloud(input('Whose lyrics?'),drake_face)
singer = input('Whose lyrics? ')
face = input('Whose face? ')
if face == 'Drake':
    face = drake_face
elif face== 'Eminem':
    face = eminem_face
elif face== 'Queen':
    face =  queen_face
else: face = edSheeran_face

cloud(singer,face)


text:
#!/usr/bin/python3
#refactored and added audio length normalization and spectrogram code to https://github.com/manashmandal/DeadSimpleSpeechRecognizer
import librosa
import os
from sklearn.model_selection import train_test_split
import numpy as np
from tqdm import tqdm
import json

from util import *

SR=16000

def wav2mfcc(file_path, n_mfcc, max_sample_length=None):
    wave, sr = librosa.load(file_path, mono=True, sr=SR)
    #TODO see https://github.com/manashmandal/DeadSimpleSpeechRecognizer
    #I think this is like a primitive way of lowering the sample rate?
    #Smaller input would mean better perf but would it hurt accuracy?
    #If included, need to move this to data_to_mfcc because udp_listen uses that fn
    # wave = wave[::3]

    if max_sample_length:
        wave = random_pad(wave, max_sample_length)

    return data_to_mfcc(wave, n_mfcc)

def data_to_mfcc(data, n_mfcc):
    #upstream if we call data[::3] we need to call this or get NaN fortran continuous error
    wave = np.asfortranarray(data)

    #default n_mfcc is 20, 40 is used in kaggle tutorial
    mfcc = librosa.feature.mfcc(wave, sr=SR, n_mfcc=n_mfcc)

    return mfcc

def wav2spec(file_path, max_sample_length=None):
    wave, sr = librosa.load(file_path, mono=True, sr=SR)
    # wave = wave[::3]

    if max_sample_length:
        wave = random_pad(wave, max_sample_length)

    return data_to_spec(wave)

def data_to_spec(data):
    wave = np.asfortranarray(data)
    spectrogram = librosa.feature.melspectrogram(wave, sr=SR)
    log_spec = librosa.power_to_db(spectrogram, ref=np.max)
    return log_spec

def save_data_to_array(path=DATA_PATH, n_mfcc=40):
    labels, _, _ = get_labels(path)

    max_sample_length = max_sample_len()

    for label in labels:
        mfcc_vectors = []

        wavfiles = [path + label + '/' + wavfile for wavfile in os.listdir(path + '/' + label)]
        for wavfile in tqdm(wavfiles, "Saving vectors of label - '{}'".format(label)):
            mfcc = wav2mfcc(wavfile, n_mfcc=n_mfcc, max_sample_length=max_sample_length)
            mfcc_vectors.append(mfcc)
        np.save(PREPROC_PATH + label + '.npy', mfcc_vectors)

def save_raw_data_to_array(path=DATA_PATH):
    labels, _, _ = get_labels(path)

    max_sample_length = max_sample_len()

    for label in labels:
        raw_audio = []

        wavfiles = [path + label + '/' + wavfile for wavfile in os.listdir(path + '/' + label)]
        for wavfile in tqdm(wavfiles, "Saving audio of label - '{}'".format(label)):
            wave, sr = librosa.load(wavfile, mono=True, sr=SR)
            wave = random_pad(wave, max_sample_length)
            raw_audio.append(wave)
        np.save(PREPROC_PATH + label + '.npy', raw_audio)

def save_spec_data_to_array(path=DATA_PATH):
    labels, _, _ = get_labels(path)

    max_sample_length = max_sample_len()

    for label in labels:
        spectrograms = []

        wavfiles = [path + label + '/' + wavfile for wavfile in os.listdir(path + '/' + label)]
        for wavfile in tqdm(wavfiles, "Saving spectrograms of label - '{}'".format(label)):
            spec = wav2spec(wavfile, max_sample_length=max_sample_length)
            spectrograms.append(spec)
        np.save(PREPROC_PATH + label + '.npy', spectrograms)

def get_train_test(split_ratio=0.6, random_state=42):
    # Get available labels
    labels, indices, _ = get_labels(DATA_PATH)

    # Getting first arrays
    X = np.load(PREPROC_PATH + labels[0] + '.npy')
    y = np.zeros(X.shape[0])

    # Append all of the dataset into one single array, same goes for y
    for i, label in enumerate(labels[1:]):
        x = np.load(PREPROC_PATH + label + '.npy')
        #stack is [X,x]
        X = np.vstack((X, x))
        #append spreads [...y, ...np.full]
        y = np.append(y, np.full(x.shape[0], fill_value= (i + 1)))

    assert X.shape[0] == len(y)

    return train_test_split(X, y, test_size= (1 - split_ratio), random_state=random_state, shuffle=True)

max_stars_repo_name: grwgreg/streaming-audio-recognition

max_issues_repo_name: grwgreg/streaming-audio-recognition

max_forks_repo_name: grwgreg/streaming-audio-recognition


text:
import argparse
import os

import cv2

from keras.applications.vgg16 import VGG16, preprocess_input

import numpy as np

from sklearn.cluster import KMeans

from tensorflow.keras.models import Model

parser = argparse.ArgumentParser(description='Summarize the given text')
parser.add_argument('-p', '--path', help="Path to the directory", type=str)
parser.add_argument('-c', '--cluster', help="Number of cluster", type=int)
parser.add_argument('-o', '--output', help="output location (default: OUTPUT)",
                    type=str, default="./OUTPUT/")
args = parser.parse_args()

if not (args.path):
    parser.error("input path is not provided")
if not (args.cluster):
    parser.error("number of cluster is not provided")

PATH_TO_FILES = args.path
CLUSTER = args.cluster
OUTPUT = args.output

def get_model():
    base_model = VGG16(weights='imagenet', include_top=True)
    model = Model(inputs=base_model.input,
                  outputs=base_model.layers[-2].output)
    return model

def get_files(path_to_files, size):
    fn_imgs = []
    files = [file for file in os.listdir(path_to_files)]
    for file in files:
        img = cv2.resize(cv2.imread(os.path.join(path_to_files, file)), size)
        fn_imgs.append([file, img])
    return dict(fn_imgs)

def feature_vector(img_arr, model):
    if img_arr.shape[2] == 1:
        img_arr = img_arr.repeat(3, axis=2)
    arr4d = np.expand_dims(img_arr, axis=0)
    arr4d_pp = preprocess_input(arr4d)
    return model.predict(arr4d_pp)[0, :]

def feature_vectors(imgs_dict, model):
    f_vect = {}
    for fn, img in imgs_dict.items():
        f_vect[fn] = feature_vector(img, model)
    return f_vect

def clustering(img_feature_vector):
    images = list(img_feature_vector.values())
    kmeans = KMeans(n_clusters=CLUSTER, init='k-means++')
    kmeans.fit(images)
    y_kmeans = kmeans.predict(images)
    file_names = list(imgs_dict.keys())
    return y_kmeans, file_names

def separate(y_kmeans, file_names):
    n_clusters = CLUSTER
    cluster_path = OUTPUT
    path_to_files = PATH_TO_FILES
    if(not os.path.exists(cluster_path)):
        os.mkdir(cluster_path)
    for c in range(0, n_clusters):
        if not os.path.exists(cluster_path + 'cluster_' + str(c)):
            os.mkdir(cluster_path + 'cluster_' + str(c))
    for fn, cluster in zip(file_names, y_kmeans):
        image = cv2.imread(os.path.join(path_to_files, fn))
        cv2.imwrite(cluster_path + 'cluster_' + str(cluster) + '/' + fn, image)

if __name__ == '__main__':

    imgs_dict = get_files(path_to_files=PATH_TO_FILES, size=(224, 224))
    model = get_model()
    img_feature_vector = feature_vectors(imgs_dict, model)
    y_kmeans, file_names = clustering(img_feature_vector)
    separate(y_kmeans, file_names)

max_stars_repo_name: sre-harine/Automation-scripts

max_issues_repo_name: sre-harine/Automation-scripts

max_forks_repo_name: sre-harine/Automation-scripts


text:
from sympy import *
import matplotlib.pyplot as plt
import numpy as np

# The results from a research study in psychology are shown in the chart. Complete parts (a) and (b) below.

points = [ [ 1, 2 ], [ 2, 3 ], [ 3, 11 ], [ 4, 10 ], [ 5, 13 ], [ 6, 18], [ 7, 24] ]

def trapezoidal_rule( V, a, b, n ):
	area = 0

	steps = np.linspace( a, b, n + 1, endpoint = True )

	for i in range( 0, n + 1 ):

		v = V[ i ]

		if i == 0 or i == n:
			area += .5 * v
		else:
			area += v

	return area * ( b - a ) / n

def simpsons_rule( V, a, b, n ):
	area = 0

	steps = np.linspace( a, b, n + 1, endpoint = True )

	for i in range( 0, n + 1 ):

		v = V[ i ]

		if i == 0 or i == n:
			area += v
		elif i % 2 == 0:
			area += 2 * v
		else:
			area += 4 * v

	return area * ( b - a ) / ( 3*n )

p_x = []
p_y = []

for p in points:
	p_x.append( p[ 0 ] )
	p_y.append( p[ 1 ] )

a, b, n = min( p_x ), max( p_x ), len( p_x ) - 1

area = simpsons_rule( p_y, a, b, n )
round( area, 3 )

plt.plot( p_x, p_y, color = 'black', marker = 'o' )

plt.vlines( x = a, ymin = 0, ymax = p_y[a - 1], color = 'Red' )
plt.vlines( x = b, ymin = 0, ymax = p_y[b - 1], color = 'Red' )

plt.show()

max_stars_repo_name: bmoretz/Python-Playground

max_issues_repo_name: bmoretz/Python-Playground

max_forks_repo_name: bmoretz/Python-Playground

reason: import numpy,from sympy


text:
import json
import numpy as np

from nltk import FreqDist
from gensim.models.word2vec import Word2Vec

from data_processing.util import NumpyArrayEncoder
from text_models.abstract_model import AbstractModel

class RandomEmbeddingModel(AbstractModel):
    def __init__(
        self,
        train_corpus=None,
        vector_size=300,
        min_count=1,
        random_seed=42,
        rand_by_w2v=False,
        save_to_path="./",
        models_suffixes=None,
    ):
        super().__init__(
            vector_size=vector_size, seed=random_seed, save_to_path=save_to_path, models_suffixes=models_suffixes
        )
        self.min_count = min_count

        freq_dict = FreqDist()
        for docs in train_corpus:
            freq_dict.update(FreqDist(docs))

        dumb_w2v = None
        if rand_by_w2v:
            dumb_w2v = Word2Vec(vector_size=self.vector_size, seed=random_seed, min_count=self.min_count)
            dumb_w2v.build_vocab(train_corpus)

        self.model = {}
        for word, freq in freq_dict.items():
            if freq >= self.min_count:
                self.model[word] = dumb_w2v.wv[word] if rand_by_w2v else np.random.rand(self.vector_size)

    name = "random"

    def save(self, path):
        with open(path, "w+") as fp:
            json.dump(self.model, fp, cls=NumpyArrayEncoder)

    @classmethod
    def load(cls, path):
        with open(path) as json_file:
            model = json.load(json_file)
        for word, emb in model.items():
            model[word] = np.array(emb)

        random_model = RandomEmbeddingModel()
        random_model.model = model
        random_model.vector_size = len(list(model.values())[0])
        return random_model

    def train_from_scratch(self, corpus):
        pass

    def train_pretrained(self, corpus):
        pass

    def train_finetuned(self, base_corpus, extra_corpus):
        pass

max_stars_repo_name: JetBrains-Research/docs-fine-tuning

max_issues_repo_name: JetBrains-Research/docs-fine-tuning

max_forks_repo_name: JetBrains-Research/docs-fine-tuning


text:
import numpy as np

from enum import Enum
from time import time
from sklearn.cluster import KMeans
from .helper import centering
from .helper import to_dict

class Binning_Types(Enum):
    REGULAR = 1
    IRREGULAR = 2
    CLUSTERED = 3
    G2PROGRESSION = 4
    USER = 98
    SUBBINNING = 99

class Pad_Modes(Enum):
    EPSILON = 1
    MIRROR = 2

class Binning():

    def __init__(self, type, edges, random_seed=0):
        self.__type = type
        self.__edges = list(np.copy(edges))
        self.__counts = [len(x) - 1 for x in edges]
        self.__domain = [x.max() for x in self.edges]
        self.__random_seed = random_seed
        self.__distances = [edges_along_dim[1:] - edges_along_dim[:-1]
                            for edges_along_dim in edges]
        self.__centers = list(map(centering, edges))
        self.__total_volume = np.prod([edges_along_dim[-1] - edges_along_dim[0]
                                       for edges_along_dim in edges])

        mg_dists = np.meshgrid(*self.__distances, indexing='ij')
        self.__volumes = mg_dists[0]
        for mgd in mg_dists[1:]:
            self.__volumes = np.multiply(self.__volumes, mgd)

        self.__meshgrids = np.meshgrid(*self.__centers, indexing='ij')

    def copy(self):
        return Binning(self.type, self.edges, self.random_seed)

    @property
    def type(self):
        return self.__type

    @property
    def counts(self):
        return self.__counts

    @property
    def edges(self):
        return self.__edges

    @property
    def random_seed(self):
        return self.__random_seed

    @property
    def distances(self):
        return self.__distances

    @property
    def centers(self):
        return self.__centers

    @property
    def total_volume(self):
        return self.__total_volume

    @property
    def volumes(self):
        return self.__volumes

    @property
    def meshgrids(self):
        return self.__meshgrids

    @property
    def dimensions(self):
        return len(self.edges)

    @property
    def domain(self):
        return self.__domain

    def to_dict(self):
        return to_dict(self, [
            "type",
            ("edges", lambda x: [y.tolist() for y in x]),
            "random_seed"
        ])

    @staticmethod
    def from_dict(d):
        d["edges"] = [np.array(y) for y in d["edges"]]
        return Binning(**d)

class RegularBinning(Binning):

    def __init__(self, counts, domain):
        if isinstance(counts, int):
            counts = [counts] * len(domain)
        elif isinstance(counts, list) and isinstance(domain, list):
            if len(counts) != len(domain):
                raise Exception("Dimensions of counts and domain mismatch.")

        edges = self.__regularBinEdgeGenerator(counts, domain)
        super().__init__(type=Binning_Types.REGULAR,
                         edges=edges)

    def __regularBinEdgeGenerator(self, counts, domain):
        return [np.linspace(0.0, d, n + 1)
                for n, d in zip(counts, domain)]

class IrregularBinning(Binning):

    def __init__(self, counts, domain, spread=0.3):
        if isinstance(counts, int):
            counts = [counts] * len(domain)
        elif isinstance(counts, list) and isinstance(domain, list):
            if len(counts) != len(domain):
                raise Exception("Dimensions of counts and domain mismatch.")

        random_seed = np.random.randint(2**32-1)
        edges = self.__irregularBinEdgeGenerator(
            counts, domain, spread, random_seed)
        super().__init__(type=Binning_Types.IRREGULAR,
                         edges=edges,
                         random_seed=random_seed)

    def __irregularBinEdgeGenerator(self, counts, domain, spread,
                                    random_seed):
        def single_dim(count, domain):
            dst = np.random.randint(1, int(count * (1. + spread)), size=count)
            dst = dst / dst.sum()
            dst *= domain

            bins = []
            ws = 0
            for w in dst:
                ws += w
                bins.append(ws)

            bins = [0.0] + bins
            bins[-1] = domain
            return np.array(bins)

        np.random.seed(random_seed)
        return [single_dim(c, d) for c, d in zip(counts, domain)]

class ClusteredBinning(Binning):

    def __init__(self, counts, src):
        if isinstance(counts, int):
            counts = [counts] * len(src.domain)
        elif isinstance(counts, list) and isinstance(src.domain, list):
            if len(counts) != len(src.domain):
                raise Exception("Dimensions of counts and domain mismatch.")

        random_seed = np.random.randint(2**32-1)
        edges = self.__clusteredBinEdgeGenerator(counts, src.domain,
                                                 src.data, random_seed)
        super().__init__(type=Binning_Types.CLUSTERED,
                         edges=edges,
                         random_seed=random_seed)

    def __clusteredBinEdgeGenerator(self, counts, domain, data,
                                    random_seed):
        def single_dim(count, domain, data1D):
            kmeans = KMeans(n_clusters=count, init='k-means++',
                            random_state=random_seed)
            kmeans = kmeans.fit(data1D.reshape((data1D.shape[0], 1)))
            centers = np.unique(kmeans.cluster_centers_.reshape((count,)))
            return np.concatenate(([0], centering(centers), [domain]),
                                  axis=None)

        return [single_dim(c, d, col)
                for c, d, col in zip(counts, domain, data.T)]

class G2ProgressionBinning(Binning):

    def __init__(self, counts, domain):
        if isinstance(counts, int):
            counts = [counts] * len(domain)
        elif isinstance(counts, list) and isinstance(domain, list):
            if len(counts) != len(domain):
                raise Exception("Dimensions of counts and domain mismatch.")

        edges = self.__g2progressionBinEdgeGenerator(counts, domain)
        super().__init__(type=Binning_Types.G2PROGRESSION,
                         edges=edges)

    def __g2progressionBinEdgeGenerator(self, counts, domain):
        return [d / np.logspace(0.0, n, n + 1, base=2)[::-1]
                for n, d in zip(counts, domain)]

class BinningGenerator():

    @staticmethod
    def generate(type, counts, domain, src=None, spread=0.3):
        if type == Binning_Types.REGULAR:
            return RegularBinning(counts, domain)
        elif type == Binning_Types.IRREGULAR:
            return IrregularBinning(counts, domain, spread)
        elif type == Binning_Types.CLUSTERED:
            return ClusteredBinning(counts, src)
        elif type == Binning_Types.G2PROGRESSION:
            return G2ProgressionBinning(counts, domain)
        else:
            raise Exception("Invalid binning type.")

class BinningExtender():

    @staticmethod
    def mirror(binning):
        extended_bin_edges = [np.concatenate((
            [edges_along_dim[0] + edges_along_dim[0] - edges_along_dim[1]],
            edges_along_dim,
            [edges_along_dim[-1] +
             edges_along_dim[-1] - edges_along_dim[-2]]
        )) for edges_along_dim in binning.edges]
        return Binning(binning.type, extended_bin_edges,
                       binning.random_seed)

    @staticmethod
    def epsilon(binning):
        # Calculate avg. bin size along all dimensions
        epsilons = [np.mean(dists) for dists in binning.distances]
        extended_bin_edges = [np.concatenate((
            [edges_along_dim[0] - epsilon],
            edges_along_dim,
            [edges_along_dim[-1] + epsilon]
        )) for epsilon, edges_along_dim in zip(epsilons, binning.edges)]
        return Binning(binning.type, extended_bin_edges,
                       binning.random_seed)

    @classmethod
    def extend(cls, binning, pad_mode):
        if pad_mode == Pad_Modes.EPSILON:
            return cls.epsilon(binning)
        elif pad_mode == Pad_Modes.MIRROR:
            return cls.mirror(binning)
        else:
            raise Exception("Invalid pad_mode.")

max_stars_repo_name: dianagudu/inputgen

max_issues_repo_name: dianagudu/inputgen

max_forks_repo_name: dianagudu/inputgen


text:
from torch import nn
from scipy.stats import norm as dist_model

from few_shot.utils import pairwise_distances

#fit a gaussian model
def fit(prob_pos_X):
    prob_pos = [p for p in prob_pos_X]+[1-p for p in prob_pos_X]
    pos_mu, pos_std = dist_model.fit(prob_pos)
    return pos_mu, pos_std

#calculate mu, std of each seen class
def get_class_fit(support, prototypes, k_way, n_shot, distance_metric):
    sigmoid = nn.Sigmoid()
    support_distances = pairwise_distances(support, prototypes, distance_metric)
    support_prob = sigmoid(-support_distances)
    support_prob = support_prob.reshape(k_way, n_shot, -1)
    #print(support_prob.reshape(k_way, n_shot, -1).mean(dim=1))
    support_prob = support_prob.cpu().detach().numpy()
    mu_stds = []
    for i in range(k_way):
        pos_mu, pos_std = fit(support_prob[i,:, i])
        mu_stds.append([pos_mu, pos_std])

    return mu_stds

reason: from scipy


text:
import money
import decimal
import operator

import pytest
import six
from hypothesis.strategies import integers, lists, tuples
from hypothesis import given, example

import numpy as np
import numpy.testing as npt
import pandas as pd
import moneypandas as mpd
import pandas.util.testing as tm

def test_make_container():
    values = mpd.MoneyArray([1, 2, 3], 'GBP')
    npt.assert_array_equal(
        values.data,
        np.array([(1, 'GBP'),
                  (2, 'GBP'),
                  (3, 'GBP')], dtype=values.dtype._record_type)
    )

def test_repr_works():
    values = mpd.MoneyArray([0, 1, 2, 3], 'GBP')
    result = repr(values)
    expected = ("<MoneyArray[GBP]>\n[GBP 0.00, GBP 1.00, GBP 2.00, GBP 3.00]\nLength: 4, dtype: money")
    assert result == expected

def test_isna():
    v = mpd.MoneyArray([None, 2], 'GBP')
    r1 = v.isna()
    r2 = pd.isna(v)
    expected = np.array([True, False])

    np.testing.assert_array_equal(r1, expected)
    np.testing.assert_array_equal(r2, expected)

def test_array():
    v = mpd.MoneyArray([1, 2, 3], 'GBP')
    result = np.array(v)
    expected = np.array([
        money.XMoney(1, 'GBP'),
        money.XMoney(2, 'GBP'),
        money.XMoney(3, 'GBP'),
    ])
    tm.assert_numpy_array_equal(result, expected)

def test_tolist():
    v = mpd.MoneyArray([1, 2, 3], 'USD')
    result = v.tolist()
    expected = [(1, 'USD'), (2, 'USD'), (3, 'USD')]
    assert result == expected

def test_to_pymoney():
    v = mpd.MoneyArray([1, 2, 3], 'USD')
    result = v.to_pymoney()
    expected = [
        money.XMoney(1, 'USD'),
        money.XMoney(2, 'USD'),
        money.XMoney(3, 'USD'),
    ]
    assert result == expected

def test_equality():
    v1 = mpd.to_money([
        u'123 EUR',
        u'345 GBP',
    ])
    assert np.all(v1 == v1)
    assert v1.equals(v1)

    v2 = mpd.to_money([
        u'124 EUR',
        u'345 GBP',
    ])
    result = v1 == v2
    expected = np.array([False, True])
    tm.assert_numpy_array_equal(result, expected)

    result = bool(v1.equals(v2))
    assert result is False

    with pytest.raises(TypeError):
        v1.equals("a")

@pytest.mark.parametrize('op', [
    operator.lt,
    operator.le,
    operator.ge,
    operator.gt,
])
@pytest.mark.skipif(six.PY2, reason="Flexible comparisons")
def test_comparison_raises(op):
    arr = mpd.MoneyArray([0, 1, 2], 'JPY')
    with pytest.raises(TypeError):
        op(arr, 'a')

    with pytest.raises(TypeError):
        op('a', arr)

@given(
    tuples(
        lists(integers(min_value=0, max_value=99999)),
        lists(integers(min_value=0, max_value=99999))
    ).filter(lambda x: len(x[0]) == len(x[1]))
)
@example((1, 1))
@example((0, 0))
@example((0, 1))
@example((1, 0))
@example((1, 2))
@example((2, 1))
@pytest.mark.skip(reason="Flaky")
def test_ops(tup):
    a, b = tup
    v1 = mpd.MoneyArray(a, 'GBP')
    v2 = mpd.MoneyArray(b, 'GBP')

    r1 = v1 <= v2
    r2 = v2 >= v1
    tm.assert_numpy_array_equal(r1, r2)

@pytest.mark.xfail(reason='upstream')
def test_value_counts():
    x = mpd.MoneyArray([0, 0, 1], 'USD')
    result = x.value_counts()
    assert len(result)

def test_iter_works():
    x = mpd.MoneyArray([0, 1, 2], 'GBP')
    result = list(x)
    expected = [
        money.XMoney(0, 'GBP'),
        money.XMoney(1, 'GBP'),
        money.XMoney(2, 'GBP'),
    ]
    assert result == expected

def test_todecimal():
    values = [0, 1, 2]
    arr = mpd.MoneyArray(values, 'EUR')
    result = arr.to_decimals()
    assert all([r == decimal.Decimal(v) for r, v in zip(result, values)])

def test_getitem_scalar():
    ser = mpd.MoneyArray([0, 1, 2], 'USD')
    result = ser[1]
    assert result == money.XMoney(1, 'USD')

def test_getitem_slice():
    ser = mpd.MoneyArray([0, 1, 2], 'USD')
    result = ser[1:]
    expected = mpd.MoneyArray([1, 2], 'USD')
    assert result.equals(expected)

@pytest.mark.parametrize('value', [
    u'123 USD',
    123,
    money.XMoney(123, 'USD'),
])
def test_setitem_scalar(value):
    ser = mpd.MoneyArray([0, 1, 2], 'USD')
    ser[1] = value
    expected = mpd.MoneyArray([0, 123, 2], 'USD')
    assert ser.equals(expected)

def test_setitem_array():
    ser = mpd.MoneyArray([0, 1, 2], 'USD')
    ser[[1, 2]] = ['10 USD', '20 USD']
    expected = mpd.MoneyArray([0, 10, 20], 'USD')
    assert ser.equals(expected)

def test_bytes_roundtrip():
    arr = mpd.MoneyArray([1, 2, 3], 'USD')
    bytestring = arr.to_bytes()
    assert isinstance(bytestring, bytes)

    result = mpd.MoneyArray.from_bytes(bytestring)
    assert result.equals(arr)

def test_unique():
    arr = mpd.MoneyArray([3, 3, 1, 2, 3], 'USD')
    result = arr.unique()
    assert isinstance(result, mpd.MoneyArray)

    result = result.astype(object)
    expected = pd.unique(arr.astype(object))
    tm.assert_numpy_array_equal(result, expected)

def test_factorize():
    arr = mpd.MoneyArray([3, 3, 1, 2, 3], 'USD')
    labels, uniques = arr.factorize()
    expected_labels, expected_uniques = pd.factorize(arr.astype(object))

    assert isinstance(uniques, mpd.MoneyArray)

    uniques = uniques.astype(object)
    tm.assert_numpy_array_equal(labels, expected_labels)
    tm.assert_numpy_array_equal(uniques, expected_uniques)

@pytest.mark.parametrize('values', [
    [0, 1, 2],
])
def test_from_ndarray(values):
    result = mpd.MoneyArray(np.asarray(values), 'USD')
    expected = mpd.MoneyArray(values, 'USD')
    assert result.equals(expected)

max_stars_repo_name: albcunha/moneypandas

max_issues_repo_name: albcunha/moneypandas

max_forks_repo_name: albcunha/moneypandas


text:
"""Fixtures for testing."""

import pytest
import numpy as np

SEED = 10

@pytest.fixture
def random_X_data():
    return {
        "bin": create_random_binary_X(),
        "cont": create_random_continuous_X()}

def create_random_X(data_fixture):
    return np.concatenate([
        data_fixture["bin"], data_fixture["cont"],
        data_fixture["bin"], data_fixture["cont"]], axis=1)

def create_random_binary_X():
    np.random.seed(SEED)
    return np.random.randint(0, 2, (10, 3))

def create_random_continuous_X():
    np.random.seed(SEED)
    return np.random.randint(0, 100, (10, 3))

def create_linear_X():
    return np.array([range(10), range(11, 21)]).T

def create_y():
    return np.array([0, 0, 0, 0, 0, 1, 1, 1, 1, 1])

def create_s():
    return np.array([1, 1, 1, 1, 1, 0, 0, 0, 0, 0])

max_stars_repo_name: KevinKGWong/themis-ml

max_issues_repo_name: KevinKGWong/themis-ml

max_forks_repo_name: KevinKGWong/themis-ml


text:
import cv2
import numpy as np
from matplotlib import pyplot as plt

img = cv2.imread('sud2.jpeg',0)
img = cv2.medianBlur(img,5)

ret,th1 = cv2.threshold(img,127,255,cv2.THRESH_BINARY)
th2 = cv2.adaptiveThreshold(img,255,cv2.ADAPTIVE_THRESH_MEAN_C,\
            cv2.THRESH_BINARY,11,2)
th3 = cv2.adaptiveThreshold(img,255,cv2.ADAPTIVE_THRESH_GAUSSIAN_C,\
            cv2.THRESH_BINARY,11,2)

titles = ['Original Image', 'Global Thresholding (v = 127)',
            'Adaptive Mean Thresholding', 'Adaptive Gaussian Thresholding']
images = [img, th1, th2, th3]

for i in range(4):
    plt.subplot(2,2,i+1),plt.imshow(images[i],'gray')
    plt.title(titles[i])
    plt.xticks([]),plt.yticks([])
plt.show()


text:
from glue.core.fitters import BaseFitter1D
from glue.core.simpleforms import IntOption
from glue.config import fit_plugin

import numpy as np
import emcee

def gaussian(x, mean, amplitude, stddev):
    return np.exp(-(x - mean) ** 2 / (2 * stddev ** 2)) * amplitude

def lnprob(param, x, y, dy):
    # mean, amplitude, stddev = param
    if param[-1] < 0:
        return -np.inf
    yp = gaussian(x, *param)
    diff = (y - yp)
    if dy is not None:
        diff /= dy
    return -np.dot(diff, diff)

@fit_plugin
class EmceeGaussianFitter(BaseFitter1D):
    label = "Emcee Gaussian"
    walkers = IntOption(label="Walkers", min=1, max=200, default=50)
    burnin = IntOption(label="Burn in steps", min=1, max=10000, default=500)
    steps = IntOption(label="Steps", min=10, max=10000, default=500)

    def fit(self, x, y, dy, constraints,
            walkers=50, burnin=500, steps=500):
        ndim = 3
        # initialize walker parameters
        amp = y.max()
        mean = (x * y).sum() / y.sum()
        var = ((x - mean) ** 2 * y).sum() / y.sum()
        p0 = np.array([amp, mean, np.sqrt(var)]).reshape(1, -1)
        p0 = np.random.lognormal(sigma=.1, size=(walkers, ndim)) * p0
        sampler = emcee.EnsembleSampler(walkers, ndim, lnprob,
                                        args=[x, y, dy])

        # burnin
        pos, prob, state = sampler.run_mcmc(p0, burnin)
        sampler.reset()

        # run
        sampler.run_mcmc(pos, steps)
        return sampler

    def predict(self, fit_result, x):
        chain = fit_result.flatchain
        params = np.mean(chain, axis=0)
        return gaussian(x, *params)

    def summarize(self, fit_result, x, y, dy):
        af = fit_result.acceptance_fraction.mean()
        chain = fit_result.flatchain
        amp, mean, sigma = chain.mean(axis=0)
        damp, dmean, dsigma = np.std(chain, axis=0)
        walkers, steps, dim = fit_result.chain.shape

        result = [
            "Walkers: %i" % walkers,
            "Steps:   %i" % steps,
            "Acceptance fraction: %0.2f" % af,
            "-------------------------",
            "amplitude = %0.3e +/- %0.1e" % (amp, damp),
            "mean      = %0.3e +/- %0.1e" % (mean, dmean),
            "stddev    = %0.3e +/- %0.1e" % (sigma, dsigma)
        ]
        return '\n'.join(result)

    def plot(self, fit_result, axes, x):
        chain = fit_result.flatchain
        result = []

        # background samples
        for i in range(100):
            row = np.random.randint(0, chain.shape[0])
            params = chain[row]
            y = gaussian(x, *params)
            result.extend(axes.plot(x, y, 'k', alpha=.08))

        # foreground prediction of posterior mean model
        result.extend(
            super(EmceeGaussianFitter, self).plot(fit_result, axes, x))

        return result

max_stars_repo_name: HPLegion/glue

max_issues_repo_name: HPLegion/glue

max_forks_repo_name: HPLegion/glue