text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Apr  7 17:57:20 2021

@author: Jesimar da Silva Arantes
@email: jesimar.arantes@gmail.com

Código para fazer a Análise Explatória dos Dados (EDA) para conhecer melhor o problema.

"""

import os
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from PIL import Image
from scipy.stats import pearsonr
from scipy.stats import spearmanr
from scipy.stats import kurtosis
from scipy.stats import skew

def get_class_instance(filename):
    df = pd.read_csv(os.path.join('..', 'data', 'instances', filename))
    class_instance = df.iloc[0][800002]
    return class_instance

def experiment_analyze_problem_classes():
    classes = []
    for i in range(1, 168):
        vclass = get_class_instance('instance-{}.csv'.format(i))
        classes.append(vclass)
        print('instance = {} -> class = {}'.format(i, vclass))
    print(classes)

def simplification_of_instances(filename, n_line_init=10000):
    #cria arquivos com cerca de 160 KB facilitando a manipulação
    df = pd.read_csv(os.path.join('..', 'data', 'instances', filename), header=None)
    df_save = pd.DataFrame()
    for i in range(0, 3):
        df_init = df.iloc[i][:n_line_init]
        df_final = df.iloc[i][800001:800003] #foi removido também o signal_id (800000), pois ele não faz sentido no problema
        df_simple = df_init.append(df_final)
        df_save = df_save.append(df_simple)
    df_save.to_csv(os.path.join('..', 'data', 'instances-simplificadas', filename),
                   index=False, header=False, index_label=False)

def experiment_simplification_of_instances():
    # instâncias sem problemas
    simplification_of_instances('instance-1.csv')
    simplification_of_instances('instance-3.csv')
    simplification_of_instances('instance-4.csv')
    simplification_of_instances('instance-5.csv')
    simplification_of_instances('instance-6.csv')

    # instâncias com problemas
    simplification_of_instances('instance-2.csv')
    simplification_of_instances('instance-68.csv')
    simplification_of_instances('instance-77.csv')
    simplification_of_instances('instance-91.csv')
    simplification_of_instances('instance-94.csv')

def simplification_of_instances_moving_average(filename, size_ma=1000):
    df = pd.read_csv(os.path.join('..', 'data', 'instances', filename), header=None)
    df_avg = [[0], [0], [0]]
    for i in range(0, 3):
        df_avg[i] = df.iloc[i].rolling(window=size_ma).mean().values[size_ma::size_ma]
    df_save = pd.DataFrame(df_avg)
    df_save.to_csv(os.path.join('..', 'data', 'instances-moving-average-{}'.format(size_ma), filename),
                   index=False, header=False, index_label=False)

def experiment_simplification_of_instances_moving_average():
    #cria arquivos com aproximadamente 17 KB facilitando a manipulação.
    # instâncias sem problemas
    simplification_of_instances_moving_average('instance-1.csv')
    simplification_of_instances_moving_average('instance-3.csv')
    simplification_of_instances_moving_average('instance-4.csv')
    simplification_of_instances_moving_average('instance-5.csv')
    simplification_of_instances_moving_average('instance-6.csv')

    # instâncias com problemas
    simplification_of_instances_moving_average('instance-2.csv')
    simplification_of_instances_moving_average('instance-68.csv')
    simplification_of_instances_moving_average('instance-77.csv')
    simplification_of_instances_moving_average('instance-91.csv')
    simplification_of_instances_moving_average('instance-94.csv')

def creating_image_by_timeseries(path, filename):
    df = pd.read_csv(os.path.join(path, filename), header=None)
    red   = df.values[0][:-2]
    green = df.values[1][:-2]
    blue  = df.values[2][:-2]
    max_r = 40  #300  #max(red)
    min_r = -40 #-150 #min(red)
    max_g = 40  #300  #max(green)
    min_g = -40 #-150 #min(green)
    max_b = 40  #300  #max(blue)
    min_b = -40 #-150 #min(blue)
    red   = 255 * (red - min_r)/(max_r - min_r)
    green = 255 * (green - min_g)/(max_g - min_g)
    blue  = 255 * (blue - min_b)/(max_b - min_b)
    dimx = 894
    dimy = 894
    pixels = np.zeros((dimx, dimx, 3), dtype=np.uint8)
    v = 0
    for i in range(dimx):
        for j in range(dimy):
            red[v] = max(min(red[v], 255), 0)
            green[v] = max(min(green[v], 255), 0)
            blue[v] = max(min(blue[v], 255), 0)
            if red[v] == 255 or green[v] == 255 or blue[v] == 255:
                pixels[i][j] = [255, 255, 255]
                #pixels[i][j] = [0, 0, 0]
            elif red[v] == 0 or green[v] == 0 or blue[v] == 0:
                pixels[i][j] = [255, 255, 255]
                #pixels[i][j] = [0, 0, 0]
            else:
                pixels[i][j] = [red[v], green[v], blue[v]]
            v += 1
            if v > 800000:
                break
        if v > 800000:
            break
    # Converte os pixels em um array usando numpy
    array = np.array(pixels, dtype=np.uint8)
    # Use PIL para criar uma imagem do novo array de pixel
    new_image = Image.fromarray(array)
    new_image.save('image-{}.png'.format(filename.replace('.csv', '')))

def experiment_create_images_by_timeseries():
    path = os.path.join('..', 'data', 'instances')
    # instâncias sem problemas
    creating_image_by_timeseries(path, 'instance-1.csv')
    creating_image_by_timeseries(path, 'instance-3.csv')
    creating_image_by_timeseries(path, 'instance-4.csv')
    creating_image_by_timeseries(path, 'instance-5.csv')
    creating_image_by_timeseries(path, 'instance-6.csv')
    creating_image_by_timeseries(path, 'instance-7.csv')
    creating_image_by_timeseries(path, 'instance-8.csv')
    creating_image_by_timeseries(path, 'instance-9.csv')
    creating_image_by_timeseries(path, 'instance-10.csv')
    creating_image_by_timeseries(path, 'instance-11.csv')
    creating_image_by_timeseries(path, 'instance-12.csv')

    # instâncias com problemas
    creating_image_by_timeseries(path, 'instance-2.csv')
    creating_image_by_timeseries(path, 'instance-68.csv')
    creating_image_by_timeseries(path, 'instance-77.csv')
    creating_image_by_timeseries(path, 'instance-91.csv')
    creating_image_by_timeseries(path, 'instance-94.csv')
    creating_image_by_timeseries(path, 'instance-96.csv')
    creating_image_by_timeseries(path, 'instance-99.csv')
    creating_image_by_timeseries(path, 'instance-137.csv')
    creating_image_by_timeseries(path, 'instance-145.csv')
    creating_image_by_timeseries(path, 'instance-146.csv')
    creating_image_by_timeseries(path, 'instance-153.csv')

def plot_timeseries_from_file(path, filename):
    df = pd.read_csv(os.path.join(path, filename), header=None)
    phase0 = df.iloc[0]
    phase1 = df.iloc[1]
    phase2 = df.iloc[2]
    plt.plot(phase0)
    plt.plot(phase1)
    plt.plot(phase2)

def experiment_plot_timeseries():
    #path = os.path.join('..', 'data', 'instances-moving-average-8000')
    #plot_timeseries_from_file(path, 'instance-1.csv')

    path = os.path.join('..', 'data', 'instances-moving-average-1000')
    # instâncias sem problemas
    plot_timeseries_from_file(path, 'instance-1.csv')
    #plot_timeseries_from_file(path, 'instance-3.csv')
    #plot_timeseries_from_file(path, 'instance-4.csv')
    #plot_timeseries_from_file(path, 'instance-5.csv')
    #plot_timeseries_from_file(path, 'instance-6.csv')

    # instâncias com problemas
    #plot_timeseries_from_file(path, 'instance-2.csv')
    #plot_timeseries_from_file(path, 'instance-68.csv')
    #plot_timeseries_from_file(path, 'instance-77.csv')
    #plot_timeseries_from_file(path, 'instance-91.csv')
    #plot_timeseries_from_file(path, 'instance-94.csv')

    #path = os.path.join('..', 'data', 'instances')
    # instâncias sem problemas
    #plot_timeseries_from_file(path, 'instance-1.csv')
    #plot_timeseries_from_file(path, 'instance-3.csv')
    #plot_timeseries_from_file(path, 'instance-4.csv')
    #plot_timeseries_from_file(path, 'instance-5.csv')
    #plot_timeseries_from_file(path, 'instance-6.csv')
    # instâncias com problemas
    #plot_timeseries_from_file(path, 'instance-2.csv')
    #plot_timeseries_from_file(path, 'instance-68.csv')
    #plot_timeseries_from_file(path, 'instance-77.csv')
    #plot_timeseries_from_file(path, 'instance-91.csv')
    #plot_timeseries_from_file(path, 'instance-94.csv')

#Relações lineares
#A correlação de Pearson, assume que os dados sejam normalmente distribuídos.
def correlation_pearson_r(data1, data2):
    corr, pvalue = pearsonr(data1, data2)
    alpha = 0.05
    print("Teste Correlação de Pearson R:")
    print("    coeficiente da correlação R: {:.4f}".format(corr))
    print("    p-value reference (alpha): {:.4f}".format(alpha))
    print("    p-value: {:.4f}".format(pvalue))
    if pvalue <= alpha:
        print("    Interpretação: Baixa probabilidade da correlação ser ao acaso")
    else:
        print("    Interpretação: Alta probabilidade da correlação ser ao acaso")
    if 0.9 <= abs(corr) <= 1.0:
        print("    Interpretação: Correlação muito forte")
    elif 0.7 <= abs(corr) < 0.9:
        print("    Interpretação: Correlação forte")
    elif 0.4 <= abs(corr) < 0.7:
        print("    Interpretação: Correlação moderada")
    elif 0.2 <= abs(corr) < 0.4:
        print("    Interpretação: Correlação fraca")
    if 0.0 <= abs(corr) < 0.2:
        print("    Interpretação: Correlação muito fraca")

#Relações não lineares monotonas
#Diferentemente da correlação de Pearson, a correlação de Spearman não
#assume que os dois conjuntos de dados sejam normalmente distribuídos.
def correlation_spearman_rho(data1, data2):
    corr, pvalue = spearmanr(data1, data2)
    alpha = 0.05
    print("Teste Correlação de Spearman R:")
    print("    coeficiente da correlação R: {:.4f}".format(corr))
    print("    p-value reference (alpha): {:.4f}".format(alpha))
    print("    p-value: {:.4f}".format(pvalue))
    if pvalue <= alpha:
        print("    Interpretação: Baixa probabilidade da correlação ser ao acaso")
    else:
        print("    Interpretação: Alta probabilidade da correlação ser ao acaso")
    if 0.9 <= abs(corr) <= 1.0:
        print("    Interpretação: Correlação muito forte")
    elif 0.7 <= abs(corr) < 0.9:
        print("    Interpretação: Correlação forte")
    elif 0.4 <= abs(corr) < 0.7:
        print("    Interpretação: Correlação moderada")
    elif 0.2 <= abs(corr) < 0.4:
        print("    Interpretação: Correlação fraca")
    if 0.0 <= abs(corr) < 0.2:
        print("    Interpretação: Correlação muito fraca")

def analyze_correlation_between_timeseries(path, filename):
    df = pd.read_csv(os.path.join(path, filename), header=None)
    phase_0 = df.values[0][:-2]
    phase_1 = df.values[1][:-2]
    phase_2 = df.values[2][:-2]
    correlation_pearson_r(phase_0, phase_1)
    correlation_pearson_r(phase_1, phase_2)
    correlation_pearson_r(phase_2, phase_0)
    correlation_spearman_rho(phase_0, phase_1)
    correlation_spearman_rho(phase_1, phase_2)
    correlation_spearman_rho(phase_2, phase_0)

def analyze_correlation_between_timeseries_shift(path, filename, shift=266667):
    df = pd.read_csv(os.path.join(path, filename), header=None)
    phase_0 = df.values[0][:-2]
    phase_1 = df.values[1][:-2]
    phase_2 = df.values[2][:-2]
    nphase_1 = []
    for i in range(len(phase_1)):
        if i+shift < len(phase_1):
            nphase_1.append(phase_1[i + shift])
        else:
            nphase_1.append(phase_1[i - (len(phase_1) - shift)])
    nphase_2 = []
    for i in range(len(phase_2)):
        if i+2*shift < len(phase_2):
            nphase_2.append(phase_2[i + 2*shift])
        else:
            nphase_2.append(phase_2[i - (len(phase_2) - 2*shift)])
    correlation_pearson_r(phase_0, nphase_1)
    correlation_pearson_r(nphase_1, nphase_2)
    correlation_pearson_r(nphase_2, phase_0)
    correlation_spearman_rho(phase_0, nphase_1)
    correlation_spearman_rho(nphase_1, nphase_2)
    correlation_spearman_rho(nphase_2, phase_0)

def experiment_analyze_correlation():
    path = os.path.join('..', 'data', 'instances')
    # instâncias sem problemas
    analyze_correlation_between_timeseries(path, 'instance-1.csv')
    analyze_correlation_between_timeseries(path, 'instance-3.csv')
    analyze_correlation_between_timeseries(path, 'instance-4.csv')
    analyze_correlation_between_timeseries(path, 'instance-5.csv')
    analyze_correlation_between_timeseries(path, 'instance-6.csv')

    # instâncias com problemas
    analyze_correlation_between_timeseries(path, 'instance-2.csv')
    analyze_correlation_between_timeseries(path, 'instance-68.csv')
    analyze_correlation_between_timeseries(path, 'instance-77.csv')
    analyze_correlation_between_timeseries(path, 'instance-91.csv')
    analyze_correlation_between_timeseries(path, 'instance-94.csv')

    # instâncias sem problemas
    analyze_correlation_between_timeseries_shift(path, 'instance-1.csv')
    analyze_correlation_between_timeseries_shift(path, 'instance-3.csv')
    analyze_correlation_between_timeseries_shift(path, 'instance-4.csv')
    analyze_correlation_between_timeseries_shift(path, 'instance-5.csv')
    analyze_correlation_between_timeseries_shift(path, 'instance-6.csv')
    # instâncias com problemas
    analyze_correlation_between_timeseries_shift(path, 'instance-2.csv')
    analyze_correlation_between_timeseries_shift(path, 'instance-68.csv')
    analyze_correlation_between_timeseries_shift(path, 'instance-77.csv')
    analyze_correlation_between_timeseries_shift(path, 'instance-91.csv')
    analyze_correlation_between_timeseries_shift(path, 'instance-94.csv')

def stats_extraction_of_timeseries(path, filename):
    df = pd.read_csv(os.path.join(path, filename), header=None)
    phase_0 = df.values[0][:-2]
    phase_1 = df.values[1][:-2]
    phase_2 = df.values[2][:-2]
    mean_0 = np.mean(phase_0)
    mean_1 = np.mean(phase_1)
    mean_2 = np.mean(phase_2)
    median_0 = np.median(phase_0)
    median_1 = np.median(phase_1)
    median_2 = np.median(phase_2)
    rms_0 = np.sqrt(np.mean(phase_0**2))
    rms_1 = np.sqrt(np.mean(phase_1**2))
    rms_2 = np.sqrt(np.mean(phase_2**2))
    std_0 = np.std(phase_0, ddof=0)
    std_1 = np.std(phase_1, ddof=0)
    std_2 = np.std(phase_2, ddof=0)
    kurtosis_0 = kurtosis(phase_0)
    kurtosis_1 = kurtosis(phase_1)
    kurtosis_2 = kurtosis(phase_2)
    skew_0 = skew(phase_0)
    skew_1 = skew(phase_1)
    skew_2 = skew(phase_2)
    min_0 = np.min(phase_0)
    min_1 = np.min(phase_1)
    min_2 = np.min(phase_2)
    max_0 = np.max(phase_0)
    max_1 = np.max(phase_1)
    max_2 = np.max(phase_2)
    print('mean0; {}'.format(mean_0))
    print('mean1; {}'.format(mean_1))
    print('mean2; {}'.format(mean_2))
    print('median0; {}'.format(median_0))
    print('median1; {}'.format(median_1))
    print('median2; {}'.format(median_2))
    print('rms0; {}'.format(rms_0))
    print('rms1; {}'.format(rms_1))
    print('rms2; {}'.format(rms_2))
    print('std0; {}'.format(std_0))
    print('std1; {}'.format(std_1))
    print('std2; {}'.format(std_2))
    print('kurtosis0; {}'.format(kurtosis_0))
    print('kurtosis1; {}'.format(kurtosis_1))
    print('kurtosis2; {}'.format(kurtosis_2))
    print('skew0; {}'.format(skew_0))
    print('skew1; {}'.format(skew_1))
    print('skew2; {}'.format(skew_2))
    print('min0; {}'.format(min_0))
    print('min1; {}'.format(min_1))
    print('min2; {}'.format(min_2))
    print('max0; {}'.format(max_0))
    print('max1; {}'.format(max_1))
    print('max2; {}'.format(max_2))

def experiment_stats_extraction_of_timeseries():
    path = os.path.join('..', 'data', 'instances')
    # instâncias sem problemas
    stats_extraction_of_timeseries(path, 'instance-1.csv')
    stats_extraction_of_timeseries(path, 'instance-3.csv')
    stats_extraction_of_timeseries(path, 'instance-4.csv')
    stats_extraction_of_timeseries(path, 'instance-5.csv')
    stats_extraction_of_timeseries(path, 'instance-6.csv')

    # instâncias com problemas
    stats_extraction_of_timeseries(path, 'instance-2.csv')
    stats_extraction_of_timeseries(path, 'instance-68.csv')
    stats_extraction_of_timeseries(path, 'instance-77.csv')
    stats_extraction_of_timeseries(path, 'instance-91.csv')
    stats_extraction_of_timeseries(path, 'instance-94.csv')

def print_hearder():
    formatter_header = '{};' * 38 + '{}'
    print(formatter_header.format('instance', 'mean_sum', 'mean_sum_abs', 'mean_sum2',
                                  'rms_sum', 'rms_sum_abs', 'rms_sum2',
                                  'std_sum', 'std_sum_abs', 'std_sum2',
                                  'kurtosis_sum', 'kurtosis_sum_abs', 'kurtosis_sum2',
                                  'skew_sum', 'skew_sum_abs', 'skew_sum2',
                                  'min_sum', 'min_sum2', 'max_sum2',
                                  'intensity', 'cont', 'intensity_quad', 'cont_quad',
                                  'rms_phase0', 'rms_phase1', 'rms_phase2',
                                  'std_phase0', 'std_phase1', 'std_phase2',
                                  'kurtosis_phase0', 'kurtosis_phase1', 'kurtosis_phase2',
                                  'skew_phase0', 'skew_phase1', 'skew_phase2',
                                  'corr_phase0_phase1', 'corr_phase1_phase2', 'corr_phase2_phase0',
                                  'target'
                                  ))

def features_extraction_of_data(path, filename):
    df = pd.read_csv(os.path.join(path, filename), header=None)
    phase_0 = df.values[0][:-2]
    phase_1 = df.values[1][:-2]
    phase_2 = df.values[2][:-2]

    vsum = phase_0 + phase_1 + phase_2
    vsum_abs = abs(vsum)
    vsum2 = phase_0 * phase_0 + phase_1 * phase_1 + phase_2 * phase_2

    mean_sum = np.mean(vsum)
    mean_sum_abs = np.mean(vsum_abs)
    mean_sum2 = np.mean(vsum2)
    rms_sum = np.sqrt(np.mean(vsum**2))
    rms_sum_abs = np.sqrt(np.mean(vsum_abs**2))
    rms_sum2 = np.sqrt(np.mean(vsum2**2))
    std_sum = np.std(vsum, ddof=0)
    std_sum_abs = np.std(vsum_abs, ddof=0)
    std_sum2 = np.std(vsum2, ddof=0)
    kurtosis_sum = kurtosis(vsum)
    kurtosis_sum_abs = kurtosis(vsum_abs)
    kurtosis_sum2 = kurtosis(vsum2)
    skew_sum = skew(vsum)
    skew_sum_abs = skew(vsum_abs)
    skew_sum2 = skew(vsum2)
    min_sum = np.min(vsum)
    min_sum2 = np.min(vsum2)
    max_sum2 = np.max(vsum2)

    cont = 0.0
    intensity = 0.0
    cont_quad = 0.0
    intensity_quad = 0.0
    thresold = 15
    thresold_quad = 800
    for i in range(len(vsum)):
        if abs(vsum[i]) > thresold:
            cont += 1
            intensity += abs(vsum[i]) - thresold
    for i in range(len(vsum2)):
        if abs(vsum2[i]) > thresold_quad:
            cont_quad += 1
            intensity_quad += abs(vsum2[i]) - thresold_quad
    intensity = intensity / len(vsum)
    cont = (100.0 * cont) / len(vsum)
    intensity_quad = intensity_quad / len(vsum2)
    cont_quad = (100.0 * cont_quad) / len(vsum2)

    rms_phase0 = np.sqrt(np.mean(phase_0**2))
    rms_phase1 = np.sqrt(np.mean(phase_1**2))
    rms_phase2 = np.sqrt(np.mean(phase_2**2))

    std_phase0 = np.std(phase_0, ddof=0)
    std_phase1 = np.std(phase_1, ddof=0)
    std_phase2 = np.std(phase_2, ddof=0)

    kurtosis_phase0 = kurtosis(phase_0)
    kurtosis_phase1 = kurtosis(phase_1)
    kurtosis_phase2 = kurtosis(phase_2)

    skew_phase0 = skew(phase_0)
    skew_phase1 = skew(phase_1)
    skew_phase2 = skew(phase_2)

    shift = 266667  # 800000/3
    nphase_1 = []
    nphase_2 = []
    for i in range(len(phase_1)):
        if i+shift < len(phase_1):
            nphase_1.append(phase_1[i + shift])
        else:
            nphase_1.append(phase_1[i - (len(phase_1) - shift)])
        if i+2*shift < len(phase_2):
            nphase_2.append(phase_2[i + 2*shift])
        else:
            nphase_2.append(phase_2[i - (len(phase_2) - 2*shift)])

    corr_phase0_phase1, _ = pearsonr(phase_0, nphase_1)
    corr_phase1_phase2, _ = pearsonr(nphase_1, nphase_2)
    corr_phase2_phase0, _ = pearsonr(nphase_2, phase_0)
    target = df.iloc[0][800002]

    df_sum = pd.DataFrame(vsum)
    df_sum_abs = pd.DataFrame(vsum_abs)
    df_sum.to_csv(os.path.join('..', 'data', 'instances-features', 'soma_{}'.format(filename)),
                  index=False, header=False, index_label=False)
    df_sum_abs.to_csv(os.path.join('..', 'data', 'instances-features', 'soma_abs_{}'.format(filename)),
                      index=False, header=False, index_label=False)

    header = '{};' * 38 + '{}'
    print(header.format(filename, mean_sum, mean_sum_abs, mean_sum2,
                        rms_sum, rms_sum_abs, rms_sum2,
                        std_sum, std_sum_abs, std_sum2,
                        kurtosis_sum, kurtosis_sum_abs, kurtosis_sum2,
                        skew_sum, skew_sum_abs, skew_sum2,
                        min_sum, min_sum2, max_sum2, intensity, cont,
                        intensity_quad, cont_quad,
                        rms_phase0, rms_phase1, rms_phase2,
                        std_phase0, std_phase1, std_phase2,
                        kurtosis_phase0, kurtosis_phase1, kurtosis_phase2,
                        skew_phase0, skew_phase1, skew_phase2,
                        corr_phase0_phase1, corr_phase1_phase2, corr_phase2_phase0,
                        target
                        ))
    #print('instance; {}'.format(filename))
    #print('mean_sum; {}'.format(mean_sum))
    #print('mean_sum_abs; {}'.format(mean_sum_abs))
    #print('mean_sum_quad; {}'.format(mean_sum2))
    #print('rms_sum; {}'.format(rms_sum))
    #print('rms_sum_abs; {}'.format(rms_sum_abs))
    #print('rms_sum_quad; {}'.format(rms_sum2))
    #print('std_sum; {}'.format(std_sum))
    #print('std_sum_abs; {}'.format(std_sum_abs))
    #print('std_sum_quad; {}'.format(std_sum2))
    #print('kurtosis_sum; {}'.format(kurtosis_sum))
    #print('kurtosis_sum_abs; {}'.format(kurtosis_sum_abs))
    #print('kurtosis_sum_quad; {}'.format(kurtosis_sum2))
    #print('skew_sum; {}'.format(skew_sum))
    #print('skew_sum_abs; {}'.format(skew_sum_abs))
    #print('skew_sum_quad; {}'.format(skew_sum2))
    #print('min_sum; {}'.format(min_sum))
    #print('min_sum_quad; {}'.format(min_sum2))
    #print('max_sum_quad; {}'.format(max_sum2))
    #print('intensidade_sum; {}'.format(intensidade))
    #print('cont_sum; {}'.format(cont))
    #print('intensidade_sum_quad; {}'.format(intensidade_quad))
    #print('cont_sum_quad; {}'.format(cont_quad))
    #print('rms_phase0; {}'.format(rms_phase0))
    #print('rms_phase1; {}'.format(rms_phase1))
    #print('rms_phase2; {}'.format(rms_phase2))
    #print('std_phase0; {}'.format(std_phase0))
    #print('std_phase1; {}'.format(std_phase1))
    #print('std_phase2; {}'.format(std_phase2))
    #print('kurtosis_phase0; {}'.format(kurtosis_phase0))
    #print('kurtosis_phase1; {}'.format(kurtosis_phase1))
    #print('kurtosis_phase2; {}'.format(kurtosis_phase2))
    #print('skew_phase0; {}'.format(skew_phase0))
    #print('skew_phase1; {}'.format(skew_phase1))
    #print('skew_phase2; {}'.format(skew_phase2))
    #print('corr_phase0_phase1; {}'.format(corr_phase0_phase1))
    #print('corr_phase1_phase2; {}'.format(corr_phase1_phase2))
    #print('corr_phase2_phase0; {}'.format(corr_phase2_phase0))

def experiment_features_extraction_of_data():
    path = os.path.join('..', 'data', 'instances')
    print_hearder()
    # instâncias sem problemas
    features_extraction_of_data(path, 'instance-1.csv')
    features_extraction_of_data(path, 'instance-3.csv')
    features_extraction_of_data(path, 'instance-4.csv')
    features_extraction_of_data(path, 'instance-5.csv')
    features_extraction_of_data(path, 'instance-6.csv')

    # instâncias com problemas
    features_extraction_of_data(path, 'instance-2.csv')
    features_extraction_of_data(path, 'instance-68.csv')
    features_extraction_of_data(path, 'instance-77.csv')
    features_extraction_of_data(path, 'instance-91.csv')
    features_extraction_of_data(path, 'instance-94.csv')

def experiment_features_extraction_all_instances():
    path = os.path.join('..', 'data', 'instances')
    print_hearder()
    for i in range(1, 168):
        features_extraction_of_data(path, 'instance-{}.csv'.format(i))

def plot_one_timeseries_from_file(path, filename):
    df = pd.read_csv(os.path.join(path, filename), header=None)
    plt.plot(df.transpose().values[0])

def experiment_plot_feature_timeseries():
    path = os.path.join('..', 'data', 'instances-features')
    # instâncias sem problemas
    plot_one_timeseries_from_file(path, 'soma_instance-1.csv')
    plot_one_timeseries_from_file(path, 'soma_instance-3.csv')
    plot_one_timeseries_from_file(path, 'soma_instance-4.csv')
    plot_one_timeseries_from_file(path, 'soma_instance-5.csv')
    plot_one_timeseries_from_file(path, 'soma_instance-6.csv')

    # instâncias com problemas
    #plot_one_timeseries_from_file(path, 'soma_instance-2.csv')
    #plot_one_timeseries_from_file(path, 'soma_instance-68.csv')
    #plot_one_timeseries_from_file(path, 'soma_instance-77.csv')
    #plot_one_timeseries_from_file(path, 'soma_instance-91.csv')
    #plot_one_timeseries_from_file(path, 'soma_instance-94.csv')

    # instâncias sem problemas
    #plot_one_timeseries_from_file(path, 'soma_abs_instance-1.csv')
    #plot_one_timeseries_from_file(path, 'soma_abs_instance-3.csv')
    #plot_one_timeseries_from_file(path, 'soma_abs_instance-4.csv')
    #plot_one_timeseries_from_file(path, 'soma_abs_instance-5.csv')
    #plot_one_timeseries_from_file(path, 'soma_abs_instance-6.csv')

    # instâncias com problemas
    #plot_one_timeseries_from_file(path, 'soma_abs_instance-2.csv')
    #plot_one_timeseries_from_file(path, 'soma_abs_instance-68.csv')
    #plot_one_timeseries_from_file(path, 'soma_abs_instance-77.csv')
    #plot_one_timeseries_from_file(path, 'soma_abs_instance-91.csv')
    #plot_one_timeseries_from_file(path, 'soma_abs_instance-94.csv')

# ============= Experimentos da Análise Exploratória do Problema =============

#experiment_analyze_problem_classes()

#experiment_simplification_of_instances()

#experiment_simplification_of_instances_moving_average()

#experiment_create_images_by_timeseries()

experiment_plot_timeseries()

#experiment_analyze_correlation()

#experiment_stats_extraction_of_timeseries()

#experiment_features_extraction_of_data()

#experiment_features_extraction_all_instances()

#experiment_plot_feature_timeseries()

max_stars_repo_name: jesimar/desafio-senai

max_issues_repo_name: jesimar/desafio-senai

max_forks_repo_name: jesimar/desafio-senai


text:
import sys
sys.path.append("../")

import torchaudio
import librosa
import torch
import numpy as np
from scipy.stats import sem

from memory_profiler import memory_usage
from utils import get_whitenoise, get_spectrogram, update_results, memusage_kwargs

def main():
    results = {}
    repeat = 5
    number = 10

    sample_rate = 16000
    n_fft = 400
    win_length = n_fft
    hop_length = n_fft // 4
    window = torch.hann_window(win_length)
    power = 1
    duration = 20
    n_iter = 8
    momentum = 0.99

    waveform = get_whitenoise(sample_rate=sample_rate, duration=duration, n_channels=2)
    specgram = get_spectrogram(waveform, n_fft=n_fft, hop_length=hop_length, power=power,
                               win_length=win_length, window=window)
    specgram_np = specgram[0].numpy()
    length = waveform.size(1)

    for device in [torch.device('cpu'), torch.device('cuda:0')]:
        for dtype in [torch.float32, torch.float64]:
            for jitted in [False, True]:
                if jitted:
                    print(f"[torchaudio {device} {dtype} jitted]")
                else:
                    print(f"[torchaudio {device} {dtype}]")

                input = torch.clone(specgram).detach().to(device, dtype)
                window = torch.clone(window).detach().to(device, dtype)

                transform_fn = torchaudio.functional.griffinlim
                if jitted:
                    transform_fn = torch.jit.script(transform_fn)

                def run_fn(input, window, n_fft, hop_length, win_length, power, n_iter, momentum, length, number=1):
                    for _ in range(number):
                        transform_fn(input, window=window, n_fft=n_fft, hop_length=hop_length, win_length=win_length,
                                    power=power, n_iter=n_iter, momentum=momentum, length=length, rand_init=False)

                kwargs = {"input": input, "window": window, "n_fft": n_fft,
                          "hop_length": hop_length, "win_length": win_length, "power": power, "n_iter": n_iter,
                          "momentum": momentum, "length": length}
                usages = []
                for _ in range(repeat):
                    ret = memory_usage((run_fn, [], kwargs), **memusage_kwargs)
                    usages.append(np.max(ret) - ret[0])

                print(f"{np.mean(usages)} +- {sem(usages)}")
                results[("griffinlim", "torchaudio", str(device), str(dtype), int(jitted))] = (np.mean(usages), sem(usages))

    for dtype in [np.float32, np.float64]:
        print(f"[librosa cpu {dtype}]")

        input = specgram_np.astype(dtype, copy=True)

        def run_fn(input, hop_length, n_iter, momentum, length, number=1):
            for _ in range(number):
                librosa.griffinlim(input, n_iter=n_iter, hop_length=hop_length, momentum=momentum, init=None, length=length)

        kwargs = {"input": input, "n_iter": n_iter, "hop_length": hop_length,
                  "momentum": momentum, "length": length}
        usages = []
        for _ in range(repeat):
            ret = memory_usage((run_fn, [], kwargs), **memusage_kwargs)
            usages.append(np.max(ret) - ret[0])

        print(f"{np.mean(usages)} +- {sem(usages)}")
        results[("griffinlim", "librosa", "cpu", str(dtype), int(False))] = (np.mean(usages), sem(usages))

    print(results)
    update_results(results, "./results/results.pkl")

if __name__ == "__main__":
    main()

ext: py

max_stars_repo_name: yangarbiter/torchaudio-benchmark

max_issues_repo_name: yangarbiter/torchaudio-benchmark

max_forks_repo_name: yangarbiter/torchaudio-benchmark

reason: import numpy,from scipy


text:
import numpy as np
import tensorflow as tf

from power_system_functions import create_system_matrices
from read_runge_kutta_parameters import read_runge_kutta_parameters

class PinnModel(tf.keras.models.Model):
    """
    The central model that predicts the Runge-Kutta stages h_hat and combines them into a prediction x1_hat and
    returns the mismatch in the implicit non-linear Runge-Kutta formulas.
    """

    def __init__(self, neurons_in_hidden_layer, n_runge_kutta_stages, power_system, seed=12345, *args,
                 **kwargs):
        super().__init__(*args, **kwargs)
        tf.random.set_seed(seed)

        self.CoreNetwork = CoreNetwork(n_states=power_system['n_states'],
                                       n_runge_kutta_stages=n_runge_kutta_stages,
                                       neurons_in_hidden_layer=neurons_in_hidden_layer)

        self.n_states = power_system['n_states']
        self.n_generators = power_system['n_generators']
        self.n_buses = power_system['n_buses']
        self.epoch_count = 0
        self.seed = seed

        self.n_runge_kutta_stages = n_runge_kutta_stages

        self.loss_terms = self.n_states + self.n_states + self.n_states * self.n_runge_kutta_stages

        self.IRK_alpha, self.IRK_beta, self.IRK_gamma = read_runge_kutta_parameters(
            runge_kutta_stages=n_runge_kutta_stages)

        self.A, self.B, self.C, self.D, self.F, self.G, self.u_0, self.x_0, = create_system_matrices(
            power_system=power_system)

    def call(self, inputs, training=None, mask=None):
        # bring the inputs to 'CoreNetwork' in the right format and the other variables needed in the physics and RK
        # calculation
        x_time, x_power, x_y_0 = inputs
        delta_0 = x_y_0[:, 0:1]
        u_disturbance = x_power @ self.G.T
        u_physics = u_disturbance + self.u_0.T

        x_time_expanded = tf.expand_dims(x_time, axis=-1)
        x_y_0_expanded = tf.expand_dims(x_y_0, axis=-1)
        u = tf.expand_dims(u_disturbance, axis=1) + self.u_0.reshape((1, 1, -1))

        # call the 'CoreNetwork' to compute the runge_kutta_variables h_hat and then the state prediction x1_hat
        with tf.GradientTape(watch_accessed_variables=True,
                             persistent=True) as grad_t:
            grad_t.watch(x_time)
            runge_kutta_variables = self.CoreNetwork(inputs=[x_time, x_power, delta_0])

            runge_kutta_prediction = x_y_0 + x_time * tf.squeeze(
                runge_kutta_variables @ self.IRK_beta.T, axis=2)

        # calculate the derivative of x1_hat (d/dt x1_hat)
        runge_kutta_prediction_dt = tf.squeeze(grad_t.batch_jacobian(runge_kutta_prediction,
                                                                     x_time,
                                                                     unconnected_gradients='none'), axis=2)
        del grad_t

        runge_kutta_variables = self.CoreNetwork(inputs=[x_time, x_power, delta_0])
        FCX = tf.sin(runge_kutta_prediction @ self.C.T) @ self.D.T

        network_output_physics = runge_kutta_prediction_dt - (runge_kutta_prediction @ self.A.T +
                                                              FCX @
                                                              self.F.T + u_physics @ self.B.T)

        # calculate the approximate states h_hat and transpose it (_T) before applying the physics
        approximate_state = x_y_0_expanded + x_time_expanded * (runge_kutta_variables @ self.IRK_alpha.T)

        approximate_state_T = tf.transpose(approximate_state, perm=[0, 2, 1])

        # apply f() on the approximate state
        f_approximate_state = approximate_state_T @ self.A.T + \
                              tf.sin(approximate_state_T @ self.C.T) @ self.D.T @ self.F.T + \
                              u @ self.B.T

        f_approximate_state_T = tf.transpose(f_approximate_state, perm=[0, 2, 1])

        # evaluate the mismatch of the non-linear Runge-Kutta equations
        runge_kutta_stages_difference = runge_kutta_variables - f_approximate_state_T

        # prepare for loss value calculation
        runge_kutta_stages_difference_flattened = tf.reshape(runge_kutta_stages_difference,
                                                             shape=(-1, self.n_runge_kutta_stages * self.n_states))

        return_split = tf.split(runge_kutta_prediction, num_or_size_splits=self.n_states, axis=1) + \
                       tf.split(network_output_physics, num_or_size_splits=self.n_states, axis=1) + \
                       tf.split(runge_kutta_stages_difference_flattened,
                                num_or_size_splits=self.n_runge_kutta_stages * self.n_states,
                                axis=1)
        return return_split

    def predict(self,
                x,
                batch_size=None,
                verbose=0,
                steps=None,
                callbacks=None,
                max_queue_size=10,
                workers=1,
                use_multiprocessing=False):
        x_time, x_power, x_y_0 = x
        delta_0 = x_y_0[:, 0:1]

        runge_kutta_variables = self.CoreNetwork(inputs=[x_time, x_power, delta_0], training=False)

        runge_kutta_prediction = x_y_0 + x_time * tf.squeeze(runge_kutta_variables @ self.IRK_beta.T, axis=2)

        return runge_kutta_prediction.numpy()

    def loss_term_dimension_check(self, loss_weights_complete, y_training):

        if self.loss_terms != len(loss_weights_complete):
            raise Exception('Loss terms and loss terms weight do not match in length.')

        if self.loss_terms != len(y_training):
            raise Exception('Loss terms and training data target do not match in length.')

        if len(loss_weights_complete) != len(y_training):
            raise Exception('Loss terms weight and training data target do not match in length.')

        pass

class CoreNetwork(tf.keras.models.Model):
    """
    This constitutes the core neural network within the PINN model. It outputs the prediction for the RK-stages based
    on the characteristic inputs.
    """

    def __init__(self,
                 n_states: int,
                 n_runge_kutta_stages: int,
                 neurons_in_hidden_layer: list):

        super(CoreNetwork, self).__init__()

        self.n_hidden_layers = len(neurons_in_hidden_layer)
        if not (self.n_hidden_layers > 0):
            raise Exception('Please specify the number of neurons per layer.')

        # TODO: Make it dataset dependent with .adapt(), currently normalisation hardcoded
        self.input_normalisation = tf.keras.layers.experimental.preprocessing.Normalization(axis=1,
                                                                                name='input_normalisation',
                                                                                mean=np.array([5.0,
                                                                                0.1, 0.0]),
                                                                                variance=np.array([
                                                                                8.5, 0.00367,
                                                                                0.8554]))
        self.dense_layers = [tf.keras.layers.Dense(units=units,
                                                   activation=tf.keras.activations.tanh,
                                                   use_bias=True,
                                                   kernel_initializer=tf.keras.initializers.glorot_normal,
                                                   bias_initializer=tf.keras.initializers.zeros,
                                                   name=f'dense_layer_{ii}') for (ii, units) in enumerate(
            neurons_in_hidden_layer)]

        self.normalisation_layers = [
            tf.keras.layers.BatchNormalization(name=f'normalisation_layer_{ii + 1}', trainable=True)
            for ii in range(len(neurons_in_hidden_layer))]

        self.dense_output_layer = tf.keras.layers.Dense(units=n_states * n_runge_kutta_stages,
                                                        activation=tf.keras.activations.linear,
                                                        use_bias=True,
                                                        kernel_initializer=tf.keras.initializers.glorot_normal,
                                                        bias_initializer=tf.keras.initializers.zeros,
                                                        name='output_layer')

        self.reshape_output_layer = tf.keras.layers.Reshape(target_shape=(n_states,
                                                                          n_runge_kutta_stages),
                                                            name='reshape_layer')

    def call(self, inputs, training=None, mask=None):
        input_concatenated = tf.keras.layers.concatenate(inputs, axis=1, name='input_concatenation')

        hidden_layer_input = self.input_normalisation(input_concatenated)

        for ii in range(self.n_hidden_layers):
            hidden_layer_output = self.dense_layers[ii](hidden_layer_input)
            hidden_layer_input = self.normalisation_layers[ii](hidden_layer_output)

        output_layer_output = self.dense_output_layer(hidden_layer_input)
        network_output = self.reshape_output_layer(output_layer_output)

        return network_output

reason: import numpy


text:
import unittest

import numpy

import chainer
from chainer import backend
from chainer import cuda
from chainer import functions
from chainer import gradient_check
from chainer import testing
from chainer.testing import attr
from chainer.testing import condition

@testing.parameterize(*testing.product({
    'train': [True, False],
    'shape': [(3, 2), (5, 6)],
    'dtype': [numpy.float16, numpy.float32, numpy.float64],
}))
class TestRReLU(unittest.TestCase):

    def setUp(self):
        # Avoid unstability of numeraical grad
        self.x = numpy.random.uniform(-1, 1, self.shape).astype(self.dtype)
        self.x[(-0.05 < self.x) & (self.x < 0.05)] = 0.5
        self.gy = numpy.random.uniform(-1, 1, self.shape).astype(self.dtype)
        # Asummption l < u
        self.l = numpy.random.uniform(0, 1)
        self.u = numpy.random.uniform(0, 1)
        if self.l >= self.u:
            self.l, self.u = self.u, self.l
        self.check_forward_options = {}
        self.check_backward_options = {}
        if self.dtype == numpy.float16:
            self.check_forward_options = {'atol': 1e-4, 'rtol': 1e-3}
            self.check_backward_options = {'atol': 5e-4, 'rtol': 5e-3}

    def check_forward(self, x_data):
        x = chainer.Variable(x_data)
        xp = backend.get_array_module(x)
        with chainer.using_config('train', self.train):
            y, r = functions.rrelu(x, l=self.l, u=self.u, return_r=True)
        self.assertEqual(y.data.dtype, self.dtype)
        expected = xp.where(x_data >= 0, x_data, x_data * r)
        testing.assert_allclose(
            expected, y.data, **self.check_forward_options)

    @condition.retry(3)
    def test_forward_cpu(self):
        self.check_forward(self.x)

    @attr.gpu
    @condition.retry(3)
    def test_forward_gpu(self):
        self.check_forward(cuda.to_gpu(self.x))

    def check_backward(self, x_data, y_grad):
        xp = backend.get_array_module(x_data)
        r = xp.random.uniform(self.l, self.u, x_data[
                              0].shape).astype(x_data[0].dtype)

        def f(x):
            return functions.rrelu(x, self.l, self.u, r=r)

        with chainer.using_config('train', self.train):
            gradient_check.check_backward(
                f, x_data, y_grad, dtype=numpy.float64,
                **self.check_backward_options)

    @condition.retry(10)
    def test_backward_cpu(self):
        self.check_backward(self.x, self.gy)

    @attr.gpu
    @condition.retry(10)
    def test_backward_gpu(self):
        self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(self.gy))

@testing.parameterize(*testing.product({
    'specify_r': [True, False],
    'train': [True, False],
    'shape': [(3, 2), (5, 6)],
    'dtype': [numpy.float16, numpy.float32, numpy.float64],
}))
class TestRReLUR(unittest.TestCase):

    def setUp(self):
        self.x = numpy.random.uniform(-1, 1, self.shape).astype(self.dtype)
        self.x[(-0.05 < self.x) & (self.x < 0.05)] = 0.5
        # Asummption l < u
        self.l = numpy.random.uniform(0, 1)
        self.u = numpy.random.uniform(0, 1)
        if self.l >= self.u:
            self.l, self.u = self.u, self.l
        self.r = (numpy.random.uniform(
            self.l, self.u, self.x[0].shape)).astype(self.x[0].dtype)

    def _check(self):
        r = self.r if self.specify_r else None
        with chainer.using_config('tarin', self.train):
            out, out_r = functions.rrelu(
                self.x, self.l, self.u, r=r, return_r=True)

        assert isinstance(out_r, type(out.array))
        if r is None:
            assert out_r.shape == out.array.shape
        else:
            if chainer.config.train:
                assert out_r is r

    def test_cpu(self):
        with chainer.using_config('use_ideep', 'never'):
            self._check()

    @attr.gpu
    def test_gpu(self):
        self.x = cuda.to_gpu(self.x)
        self.r = cuda.to_gpu(self.r)
        self._check()

testing.run_module(__name__, __file__)

max_forks_repo_name: bkvogel/chainer


text:
# -*- coding: utf-8 -*-
"""
Created on Sun Sep 27 22:43:17 2020

@author: Sam
"""

#impoty librarys
import numpy as np
import matplotlib.pyplot as plt
from scipy.optimize import curve_fit as cv
#Import the data from the Text File

#This is inefficiant, i dont care
data10k = np.genfromtxt("ACresonance10kNew.txt",skip_header =3,delimiter=",")

frequency=data10k[:, 0]
rlV_out10k=data10k[:,1]
imV_out10k=data10k[:,2]

data1k = np.genfromtxt("ACresonance1kNew.txt",skip_header =3,delimiter=",")
rlV_out1k=data1k[:,1]
imV_out1k=data1k[:,2]

data100 = np.genfromtxt("ACresonance100New.txt",skip_header =3,delimiter=",")
rlV_out100=data100[:,1]
imV_out100=data100[:,2]

data10 = np.genfromtxt("ACresonance10New.txt",skip_header =3,delimiter=",")
rlV_out10=data10[:,1]
imV_out10=data10[:,2]

data2 = np.genfromtxt("ACresonance2New.txt",skip_header =3,delimiter=",")
rlV_out2=data2[:,1]
imV_out2=data2[:,2]

"""
plt.figure(figsize=(20, 10))
plt.xscale("log")
plt.xlabel("frequency / Hz")
plt.ylabel("Voltage / V")
plt.plot(frequency,rlV_out10k, "g", label = "V_out_real for 10k Ohms for the resistors")
plt.plot(frequency,rlV_out1k, "b", label = "V_out_real for 1k Ohms for the resistors")
plt.plot(frequency,rlV_out100, "pink", label = "V_out_real for 100 Ohms for the resistors")
plt.plot(frequency,rlV_out10, "r", label = "V_out_real for 10 Ohms for the resistors")
plt.plot(frequency,rlV_out2, "black", label = "V_out_real for 2 Ohms for the resistors")
plt.title("AC Resonance graph")
plt.legend()
"""

x = lambda x, y : (x**2 + y**2)**0.5

V_out10k= []
for i in range (len(rlV_out10k)):
    V_out10k.append(x(rlV_out10k[i], imV_out10k[i]))

V_out1k= []
for i in range (len(rlV_out1k)):
    V_out1k.append(x(rlV_out1k[i], imV_out1k[i]))

V_out100= []
for i in range (len(rlV_out100)):
    V_out100.append(x(rlV_out100[i], imV_out100[i]))

V_out10= []
for i in range (len(rlV_out10)):
    V_out10.append(x(rlV_out10[i], imV_out10[i]))

V_out2= []
for i in range (len(rlV_out2)):
    V_out2.append(x(rlV_out2[i], imV_out2[i]))

plt.figure(figsize=(20, 10))
plt.xscale("log")
plt.xlabel("frequency / Hz")
plt.ylabel("Voltage / V")
plt.plot(frequency,V_out10k, "g", label = "V_out for 10k Ohms for the resistors (Magnitude)")
plt.plot(frequency,V_out1k, "b", label = "V_out for 1k Ohms for the resistors (Magnitude)")
plt.plot(frequency,V_out100, "r", label = "V_out for 100 Ohms for the resistors (Magnitude)")
plt.plot(frequency,V_out10, "orange", label = "V_out for 10 Ohms for the resistors (Magnitude)")
plt.plot(frequency,V_out2, "black", label = "V_out for 2 Ohms for the resistors (Magnitude)")
plt.title("AC Resonance graph (Magnitudes)")
plt.legend()

def freqAtHalfAmp (V, f):
    for i in range (len(V)):
        if V[i] > 0.244 and V[i] < 0.256:
            print(f[i])
def resonantFreq (V, f):
    for i in range (len(V)):
        if V[i] > 0.49 and V[i] < 0.51:
            print(f[i])

"""
freqAtHalfAmp(V_out1k, frequency)
freqAtHalfAmp (V_out100, frequency)
freqAtHalfAmp (V_out10, frequency)
freqAtHalfAmp (V_out2, frequency)
"""


text:
import seaborn as sns
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import sqlite3
import datetime
import matplotlib
import cStringIO
import pycurl

DB_FILE="/Users/tparker/pytroll/gina.db"

# sns.set(style="whitegrid", palette="muted")
#
# # Load the example iris dataset
# iris = sns.load_dataset("iris")
#
# # "Melt" the dataset to "long-form" or "tidy" representation
# iris = pd.melt(iris, "species", var_name="measurement")
#
# # Draw a categorical scatterplot to show each observation
# sns.swarmplot(x="measurement", y="value", hue="species", data=iris)

def main():
    conn = sqlite3.connect(DB_FILE, detect_types=sqlite3.PARSE_DECLTYPES)

    query = "SELECT * FROM sighting;"
    df = pd.read_sql_query(query, conn)
    df['proctime'] = df['proc_date'] - df['granule_date']
    print(df.dtypes)

    # proctime = pd.Series(df['proctime'], index=df['granule_date'])
    proctime = pd.Series(df['proctime'], index=df['granule_date'])
    print(df['proctime' ])
    proctime.plot()

    proctime.show()
    conn.close()

if __name__ == "__main__":
    main()

max_stars_repo_name: tparker-usgs/rsprocessing

max_issues_repo_name: tparker-usgs/rsprocessing

max_forks_repo_name: tparker-usgs/rsprocessing


text:
#!/usr/bin/env python3
"""
Unittests for the Cox and Munk algorithm:
    * if input szen, vzen, razi are not two-dimensional?
    * if dimension mismatch between szen, vzen, razi
    * if dimension mismatch between p_glint and vis_im
    * if wind_speed < 0
    * if any input data only contain a single value
"""

import pytest
import rasterio
import numpy as np

from pathlib import Path
from sungc import deglint
from sungc.algorithms import coxmunk_backend
from sungc.rasterio_funcs import load_singleband

from . import urd, create_halved_band

# specify the path to the odc_metadata.yaml of the test datasets
data_path = Path(__file__).parent / "data"
odc_meta_file = data_path / "ga_ls8c_aard_3-2-0_091086_2014-11-06_final.odc-metadata.yaml"

# specify the sub_product
sub_product = "lmbadj"

def test_cxmk_image():
    """
    Check that the generated deglinted band is nearly identical
    to the expected deglinted band
    """
    # Initiate the sunglint correction class
    g = deglint.GlintCorr(odc_meta_file, sub_product)

    # ------------------ #
    #    Cox and Munk    #
    # ------------------ #
    cxmk_xarrlist = g.cox_munk(
        vis_bands=["3"],
        wind_speed=5,
        water_val=5,
    )

    sungc_band = cxmk_xarrlist[0].lmbadj_green.values  # 3D array

    # path to expected sunglint corrected output from NIR subtraction
    exp_sungc_band = (
        data_path
        / "COX_MUNK"
        / "ga_ls8c_lmbadj_3-2-0_091086_2014-11-06_final_band03-deglint-600m.tif"
    )

    # ensure that all valid sungint corrected pixels match expected
    with rasterio.open(exp_sungc_band, "r") as exp_sungc_ds:
        urd_band = urd(sungc_band[0, :, :], exp_sungc_ds.read(1), exp_sungc_ds.nodata)
        assert urd_band.max() < 0.001

def test_glint_images():
    """
    Tes that the generated glint reflectance is nearly
    identical to expected glint reflectance
    """
    exp_glint_band = (
        data_path
        / "COX_MUNK"
        / "ga_ls8c_lmbadj_3-2-0_091086_2014-11-06_final_cm_glint.tif"
    )

    # Initiate the sunglint correction class
    g = deglint.GlintCorr(odc_meta_file, sub_product)

    vzen_file = g.find_file("satellite_view")
    szen_file = g.find_file("solar_zenith")
    razi_file = g.find_file("relative_azimuth")

    # load the required geometry images
    vzen_im, vzen_meta = load_singleband(vzen_file)
    szen_im, szen_meta = load_singleband(szen_file)
    razi_im, razi_meta = load_singleband(razi_file)
    cm_meta = vzen_meta.copy()

    # cox and munk:
    p_glint, p_fresnel = coxmunk_backend(
        view_zenith=vzen_im,
        solar_zenith=szen_im,
        relative_azimuth=razi_im,
        wind_speed=5,
        return_fresnel=False,
    )

    # convert p_glint & p_fresnel from np.float32 to np.int16
    p_nodata = cm_meta["nodata"]  # this is np.nan
    p_glint[p_glint != p_nodata] *= g.scale_factor
    p_glint[p_glint == p_nodata] = -999.0
    p_glint = np.array(p_glint, order="C", dtype=np.int16)

    # ensure that all valid sungint corrected pixels match expected
    with rasterio.open(exp_glint_band, "r") as eglint_ds:
        urd_glint = urd(p_glint, eglint_ds.read(1), eglint_ds.nodata)
        assert urd_glint.max() < 0.001

def test_cxmk_bands():
    """
    Ensure that the Cox and Munk module raises an
    Exception if the specified vis_band_id does not exist
    """
    g = deglint.GlintCorr(odc_meta_file, sub_product)

    with pytest.raises(Exception) as excinfo:
        g.cox_munk(
            vis_bands=["20"],  # this band id doesn't exist
            wind_speed=5,
            water_val=5,
        )
    assert "is missing from bands" in str(excinfo)

def test_nodata_band():
    """
    Ensure that the Cox and Munk module raises an
    Exception if the input band only contains nodata
    """
    g = deglint.GlintCorr(odc_meta_file, sub_product)

    with pytest.raises(Exception) as excinfo:
        g.cox_munk(
            vis_bands=["7"],  # dummy band only contains nodata (-999)
            wind_speed=5,
            water_val=5,
        )
    assert "only contains a single value" in str(excinfo)

def test_nodata_vzen(tmp_path):
    """
    Ensure that the Cox and Munk module raises an
    Exception if any of the solar-view geometry
    band only contains nodata. We will test this
    with the sensor view-zenith.
    """
    g = deglint.GlintCorr(odc_meta_file, sub_product)

    cxmk_dir = tmp_path / "COX_MUNK"
    cxmk_dir.mkdir()

    # create empty view-zenith tiff
    dum_vzen = cxmk_dir / "ga_ls8c_oa_3-2-0_091086_2014-11-06_final_DUMMY_view-zenith.tif"

    with rasterio.open(g.find_file("satellite_view"), "r") as vzen_ds:
        kwargs = vzen_ds.meta.copy()

        # write dummy geotiff
        nrows = vzen_ds.height
        ncols = vzen_ds.width
        dtype = np.dtype(kwargs["dtype"])
        nodata = kwargs["nodata"]  # np.nan

        arr = np.zeros([nrows, ncols], order="C", dtype=dtype)
        arr[:] = nodata

        with rasterio.open(dum_vzen, "w", **kwargs) as dst:
            dst.write(arr, 1)

    # specify dummy band
    with pytest.raises(Exception) as excinfo:
        g.cox_munk(
            vis_bands=["3"],
            vzen_file=dum_vzen,  # dummy band only contains nodata (np.nan)
            wind_speed=5,
            water_val=5,
        )
    assert "only contains a single value" in str(excinfo)

def test_geom_same_dims(tmp_path):
    """
    Ensure that the Cox and Munk module will raise an
    Exception if a view-solar geometry band does not
    have the same dimensions as the other bands.
    """
    g = deglint.GlintCorr(odc_meta_file, sub_product)

    cxmk_dir = tmp_path / "COX_MUNK"
    cxmk_dir.mkdir()

    # create a small view-zenith tiff
    resmpl_tifs, rio_meta = create_halved_band(g.find_file("satellite_view"), cxmk_dir)

    # cox_munk() should raise an Exception as vzen band does not
    # have the same shape as the solar-zenith and relative-azimuth
    # bands
    with pytest.raises(Exception) as excinfo:
        g.cox_munk(
            vis_bands=["3"],
            vzen_file=resmpl_tifs[0],
            wind_speed=5,
            water_val=5,
        )
    assert "Dimension mismatch" in str(excinfo)

def test_windspeed_lt0():
    """
    Ensure that the Cox and Munk module will raise an
    Exception if the wind speed < 0 m/s
    """
    g = deglint.GlintCorr(odc_meta_file, sub_product)

    with pytest.raises(Exception) as excinfo:
        g.cox_munk(
            vis_bands=["3"],
            wind_speed=-1,
            water_val=5,
        )
    assert "wind_speed must be greater than 0 m/s" in str(excinfo)


text:
import jellyfish
import numpy as np

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.metrics.pairwise import cosine_similarity

def jaro_winkler_similarity(word_matrix):
    '''
    Compute the Jaro-Winkler similarities among all word pairs

    :param word_matrix: Word matrix
    :return: float: Computed similarity score
    '''
    similarity_matrix = np.zeros(word_matrix.shape)

    for i in similarity_matrix.shape[0]:
        for j in similarity_matrix.shape[j]:
            similarity_matrix[i, j] = 1.0 - jellyfish.jaro_winkler(w1, w2)

    return similarity_matrix

def cosine_similarity(word_matrix):
    '''
    Compute the cosine similarities among all word pairs

    :param word_matrix: Word matrix
    :return: float: Computed similarity score
    '''
    vectorizer = CountVectorizer(word_matrix)
    vectorizer.fit(word_matrix)
    vectors = [t for t in vectorizer.transform(text).toarray()]
    return cosine_similarity(vectors)


text:
"""
Implementation of a sample horizon plot, showing both negative and positive
values in the same banded region.
"""

# Major library imports
from numpy import cumprod, linspace, random
import time

# Enthought library imports
from enable.api import Component, ComponentEditor
from traits.api import HasTraits, Instance
from traitsui.api import Item, Group, View

# Chaco imports
from chaco.api import ArrayDataSource, LinearMapper, DataRange1D, \
                      HPlotContainer, OverlayPlotContainer, PlotGrid, PlotAxis, \
                      FilledLinePlot, BandedMapper, HorizonPlot, \
                      ColorBar, RdBu as cmap
from chaco.tools.api import PanTool

from chaco.scales.api import CalendarScaleSystem
from chaco.scales_tick_generator import ScalesTickGenerator

def create_dates(numpoints, units="days"):
    """ Returns **numpoints** number of dates that evenly bracket the current
    date and time.  **units** should be one of "weeks", "days", "hours"
    "minutes", or "seconds".
    """
    units_map = { "weeks" : 7*24*3600,
                  "days" : 24*3600,
                  "hours" : 3600,
                  "minutes" : 60,
                  "seconds" : 1 }
    now = time.time()
    dt = units_map[units]
    dates = linspace(now, now+numpoints*dt, numpoints)
    return dates

#===============================================================================
# # Create the Chaco plot.
#===============================================================================
def _create_plot_components():
    # Create the data and datasource objects
    # In order for the date axis to work, the index data points need to
    # be in units of seconds since the epoch.  This is because we are using
    # the CalendarScaleSystem, whose formatters interpret the numerical values
    # as seconds since the epoch.
    high = 1.
    numpoints = 5000

    random.seed(1000)

    index = create_dates(numpoints)
    price = 100*cumprod(random.lognormal(0.0, 0.04, size=numpoints))
    changes = price/price[0] - 1.

    index_ds = ArrayDataSource(index)
    value_ds = ArrayDataSource(changes, sort_order="none")
    value_range = DataRange1D(value_ds, low=-high, high=high)

    index_mapper = LinearMapper(range=DataRange1D(index_ds), stretch_data=False)

    horizon = HorizonPlot(
            bands = 4,
            index = index_ds,
            value = value_ds,
            index_mapper = index_mapper,
            value_mapper = BandedMapper(range=DataRange1D(low=0, high=high)),
            color_mapper = cmap(range=value_range),
            )
    horizon.tools.append(PanTool(horizon, constrain=True, constrain_direction="x"))
    axis = PlotAxis(mapper = horizon.value_mapper, component=horizon, orientation="left",
                   tick_label_position="outside")
    horizon.overlays.append(axis)

    bottom_axis = PlotAxis(horizon, orientation="bottom",
                      tick_generator=ScalesTickGenerator(scale=CalendarScaleSystem()))
    horizon.overlays.append(bottom_axis)

    filled = FilledLinePlot(
            index = index_ds,
            value = value_ds,
            index_mapper = index_mapper,
            value_mapper = LinearMapper(range=value_range, stretch_data=False),
            fill_color = (0.81960784,  0.89803922,  0.94117647),
            edge_color = 'transparent',
            )
    filled.tools.append(PanTool(filled, constrain=True, constrain_direction="x"))
    axis = PlotAxis(mapper = filled.value_mapper, component=filled, orientation="left",
                   tick_label_position="outside")
    filled.overlays.append(axis)

    grid = PlotGrid(mapper = filled.value_mapper, component=filled, orientation='horizontal',
                    line_color='lightgray', line_style="dot",)
    filled.underlays.append(grid)

    colormap = horizon.color_mapper
    colorbar = ColorBar(index_mapper=LinearMapper(range=colormap.range),
                        color_mapper=colormap,
                        orientation='v',
                        resizable='v',
                        width=20,
                        padding=20)

    padding = (40, 20, 0, 0)
    over1 = HPlotContainer(use_backbuffer=True, padding=padding, padding_top=20)
    over1.add(filled)
    over1.add(colorbar)

    over2 = OverlayPlotContainer(padding = padding, padding_bottom=40)
    over2.add(horizon)

    return over1, over2

filled_size=(800,220)
horizon_size=(800,70)
title="Horizon plot example"

#===============================================================================
# # Demo class that is used by the demo.py application.
#===============================================================================
class Demo(HasTraits):
    filled = Instance(Component)
    horizon = Instance(Component)

    traits_view = View(
                    Group(
                        Item('filled', editor=ComponentEditor(size=filled_size),
                             show_label=False),
                        Item('horizon', editor=ComponentEditor(size=horizon_size),
                             show_label=False),
                        orientation = "vertical"),
                    resizable=False, title=title,
                    width=filled_size[0], height=filled_size[1]+horizon_size[1],
                    )

demo = Demo()

if __name__ == "__main__":

    filled, horizon = _create_plot_components()
    demo.set(horizon=horizon, filled=filled)
    demo.configure_traits()

#--EOF---

max_stars_repo_name: burnpanck/chaco

max_issues_repo_name: burnpanck/chaco

max_forks_repo_name: burnpanck/chaco

reason: from numpy


text:
#!/usr/bin/python
# -*- encoding: utf-8 -*-
# part of https://github.com/WolfgangFahl/play-chess-with-a-webcam

# see https://stackoverflow.com/a/54246290/1497139
# renders at > 65 fps on a mac mini
import numpy as np
import cv2
from mss import mss
from PIL import Image
import time

last_time = time.time()
bbox = {'top': 50, 'left': 50, 'width': 800, 'height': 640}
sum = 0
count = 0
sct = mss()
while 1:
    sct_img = sct.grab(bbox)
    grabtime = time.time() - last_time
    sum += 1 / grabtime
    count = count + 1
    print('grab took %3.2f s= %4.1f fps avg=%4.1f fps' % (grabtime, 1 / grabtime, sum / count))
    cv2.imshow('screen', np.array(sct_img))
    last_time = time.time()

    if cv2.waitKey(1) & 0xFF == ord('q'):
        cv2.destroyAllWindows()
        break

max_stars_repo_name: gitter-badger/play-chess-with-a-webcam

max_issues_repo_name: gitter-badger/play-chess-with-a-webcam

max_forks_repo_name: gitter-badger/play-chess-with-a-webcam


text:
import torch
import numpy as np
import math
import glog as log

class TangentFinder(object):
    def __init__(self, x, ball_center, short_radius, long_radius, plane_normal_vector, norm="l2"):
        '''
        :param x: the original image which is outside the ball, vector form.
        :param ball_center: the perturbed image lies on the decision boundary
        :param short_radius: the radius of the ball
        :param plane_normal_vector: the normal vector of hyperplane
        :param norm: l2 or linf
        '''
        self.x = x
        self.o = ball_center
        self.ox = self.x - self.o
        self.S = short_radius
        self.L = long_radius

        self.norm = norm
        self.u = plane_normal_vector # unit normal vector
        self.ord = np.inf if self.norm == "linf" else 2

        # assert self.sin_alpha() ** 2 + self.cos_alpha() ** 2 - 1.0 < 0.00001, "alpha assert error:{}".format(self.sin_alpha() ** 2 + self.cos_alpha() ** 2)
        # assert self.sin_beta() ** 2 + self.cos_beta() ** 2 - 1.0 < 0.00001, "beta assert error:{}".format(self.sin_beta() ** 2 + self.cos_beta() ** 2)
        # assert self.sin_gamma() ** 2 + self.cos_gamma() ** 2 - 1.0 < 0.00001, "gamma assert error:{}".format(self.sin_gamma() ** 2 + self.cos_gamma() ** 2)

    def compute_2D_x(self):
        x_norm = torch.norm(self.ox, p=self.ord)
        theta = self.theta()
        return (x_norm * torch.sin(theta), -x_norm * torch.cos(theta))

    def compute_tangent_point_of_ellipse(self):
        S = self.S
        L = self.L
        x_2D = self.compute_2D_x()
        x0 = x_2D[0].item()
        z0 = x_2D[1].item()
        in_sqrt = -L**2 * S**2 + L**2 * x0**2 + S**2 * z0**2
        if in_sqrt < 0:
            log.info("x_k inside sqrt < 0 ({:.7f})! convert to 0.".format(in_sqrt))
            in_sqrt = 0
        # if L**2*x0**2 + S**2*z0**2 < 1e-5:
        #     raise ZeroDivisionError("The denominator is zero!")

        # the below is another solution
        #xk = S**2 * (L**2 - z0 * (L**2 * S**2 * z0/ (L**2 * x0**2 + S**2  * z0 ** 2) - L**2 * x0 * math.sqrt(-L**2 * S**2 + L**2 * x0 ** 2 + S**2 * z0 ** 2)/(L**2 * x0**2 + S**2 * z0**2))) / (L**2 * x0)
        #zk = (L**2 * S**2 * z0)/(L**2 * x0 ** 2 + S**2 * z0**2) - (L**2 * x0 * math.sqrt(-L**2 * S**2 + L**2  * x0**2 + S**2 * z0**2))/(L**2 * x0 ** 2 + S**2 * z0**2)

        # Below is NIPS version!
        xk = S**2 * (L**2 - z0 * (L**2 * S**2 * z0/ (L**2 * x0**2 + S**2 * z0**2) + L**2 * x0 * math.sqrt(in_sqrt)/(L**2 * x0**2 + S**2 * z0**2))) / (L**2 * x0)
        zk = (L**2 * S**2 * z0)/(L**2 * x0 ** 2 + S**2 * z0**2) + (L**2 * x0 * math.sqrt(in_sqrt))/(L**2 * x0 ** 2 + S**2 * z0**2)

        return xk, zk

    def theta(self):
        return torch.acos(torch.dot(self.ox, -self.u)/(torch.norm(self.ox, p=self.ord) * torch.norm(self.u, p=self.ord)))

    def compute_tangent_point(self):
        x_k, z_k = self.compute_tangent_point_of_ellipse()
        numerator = self.ox - torch.dot(self.ox, self.u) * self.u / torch.norm(self.u) ** 2
        ok_prime = (numerator / torch.norm(numerator, p=self.ord)) * math.fabs(x_k)
        # log.info("[ellipsoid] x_k is {:.4f} z_k is {:.4f}".format(x_k, z_k))
        ok = ok_prime + z_k * self.u # / torch.norm(self.u)
        return ok + self.o

if __name__ == "__main__":
    sr = 1.0
    lr = 2.0
    import math
    import numpy as np

    x = torch.tensor([2,2]).float()
    all_tangent_point_list = []
    normal_vector = torch.zeros(2).double()
    normal_vector[0] = 0  # (0,1/tan(alpha),1)
    normal_vector[1] = 1
    normal_vector /= torch.norm(normal_vector)
    O = torch.zeros(2).double()
    O[0] = 4
    O[1] = 3
    t = TangentFinder(x, O, sr, lr, normal_vector, "l2")
    print(t.compute_tangent_point().numpy())
    #
    # #
    # normal_vector = torch.zeros(3)
    # normal_vector[2] = 1
    # t = TangentFinder(torch.tensor([1.5 * r, 0, -0.5 * r]), torch.zeros(3), r, normal_vector, "l2")
    # tangent = t.compute_tangent_point()
    # print(tangent)
    # # 计算直线和平面的交点
    # print(compute_line_plane_intersect_point(torch.tensor([1.5*r,0,-0.5*r]), tangent, torch.zeros(3),normal_vector))

    # figure 1
    # normal_vector = torch.zeros(3).float()
    # normal_vector[-1] = 1
    # normal_vector /= torch.norm(normal_vector)
    # x = torch.tensor([1,2]).float()
    # t = TangentFinder(x, torch.tensor([0,0,0]).float(),sr,lr, normal_vector, "l2")
    # tangent = t.compute_tangent_point()
    # print(tangent)
    # # 计算直线和平面的交点
    # print(compute_line_plane_intersect_point(x, torch.tensor([4,4]).float(), torch.tensor([4,3]).float(),
    #                                          normal_vector))
    # print(compute_line_plane_intersect_point(x, tangent,
    #                                          torch.tensor([4, 3]).float(),
    #                                          normal_vector))

max_stars_repo_name: machanic/TangentAttack

max_issues_repo_name: machanic/TangentAttack

max_forks_repo_name: machanic/TangentAttack


text:
import torch
import numpy as np
import yaml
from . import networks
from . import math_utils
from ...common.utils import pprint_dict

def _combine(accel_pred, metric_pred, jacobians, extra_metrics):
    B = []
    C = []

    for i in range(len(jacobians)):
        jacobian = jacobians[i]
        metric = metric_pred[i]
        accel = accel_pred[i]
        B.append(np.linalg.multi_dot((jacobian.T, metric, accel)))
        C.append(np.linalg.multi_dot((jacobian.T, metric, jacobian)))

    for jacobian, metric, ddx in extra_metrics:
        B.append(np.linalg.multi_dot((jacobian.T, metric, ddx)))
        C.append(np.linalg.multi_dot((jacobian.T, metric, jacobian)))

    B = np.sum(B, axis=0)
    C = np.sum(C, axis=0)

    accel = np.dot(np.linalg.pinv(C), B)
    return accel

def make_nets(specs, device):
    ret = {}
    for net_name, spec in specs.items():
        net_class = getattr(networks, spec['class'])
        net_args = spec.get('net_kwargs', {})
        net = net_class(**net_args).to(device)
        ret[net_name] = net
    return ret

class NeuralRMP(object):
    def __init__(self, weights_file, n_control_points=4, device='cuda'):
        state_dict = torch.load(weights_file, map_location='cpu')
        print('loaded %s' % weights_file)
        g = state_dict.get('global_args', {})
        print('global args:')
        print(pprint_dict(g))
        self.g = g

        if isinstance(g.model_spec, dict):
            nets = make_nets(g.model_spec, device)
        else:
            nets = make_nets(yaml.load(open(g.model_spec).read()), device)

        for name, net in nets.items():
            net.load_state_dict(state_dict['nets'][name])
            net.train(False)

        self.device = device
        self.nets = nets

        self.n_control_points = n_control_points
        self.control_point_accels = None
        self.control_point_metrics = None

    def _run_model(self, img, wp_local, vel_local, angular_vel):
        with torch.no_grad():
            nets = self.nets
            img_th = torch.as_tensor(img, dtype=torch.float32, device=self.device)
            wp_th = torch.as_tensor(wp_local, dtype=torch.float32, device=self.device)
            vel_th = torch.as_tensor(vel_local, dtype=torch.float32, device=self.device)
            angular_vel_th = torch.as_tensor(angular_vel, dtype=torch.float32, device=self.device)

            img_feature = nets['img_encoder'](img_th)
            goal_feature = nets['wp_encoder'](wp_th)
            vel_feature = nets['vel_encoder'](vel_th)
            angular_vel_feature = nets['angular_vel_encoder'](angular_vel_th)

            feature = torch.cat([img_feature, goal_feature, vel_feature, angular_vel_feature],
                                dim=-1)

            pred_accel, pred_metric = nets['rmp_regressor'](feature)

            pred_accel = pred_accel.data.cpu().numpy().reshape((self.n_control_points, 2)) / self.g.accel_output_scale
            pred_metric = pred_metric.data.cpu().numpy().reshape((self.n_control_points, 2, 2)) / self.g.metric_output_scale

            if self.g.get('log_metric', False):
                pred_metric = np.array([math_utils.exp_psd_matrix(m) for m in pred_metric], np.float32)

            return pred_accel, pred_metric

    def compute_optimal_accel(
            self, img, wp_local, vel_local, angular_vel, jacobians, extra_metrics):
        '''
        :param extra_metrics: a list of tuples (jacobian, metric, acceleration) to be combined.
        :return:
        '''

        accel_pred, metric_pred = self._run_model(
            img[None, :].astype(np.float32),
            wp_local[None, :].astype(np.float32) * self.g.waypoint_scale,
            vel_local[None, :].astype(np.float32),
            np.atleast_1d(angular_vel)[None, :].astype(np.float32))

        self.control_point_accels = accel_pred
        self.control_point_metrics = metric_pred
        return _combine(accel_pred, metric_pred, jacobians, extra_metrics)


text:
#!/usr/bin/python3

"""Draw QC for images with their masks. The dummy dataset as a samplelist means
that all listed case numbers are attempted to use, with maximum number of
lesions."""

import logging

import numpy as np
from skimage import measure

import dwi.conf
import dwi.dataset
import dwi.mask
import dwi.plot
import dwi.util
from dwi.types import ImageMode

def parse_args():
    """Parse command-line arguments."""
    p = dwi.conf.get_parser(description=__doc__)
    p.add('-m', '--modes', nargs='+', type=ImageMode,
          default=['DWI-Mono-ADCm',
                   # 'DWI-Kurt-ADCk', 'DWI-Kurt-K',
                   # 'T2w-std',
                   ],
          help='imaging modes')
    p.add('-s', '--samplelist',
          help='samplelist identifier (exclude to use the dummy dataset)')
    p.add('-c', '--cases', nargs='+', type=int,
          help='cases to include, if not all')
    p.add('-A', '--all_slices', action='store_true',
          help='include all slices')
    p.add('-R', '--include_raw', action='store_true',
          help='include "raw" b=2000 mode')
    p.add('-C', '--connected_regions', action='store_true',
          help='colorize connected regions')
    p.add('-L', '--label', action='store_true',
          help='write text label')
    return p.parse_args()

def read_case(mode, case, scan, lesions, all_slices, include_raw, mbb_pad=10):
    """Read case. Return raw DWI, pmap, pmask, lmasks, initial prostate slice
    index.
    """
    img = dwi.dataset.read_tmap(mode, case, scan)[:, :, :, 0]
    img = dwi.util.normalize(img, mode)
    pmask = dwi.dataset.read_prostate_mask(mode, case, scan)
    lmasks = []
    for i in lesions:
        try:
            lmasks.append(dwi.dataset.read_lesion_mask(mode, case, scan, i))
        except FileNotFoundError as e:
            # No more lesion masks.
            logging.info('%s (got %d)', e, len(lmasks))
            if not lmasks:
                raise

    if mbb_pad is not None:
        mbb = pmask.mbb(mbb_pad)
        img = img[mbb]
        pmask = pmask[mbb]
        lmasks = [x[mbb] for x in lmasks]

    if all_slices:
        pad = (np.inf, np.inf, np.inf)
    else:
        pad = (0, np.inf, np.inf)
    prostate_slices = pmask.mbb(pad=pad)
    img = img[prostate_slices]
    pmask = pmask[prostate_slices]
    lmasks = [x[prostate_slices] for x in lmasks]
    images = [img]

    if include_raw:
        rawmode = ImageMode('DWI-b2000')
        raw = dwi.dataset.read_tmap(rawmode[0], case, scan,
                                    params=[-1])[:, :, :, 0]
        raw = dwi.util.normalize(raw, rawmode)
        if mbb_pad is not None:
            raw = raw[mbb]
        raw = raw[prostate_slices]
        images = [raw] + images

    return images, pmask, lmasks, prostate_slices[0].start

def get_label_plot(mask):
    """Get connected regions for visualization."""
    labels, n = measure.label(mask, return_num=True, connectivity=None)
    labels = (labels / n).astype(np.float32)
    logging.debug([mask.info['path'], n])
    return labels

def plot_case(imgs, masks, label, path, connected_regions):
    """Plot case."""
    overlay = dwi.mask.overlay_masks(masks)
    if connected_regions:
        masks = [get_label_plot(x) for x in masks]
    d = dict(nrows=len(imgs) + len(masks), ncols=len(imgs[0]), suptitle=label,
             path=path)
    d['titles'] = [None] * (d['nrows'] * d['ncols'])
    for i, plt in enumerate(dwi.plot.generate_plots(**d)):
        plt.rcParams['savefig.dpi'] = '60'
        row, col = i // d['ncols'], i % d['ncols']
        kwargs = dict(vmin=0, vmax=1, cmap='hot')
        if row < len(imgs):
            # It's an image.
            plt.imshow(imgs[row][col], cmap='gray')
            plt.imshow(overlay[col], **kwargs, alpha=1.0)
        else:
            # It's a mask.
            mask = masks[row - len(imgs)]
            plt.imshow(mask[col], **kwargs)
            # plt.imshow(mask[col], vmin=-1, vmax=1, cmap='gray')

def draw_dataset(ds, all_slices, include_raw, connected_regions, label_fmt,
                 path_fmt):
    """Process a dataset."""
    logging.info('Mode: %s', ds.mode)
    logging.info('Samplelist: %s', ds.samplelist)
    for i, (case, scan, lesions) in enumerate(ds.each_image_id(), 1):
        try:
            imgs, pmask, lmasks, _ = read_case(ds.mode, case, scan, lesions,
                                               all_slices, include_raw)
        except FileNotFoundError as e:
            logging.warning(e)
        else:
            d = dict(i=i, c=case, s=scan, m=ds.mode)
            label = label_fmt.format(**d) if label_fmt else None
            path = path_fmt.format(**d)
            print('Plotting to {}'.format(path))
            plot_case(imgs, [pmask] + lmasks, label, path, connected_regions)

def main():
    """Main."""
    args = parse_args()
    cls = dwi.dataset.Dataset if args.samplelist else dwi.dataset.DummyDataset
    datasets = (cls(x, args.samplelist, cases=args.cases) for x in args.modes)
    label_fmt = ' {m} {c}-{s}' if args.label else None
    path_fmt = 'work/fig/masks/{m}_{c:03d}-{s}.png'
    for ds in datasets:
        draw_dataset(ds, args.all_slices, args.include_raw,
                     args.connected_regions, label_fmt, path_fmt)

if __name__ == '__main__':
    main()

max_stars_repo_name: jupito/dwilib

max_issues_repo_name: jupito/dwilib

max_forks_repo_name: jupito/dwilib


text:
# ================================================
# Skeleton codes for HW5
# Read the skeleton codes carefully and put all your
# codes into function "reconstruct_from_binary_patterns"
# ================================================

import sys
import cv2
import numpy as np
from math import log, ceil, floor
import matplotlib.pyplot as plt
import pickle

def help_message():
    # Note: it is assumed that "binary_codes_ids_codebook.pckl", "stereo_calibration.pckl",
    # and images folder are in the same root folder as your "generate_data.py" source file.
    # Same folder structure will be used when we test your program

    print("Usage: [Output_Directory]")
    print("[Output_Directory]")
    print("Where to put your output.xyz")
    print("Example usages:")
    print(sys.argv[0] + " ./")

def reconstruct_from_binary_patterns():
    scale_factor = 1.0
    ref_white = cv2.resize(cv2.imread("images/pattern000.jpg", cv2.IMREAD_GRAYSCALE) / 255.0, (0,0), fx=scale_factor,fy=scale_factor)
    ref_black = cv2.resize(cv2.imread("images/pattern001.jpg", cv2.IMREAD_GRAYSCALE) / 255.0, (0,0), fx=scale_factor,fy=scale_factor)

    ref_avg   = (ref_white + ref_black) / 2.0

    ref_on    = ref_avg + 0.05 # a threshold for ON pixels
    ref_off   = ref_avg - 0.05 # add a small buffer region

    h,w = ref_white.shape

    # mask of pixels where there is projection
    proj_mask = (ref_white > (ref_black + 0.05))

    scan_bits = np.zeros((h,w), dtype=np.uint16)

    # analyze the binary patterns from the camera
    for i in range(0,15):
        # read the file

        patt_gray = cv2.resize(cv2.imread("images/pattern%03d.jpg"%(i+2), cv2.IMREAD_GRAYSCALE) / 255.0, (0,0), fx=scale_factor,fy=scale_factor)

        # mask where the pixels are ON
        on_mask = (patt_gray > ref_on) & proj_mask

        on_mask_img = np.zeros((h,w), dtype=np.float32)
        tempv = np.zeros((h,w), dtype=np.float32)

        # this code corresponds with the binary pattern code
        bit_code = np.uint16(1 << i)

        #scan_bits = bit_code | (int)(on_mask)

        for x in range(0,h):
            for j in range(0,w):
                if on_mask[i][j] == True:
                    scan_bits[x][j] = scan_bits[x][j] | bit_code

        # TODO: populate scan_bits by putting the bit_code according to on_mask

    print("load codebook")
    # the codebook translates from <binary code> to (x,y) in projector screen space
    with open("binary_codes_ids_codebook.pckl","rb") as f:
        binary_codes_ids_codebook = pickle.load(f, encoding='latin1')

    camera_points = []
    projector_points = []
    half_img = np.zeros((h,w,3), np.float32)
    s=0
    for x in range(w):
        for y in range(h):
            if not proj_mask[y,x]:
                continue # no projection here
            if scan_bits[y,x] not in binary_codes_ids_codebook:
                continue # bad binary code

            # TODO: use binary_codes_ids_codebook[...] and scan_bits[y,x] to
            # TODO: find for the camera (x,y) the projector (p_x, p_y).
            # TODO: store your points in camera_points and projector_points

            # IMPORTANT!!! : due to differences in calibration and acquisition - divide the camera points by 2

            p_x, p_y = binary_codes_ids_codebook[scan_bits[y,x]]
            Red = p_x
            Green = p_y
            Blue = 0
            rgb_sum = Red + Green + Blue;
            NormalizedRed = Red/rgb_sum;
            NormalizedGreen = Green/rgb_sum;
            NormalizedBlue = Blue/rgb_sum;
            if p_x >= 1279 or p_y >= 799: # filter
                continue
            half_img[y][x] = [0, p_y*255/799.0,p_x*255/1279.0]
            #projector_points[s][0] = p_x
            #projector_points[s][1] = p_y
            #s = s+1

    cv2.imshow("Half", half_img)
    cv2.waitKey(0)

    # now that we have 2D-2D correspondances, we can triangulate 3D points!

    # load the prepared stereo calibration between projector and camera
    with open("stereo_calibration.pckl","r") as f:
        d = pickle.load(f)
        camera_K    = d['camera_K']
        camera_d    = d['camera_d']
        projector_K = d['projector_K']
        projector_d = d['projector_d']
        projector_R = d['projector_R']
        projector_t = d['projector_t']
        # TODO: use cv2.undistortPoints to get normalized points for camera, use camera_K and camera_d
        # TODO: use cv2.undistortPoints to get normalized points for projector, use projector_K and projector_d
        # TODO: use cv2.triangulatePoints to triangulate the normalized points
        # TODO: use cv2.convertPointsFromHomogeneous to get real 3D points
        # TODO: name the resulted 3D points as "points_3d"
    return points_3d

def write_3d_points(points_3d):

	# ===== DO NOT CHANGE THIS FUNCTION =====
    print("write output point cloud")
    print(points_3d.shape)
    output_name = sys.argv[1] + "output.xyz"
    with open(output_name,"w") as f:
        for p in points_3d:
            f.write("%d %d %d\n"%(p[0,0],p[0,1],p[0,2]))

    return points_3d, camera_points, projector_points

if __name__ == '__main__':

	# ===== DO NOT CHANGE THIS FUNCTION =====

	# validate the input arguments
    if (len(sys.argv) != 2):
        help_message()
        sys.exit()

    points_3d = reconstruct_from_binary_patterns()
    write_3d_points(points_3d)


text:
import time, datetime, random, os

import numpy as np
import tensorflow as tf

from HomoProtReader import HomoProtReader
from SameLenProtReader import NewProtReader
from layers import *
from BN_layers import BN_Dense

from dilated_encoder import Dilated_Encoder
from reducing_encoder import Reducing_Encoder_243
from sample_RNN_decoder import Sample_RNN

def basic_kl(mu, var):
	return -0.5 * tf.reduce_sum(1 + tf.log(var) - tf.square(mu) - var, 1)

def full_kl(mu, var):
	return -0.5 * (tf.log(var) - tf.square(mu) - var + 1)

def setup_log(prefix = "model_243_"):
	run_time = datetime.datetime.now()
	run_day = "%02d" %(run_time.day)
	run_month = "%02d" %(run_time.month)
	run_hour = "%02d" %(run_time.hour)
	run_minute = "%02d" %(run_time.minute)
	log_dir = prefix + run_month + run_day + "/"
	log_subdir = run_hour + "h_" + run_minute + "m/"

	if not os.path.isdir(log_dir):
		os.mkdir(log_dir)
	if not os.path.isdir(log_dir + log_subdir):
		os.mkdir(log_dir + log_subdir)

	return log_dir, log_subdir

class Homo_Model_243(object):
	def __init__(self, batch_size, max_seq_len, tab, model_hidden, prot_latent_rep, clust_size = 4, is_training = True):
		self.batch_size = batch_size
		self.max_seq_len = max_seq_len
		self.prot_latent_rep = prot_latent_rep
		self.clust_size = clust_size

		self.hash = tab

		self.prot_wave_in = Dilated_Encoder("prot_wave", is_training, self.batch_size, self.max_seq_len, channels = model_hidden, embedding_size = 64, do_embed = True, use_skip = True)
		self.prot_encoder = Reducing_Encoder_243("prot_encoder", is_training, self.batch_size, channels = model_hidden)
		self.prot_mu = BN_Dense("prot_mu", is_training, model_hidden, prot_latent_rep)
		self.prot_var = BN_Dense("prot_var", is_training, model_hidden, prot_latent_rep)

		self.decoder = Sample_RNN(is_training, [81, 27, 9, 3], 1, model_hidden, True, 22, self.batch_size * (self.clust_size - 1),
									self.max_seq_len, global_conditioning = True, global_latent_size = prot_latent_rep, up_factor = 3)

	def one_hot(self, x):
		transformed = tf.one_hot(x, depth=22)
		transformed = tf.squeeze(transformed, [2])
		return tf.cast(transformed, tf.float32)

	def activated_on(self, rep_seq, homo_seq):
		prot_wave = self.prot_wave_in.activated_on(rep_seq)
		prot_code = self.prot_encoder.activated_on(prot_wave)

		prot_mu = self.prot_mu.activated_on(prot_code)
		prot_var = self.prot_var.activated_on(prot_code)
		prot_var = tf.nn.softplus(prot_var + 0.541325)

		prot_z = prot_mu + tf.sqrt(prot_var) * tf.random_normal(prot_mu.get_shape())

		prot_kl = basic_kl(prot_mu, prot_var)
		full_kl_loss = full_kl(prot_mu, prot_var)

		tf.summary.image("kl_viz", tf.expand_dims(tf.expand_dims(full_kl_loss, [0]), [3]))
		tf.summary.scalar("prot_KL", tf.reduce_mean(prot_kl, 0))

		triple_z = tf.reshape(tf.tile(prot_z, [1, 3]), [self.batch_size * 3, self.prot_latent_rep])
		recon = self.decoder.activated_on(homo_seq, triple_z)

		return recon, prot_kl

	def loss(self, catted_seq, kl_strength = 1.0):
		kl_strength = tf.minimum(kl_strength, 1.0)
		kl_strength = tf.maximum(0.0, kl_strength)

		tf.summary.scalar("kl_strength", kl_strength)

		reshaped_seq = tf.reshape(catted_seq, [self.batch_size, self.clust_size, self.max_seq_len, 1])
		rep_seq = reshaped_seq[:, 0, :, :] #get the first of the batch elements.
		homo_seq = tf.reshape(reshaped_seq[:, 1:, :, :], [self.batch_size * (self.clust_size - 1), self.max_seq_len, 1])

		rep_1h = self.one_hot(rep_seq)
		homo_1h = self.one_hot(homo_seq)

		rep_64 = tf.cast(tf.argmax(rep_1h, 2), tf.int64)
		tf.summary.text("rep", tf.slice(tf.reduce_join(self.hash.lookup(rep_64), reduction_indices=[1]), [0], [10]))

		homo_64 = tf.cast(tf.argmax(homo_1h, 2), tf.int64)
		tf.summary.text("homologs", tf.slice(tf.reduce_join(self.hash.lookup(homo_64), reduction_indices=[1]), [0], [10]))
		#Okay I feel more confident in this pipeline.

		homo_input = tf.pad(homo_1h, [[0, 0], [1, 0], [0, 0]])[:, :-1, :]

		seq_mask = tf.cast(tf.sign(tf.argmax(homo_1h, 2)), tf.float32)

		recon, kl_loss = self.activated_on(rep_1h, homo_input)

		raw_loss = tf.nn.softmax_cross_entropy_with_logits(labels = homo_1h, logits = recon)
		aa_loss = tf.reduce_mean(tf.reduce_sum(raw_loss * seq_mask, 1) / tf.reduce_sum(seq_mask, 1), 0)

		tf.summary.scalar("aa_loss", aa_loss)

		log_p_x_given_z = tf.reduce_sum(raw_loss * seq_mask, 1)
		loss = tf.reduce_sum(log_p_x_given_z, 0) + kl_strength * tf.reduce_sum(kl_loss, 0)

		tf.summary.scalar("opt_loss", loss)
		tf.summary.scalar("true_loss_per_aa", (tf.reduce_sum(log_p_x_given_z) + tf.reduce_sum(kl_loss)) / tf.reduce_sum(seq_mask))

		return loss / tf.reduce_sum(seq_mask)

def end_training(sess, coord, threads):
	coord.request_stop()
	coord.join(threads)
	sess.close()

def homo_train(log_dir, log_subdir, do_restore = False, file_to_restore = None):
	tf.reset_default_graph()

	batch_size = 64
	max_prot_len = 243

	coord = tf.train.Coordinator()
	clust_size = 4   #this includes the "representative" sequence.

	reader = HomoProtReader("train_SHAR90_clu60f75_l243.fa", coord, batch_size, max_prot_len = max_prot_len, set_length = clust_size)
	cat_batch = reader.dequeue(batch_size)

	int_to_char = {0: "-", 1: "A", 2: "C", 3: "D", 4: "E", 5: "F", 6: "G", 7: "H", 8: "I", 9: "K", 10: "L", 11: "M", 12: "N",
					13: "P", 14: "Q", 15: "R", 16: "S", 17: "T", 18: "V", 19: "W", 20: "Y", 21: "*"}
	my_hash = tf.contrib.lookup.index_to_string_table_from_tensor(tf.constant([int_to_char[i] for i in range(22)]), default_value="?")
	#This is so you can see proteins and reconstructions, but Windows computers seem to have a hard time with it so maybe don't use.

	seq_len = max_prot_len

	with tf.variable_scope("model"):
		kl_strength = tf.get_variable("kl_strength", [], tf.float32, tf.constant_initializer(0.01), trainable = False)
		increase_kl = tf.assign_add(kl_strength, 1e-5)
		learning_rate = tf.get_variable("learning_rate", [], tf.float32, tf.constant_initializer(0.001), trainable = False)

		model = Homo_Model_243(batch_size, seq_len, my_hash, 512, 512, clust_size = clust_size, is_training = True)

		loss = model.loss(cat_batch, kl_strength)
		opt = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(loss)

	restorer = tf.train.Saver(tf.global_variables())
	saver = tf.train.Saver(tf.global_variables())

	summary_op = tf.summary.merge_all()
	sess = tf.Session()

	init = tf.global_variables_initializer()

	writer = tf.summary.FileWriter(log_dir + "/" + log_subdir + "/")
	writer.add_graph(tf.get_default_graph())
	run_metadata = tf.RunMetadata()

	tf_vars = tf.trainable_variables()
	total_params = 0
	for v in tf_vars:
		total_params += np.prod([int(xa) for xa in v.get_shape()])
	print("Total parameters: %i" %(total_params))

	my_hash.init.run(session = sess)

	threads = tf.train.start_queue_runners(sess = sess, coord = coord)
	reader.start_threads(sess)
	print("Started threads")

	if do_restore:
		restorer.restore(sess, file_to_restore)
	else:
		sess.run(init)
	print("Initialized variables.")

	begin = time.time()
	last_update = begin

	print("Finalizing graph.")
	tf.get_default_graph().finalize()
	steps_per_epoch = 1280000 // batch_size
	num_epochs = 20

	step = 0
	save_step = 2
	last_mean_aa = None

	try:
		for epoch in range(num_epochs):
			for iter in range(steps_per_epoch):
				fetches = [loss, opt, increase_kl]

				if step < 10 or step % 100 == 0:
					fetches += [summary_op]

				fetched = sess.run(fetches)

				mean_loss = fetched[0]
				if step < 10 or step % 100 == 0:
					print("Iteration %i, mean_loss is %.2f" %(step, mean_loss))
					writer.add_summary(fetched[-1], step)
					writer.flush()
				if step % 10000 == 0:
					saver.save(sess, log_dir + "/" + log_subdir + "/ckpt", global_step = step)
				step += 1
		print("Done training")
		saver.save(sess, log_dir + "/" + log_subdir + "/final_ckpt.ckpt")
		end_training(sess, coord, threads)

	except KeyboardInterrupt:
		print("Training interrupted, saving final checkpoint. Please wait a second.")
		saver.save(sess, log_dir + "/" + log_subdir + "/interrupt_%i_ckpt.ckpt" %(step))
		end_training(sess, coord, threads)
		print("Done saving checkpoint. Model should now quit.")

if __name__ == "__main__":
	log_dir, log_subdir = setup_log("train_vae_")
	homo_train(log_dir, log_subdir)

max_stars_repo_name: wanglabcumc/VariationalHomologEncoder

max_issues_repo_name: wanglabcumc/VariationalHomologEncoder

max_forks_repo_name: wanglabcumc/VariationalHomologEncoder


text:
#!/usr/bin/env python
"""
Preprocess high dimensional data into 2-dimensional data
for visualization using t-SNE. Saves to CSV file for input
into web visualizer.

Args:
    input_data: numpy file with dimensionality num_images x num_features
        where each row corresponds to an image in input_images_list
    input_images_list: text file with each line containing image file name
    input_images_dir: path to prepend to file names in input_images_list
"""

import argparse
import numpy as np
import random

from tsne import bh_sne

def save_to_csv(output_file, image_names, xs, ys):
    """Helper function to save data to CSV.

    Preconditions:
        length of image_names, xs and ys should be the same

    Args:
        output_file: path to output file
        image_names: list of image paths corresponding to coordinates
        xs: numpy array of x-coordinates
        ys: numpy array of y-coordinates
    """
    assert xs.shape[0] == ys.shape[0]
    assert len(image_names) == xs.shape[0]

    with open(output_file, 'w') as f:
        f.write("image_name,x,y\n") # header
        for name, x, y in zip(image_names, xs, ys):
            f.write("{0},{1},{2}\n".format(name, x, y))

def main():
    # parse arguments
    parser = argparse.ArgumentParser(description='Preprocess data using t-SNE.')
    parser.add_argument('input_data', type=str, help='Path to numpy data file')
    parser.add_argument('input_images_list', type=str, help='Path to text file listing images')
    parser.add_argument('--input_images_dir', type=str, default='public/data/', help='Path to images folder')
    parser.add_argument('--output_file', type=str, default='public/data.csv', help='Path to output CSV file')
    parser.add_argument('--max_num_points', type=int, default=1000, help='Max number of points')
    args = parser.parse_args()

    # load data
    data_load = np.load(args.input_data)
    with open(args.input_images_list, 'r') as f:
        image_names_load = [l.strip() for l in f]

    # shuffle and reduce number of data points to run faster
    # this also results in cleaner visualization
    assert len(image_names_load) > 0
    assert data_load.shape[0] == len(image_names_load)
    indices = range(data_load.shape[0])
    random.shuffle(indices)

    data = np.zeros((args.max_num_points, data_load.shape[1]))
    image_names = []
    for i, rand_index in enumerate(indices):
	if i >= args.max_num_points:
            break
        data[i,:] = data_load[rand_index,:]
        image_names.append(image_names_load[rand_index])

    assert data.shape[0] == len(image_names), '{0} and {1}'.format(data.shape[0], len(image_names))

    # run dimensionality reduction with t-SNE
    data_tsne = bh_sne(data)
    xs = data_tsne[:, 0]
    ys = data_tsne[:, 1]

    # save to csv file
    save_to_csv(args.output_file, image_names, xs, ys)

main()


text:
from unittest.mock import Mock

import numpy as np
from numpy.testing import assert_allclose
from pytest import raises

from menpo.transform import Transform

x = np.zeros([5, 5])

class MockTransform(Transform):
    def _apply(self, x, **kwargs):
        return np.array(x)

def test_transform_n_dims_return_None():
    tr = MockTransform()
    n_dims = tr.n_dims
    assert n_dims is None

def test_transform_n_dims_output_return_None():
    tr = MockTransform()
    n_dims = tr.n_dims_output
    assert n_dims is None

def test_transform_apply_x_not_transformable():
    tr = MockTransform()
    new_x = tr.apply(x)

    assert new_x is not x
    assert_allclose(new_x, x)

def test_transform_apply_x_transformable():
    mocked = Mock()
    mocked._transform.return_value = mocked
    tr = MockTransform()
    transformed_mock = tr.apply(mocked)

    assert transformed_mock is mocked
    assert mocked._transform.called

def test_transform_apply_inplace_x_transformable():
    mocked = Mock()
    mocked._transform_inplace.return_value = mocked
    tr = MockTransform()
    no_return = tr._apply_inplace(mocked)

    assert no_return is None
    assert mocked._transform_inplace.called

def test_transform_apply_inplace_x_not_transformable():
    tr = MockTransform()
    with raises(ValueError):
        tr._apply_inplace(x)

def test_transform_compose_before():
    mocked = Mock()
    mocked._apply.return_value = x
    tr = MockTransform()
    chain = tr.compose_before(mocked)
    # Check transform chain
    assert len(chain.transforms) == 2
    assert chain.transforms[0] is tr
    assert chain.transforms[1] is mocked

def test_transform_compose_after():
    mocked = Mock()
    mocked._apply.return_value = x
    tr = MockTransform()
    chain = tr.compose_after(mocked)
    # Check transform chain
    assert len(chain.transforms) == 2
    assert chain.transforms[0] is mocked
    assert chain.transforms[1] is tr

max_stars_repo_name: apapaion/menpo

max_issues_repo_name: apapaion/menpo

max_forks_repo_name: apapaion/menpo

reason: import numpy,from numpy


text:
import tensorflow as tf
import numpy as np
import os
import time

def get_timestamp(name):
    timestamp =time.asctime().replace(" ","_").replace(":","_")
    unique_name=f"{name}_at_{timestamp}"
    return unique_name

def get_callbacks(config,X_train):
    logs=config['logs']
    unique_dir_name=get_timestamp("tb_logs")
    TENSORBOARD_ROOT_LOG_DIR=os.path.join(logs['logs_dir'],logs["TENSORBOARD_ROOT_LOG_DIR"],unique_dir_name)

    os.makedirs(TENSORBOARD_ROOT_LOG_DIR,exist_ok=True)
    tensorboard_cb=tf.keras.callbacks.TensorBoard(log_dir=TENSORBOARD_ROOT_LOG_DIR)

    file_writer=tf.summary.create_file_writer(logdir=TENSORBOARD_ROOT_LOG_DIR)

    with file_writer.as_default():
        images=np.reshape(X_train[10:30],(-1,28,28,1))
        tf.summary.image("20 handwritten digit samples",images,max_outputs=25,step=0)

    params=config["params"]
    early_stopping_cb=tf.keras.callbacks.EarlyStopping(patience=params["patience"],restore_best_weights=params["restore_best_weights"])

    artifacts=config['artifacts']
    CKPT_dir=os.path.join(artifacts['artifacts_dir'],artifacts['CHECKPOINTS_DIR'])
    os.makedirs(CKPT_dir,exist_ok=True)

    CKPT_path=os.path.join(CKPT_dir,'model_ckpt.h5')
    checkpointing_cb=tf.keras.callbacks.ModelCheckpoint(CKPT_path,save_best_only=True)

    return [tensorboard_cb,early_stopping_cb,checkpointing_cb]