text:
# -*- coding: utf-8 -*-
'''
Created on Tue Aug  1 13:59:50 2017

@author: Flashcock
'''

import os
from scipy.stats import linregress as linreg
from itertools import takewhile
from numpy import array, poly1d, sqrt, absolute, arcsin, pi
from scipy.optimize import curve_fit, minimize_scalar
from scipy.integrate import quad
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
from mpl_toolkits.mplot3d import Axes3D
from numpy import log as ln

class specimen:
    '''
    Args should be formatted as follows:
        asuid (str or int), species (str), length (str), exo (str), endo (str),
        total (str), N (list [float]), mm (list [float]), Sec (list [float]).
    These are converted to their respective types below and used to calculate
        true and engineering stress and strain, young's modulus, and UTS.
    '''
    def __init__(self, asuid, species, length, exo, endo, total, N, mm, Sec):
        self.__asuid__ = asuid
        self.__species__ = species.lower()

        self.__length__ = float(length)
        self.__exo__ = float(exo)
        self.__endo__ = float(endo)
        self.__total__ = float(total)

        self.__N__ = array(N)
        self.__mm__ = array(mm) - min(mm)
        self.__Sec__ = array(Sec) - Sec[0]

        self.__sigmaN__ = self.__N__ / self.__total__ #P/A0
        self.__epsilonN__ = (self.__mm__ / self.__length__) #dL/L0
        self.__sigmaT__ = self.__sigmaN__ * (1 + self.__epsilonN__) #s(1+e)
        self.__epsilonT__ = ln(1 + self.__epsilonN__) #ln(1+e)

        self.__fmax__ = max(N)
        self.__uts__ = max(self.__sigmaN__) #evaluated from engineering stress

        self.__dmax__ = max(self.__mm__)
        self.__ufs__ = max(self.__epsilonN__) #evaluated from engineering strain

        self.__Eindex__ = []
        lower = self.__ufs__ * 0
        upper = self.__ufs__ * 0.33
        self.__Eindex__.append(len([x for x in takewhile(lambda x: x[1] <= lower, enumerate(self.__epsilonN__))]))
        self.__Eindex__.append(len([y for y in takewhile(lambda y: y[1] <= upper, enumerate(self.__epsilonN__))]))
        xvals = self.__epsilonN__[self.__Eindex__[0]:self.__Eindex__[1]]
        yvals = self.__sigmaN__[self.__Eindex__[0]:self.__Eindex__[1]]
        self.__E__ = linreg(xvals, yvals) #engineering stress and strain via Hooke's Law

        self.__Etanl__ = self.__E__[0]

        t_upper = 0
        t_lower = 0

        for i in range(len(self.__epsilonN__)-1):
            ub = self.__epsilonN__[i+1]
            lb = self.__epsilonN__[i]
            uh = self.__sigmaN__[i+1]
            lh = self.__sigmaN__[i]
            t_upper += (ub - lb) * uh
            t_lower += (ub - lb) * lh

        self.__U__ = (t_upper + t_lower) / 2

    def asuid(self):
        return self.__asuid__

    def species(self):
        return self.__species__

    def length(self):
        return self.__length__

    def exo(self):
        return self.__exo__

    def endo(self):
        return self.__endo__

    def total(self):
        return self.__total__

    def N(self):
        return self.__N__

    def mm(self):
        return self.__mm__

    def Sec(self):
        return self.__Sec__

    def sigmaN(self):
        return self.__sigmaN__

    def epsilonN(self):
        return self.__epsilonN__

    def sigmaT(self):
        return self.__sigmaT__

    def epsilonT(self):
        return self.__epsilonT__

    def Eindex(self, index):
        return self.__Eindex__[index]

    def recalcE(self, lower=0.00, upper=0.015, graphical=False, normal=False, limits='E_limits.txt'):
        self.__Eindex__ = []
        if normal:
            eps = self.__epsilonT__
            sig = self.__sigmaT__
        else:
            eps = self.__epsilonN__
            sig = self.__sigmaN__
        if graphical:
            file = open(limits, 'r')
            for line in file:
                asuid, species, lims = line[:-1].split('|', 2)
                if asuid == self.__asuid__:
                    x, y = lims.split(',')
                    lowerg = float(x[1:])
                    upperg = float(y[:-1])
                else:
                    pass
            self.__Eindex__.append(len([x for x in takewhile(lambda x: x[1] <= lowerg, enumerate(self.__Sec__))]))
            self.__Eindex__.append(len([y for y in takewhile(lambda y: y[1] <= upperg, enumerate(self.__Sec__))]))

        else:
            self.__Eindex__=[0, 1]
            low = False
            high = False
            while (not low) and (not high):
                for i, c in enumerate(self.__epsilonN__):
                    if (c >=lower) and (not low):
                        self.__Eindex__[0] = i
                        low = True
                    elif (c >= upper) and (not high):
                        self.__Eindex__[1] = i
                        high = True
                    else:
                        pass
        xvals = eps[self.__Eindex__[0] : self.__Eindex__[1]]
        yvals = sig[self.__Eindex__[0] : self.__Eindex__[1]]

        self.__E__ = linreg(xvals, yvals) #engineering stress and strain via Hooke's Law

    def E(self, index = 0):
        return self.__E__[index]

    def Etanl(self):
        return self.__Etanl__

    def fmax(self):
        return self.__fmax__

    def uts(self):
        return self.__uts__

    def dmax(self):
        return self.__dmax__

    def ufs(self):
        return self.__ufs__

    def U(self):
        return self.__U__

def as_si(x, ndp):
    s = '{x:0.{ndp:d}e}'.format(x=x, ndp=ndp)
    m, e = s.split('e')
    return r'{m:s}\times 10^{{{e:d}}}'.format(m=m, e=int(e))

def get_files(path='.\data'):
    '''
    Retrieves filenames in directory.
    '''
    filelist = os.listdir(path)
    return filelist

def spec_data(files, dims='imagedata.csv'):
    '''
    Returns a set of specimens.
    '''
    spec_set = set()

    file1 = open(dims, 'r')
    for line in file1:
        if line[0] == 'A':
            pass
        else:
            asuid, species, length, exo, endo, total = line[:-1].split(',', 5)
            N = []
            mm = []
            Sec = []
            normal_order = True
            for name in files:
                if name == '{}-{}-edit.dat'.format(asuid, species):
                    file2=open('.\data\\' + name,'r')
                    for line in file2:
                        if line[0] == 'S':
                            normal_order = False
                        elif line[0] in '1234567890':
                            x, y, z = line[:-1].split('\t', 2)
                            if normal_order:
                                N.append(float(x))
                                mm.append(float(y))
                                Sec.append(float(z))
                            else:
                                N.append(float(z))
                                mm.append(float(y))
                                Sec.append(float(x))
                        else:
                            pass
                    file2.close()
                    identifier, etc= name[:-9].split('-')
                    spec = specimen(asuid, species, length, exo, endo, total, N, mm, Sec)
                    spec_set.add(spec)
                else:
                    pass
    file1.close()
    return spec_set

def stress_strain(spec_set, save=True, i=0, j=-1):
    for spec in spec_set:
        ssplot, (ax1, ax3) = plt.subplots(2, sharex=False)

        ax1.plot(spec.Sec()[i:j], spec.N()[i:j], color='b')
        ax1.set_xlabel('Time (s)')
        ax1.set_ylabel('Axial Force (N)', color='b')
        ax1.tick_params('y', colors='b')
        ax1.set_title('Tensile Test Raw Data - ASUHIC_{} - Curculio {}'.format(spec.asuid(), spec.species()))

        ax2 = ax1.twinx()
        ax2.plot(spec.Sec()[i:j], spec.mm()[i:j], color='r')
        ax2.set_ylabel('Axial Displacement (mm)', color='r')
        ax2.tick_params('y', colors='r')

        # ax3.plot(spec.epsilonT()[i:j], spec.sigmaT()[i:j], color = 'gray')
        ax3.plot(spec.epsilonN()[i:j], spec.sigmaN()[i:j], color = 'black')
        ax3.set_ylabel('Stress (MPa)')
        ax3.set_xlabel('Strain (dL/L0)')

        print(spec.asuid()+' '+spec.species())
        new_l = spec.ufs() * 0.00
        new_u = spec.ufs() * 0.33
        spec.recalcE(lower = new_l, upper = new_u, graphical=False, normal=False)
        ax3.plot(spec.epsilonN()[spec.Eindex(0):spec.Eindex(1)],
                 spec.E()*spec.epsilonN()[spec.Eindex(0):spec.Eindex(1)] + spec.E(1),
                 color='r')

        ax3.plot([min(spec.epsilonN()), max(spec.epsilonN()[0:-1])],
                 [min(spec.sigmaN()), max(spec.sigmaN()[0:-1])],
                 color='gray')

        # true = mpatches.Patch(color='gray', label='True')
        eng = mpatches.Patch(color='black', label='Engineering')
        E1 = mpatches.Patch(color='r', label='E_low = {}MPa'.format(int(round(spec.Etanl()))))
        E2 = mpatches.Patch(color='gray', label='E_sec = {}MPa'.format(int(round(spec.uts() / spec.ufs()))))
        ax3.legend(handles=[eng, E1, E2], loc=2, fontsize = 6)
        #ax3.set_title('Stress vs. Nominal Strain')
        #ax3.set_xlim(0, 1)
        #ax3.set_ylim(0, 300)
        if save:
            ssplot.savefig('plots/rawdata/{}-{}.pdf'.format(spec.asuid(), spec.species()))
            #plt.close()
        else:
            plt.show()
            #plt.close()
            print('Plot not saved!!')

def xyplot(spec_set, save = True):
    spl = []
    regx = []
    regy = []

    fig = plt.figure()
    ax = fig.add_subplot(111)

    #plt.ylim(0,0.04)
    #plt.xlim(0,8)
    xlab = 'test (units)'
    ylab = 'test (units)'
    ax.set_xlabel(xlab)
    ax.set_ylabel(ylab)
    ax.set_title('{}_vs_{}.pdf'.format(xlab, ylab))

    car = mpatches.Patch(color='r', label='caryae')
    pro = mpatches.Patch(color='lightcoral', label='proboscideus')
    uni = mpatches.Patch(color='b', label='uniformis')
    hum = mpatches.Patch(color='skyblue', label='humeralis')
    sul = mpatches.Patch(color='dodgerblue', label='sulcatulus')
    vic = mpatches.Patch([], [], color='navy', label='victoriensis')
    ax.legend(handles=[car, pro, hum, sul, uni, vic], loc=4, fontsize = 7)

    for spec in spec_set:
        new_l = spec.ufs() * 0.67
        new_u = spec.ufs() * 1.00
        spec.recalcE(lower = new_l, upper = new_u, graphical=False, normal=False)
        x = ln(spec.length())
        y = ln(spec.Etanl())
        #spec.uts() / spec.ufs()

        if spec.species() == 'caryae':
            ax.plot(x, y, color='r', marker ='o')
        elif spec.species() == 'proboscideus':
            ax.plot(x, y, color='lightcoral', marker ='o')
        elif spec.species() == 'uniformis':
            ax.plot(x, y, color='b', marker ='o')
        elif spec.species() == 'humeralis':
            ax.plot(x, y, color='skyblue', marker ='o')
        elif spec.species() == 'sulcatulus':
            ax.plot(x, y, color='dodgerblue', marker ='o')
        elif spec.species() == 'victoriensis':
            ax.plot(x, y, color='navy', marker ='o')
        else:
            pass

        regx.append(x)
        regy.append(y)
        spl.append(str(x)+' '+str(y)+' '+spec.asuid()+' '+spec.species())

    x = array(regx)
    y = array(regy)
    regline = linreg(x, y)
    ax.plot(x, regline[0]*x+regline[1], color='black')
    ax.text(0.03,
            3,
            r'$y={:.3f}x+{:.3f}$'.format(regline[0], regline[1])
            +'\n'
            +r'$R={:.3f}$'.format(regline[2])
            +'\n'
            #+r'$p={:.3f}$'.format(regline[3]),
            +r'$p={0:s}$'.format(as_si(regline[3], 2)),
            fontsize=10)

    for  i in sorted(spl):
        print(i)

    plt.show()

    if save:
        fig.savefig('plots/xyplots/{}_vs_{}.pdf'.format(xlab, ylab))
        #plt.close()
    else:
        #plt.close()
        print('Plot not saved!!')

    print('m = {}'.format(regline[0]),
          'b = {}'.format(regline[1]),
          'r = {}'.format(regline[2]),
          'p = {}'.format(regline[3]),
          'std_err = {}'.format(regline[4]),
          sep='\n')

def xyzplot(spec_set):
    spl = []
    regx = []
    regy = []
    regz = []
    xyzfig = plt.figure()
    ax = Axes3D(xyzfig)
    for spec in spec_set:
        spec.recalcE(graphical=False, normal=False)
        x = (spec.length())
        y = (spec.U())
        z = (spec.exo() / spec.endo())
        if spec.species() == 'caryae':
            ax.scatter(x, y, z, c='r')
        elif spec.species() == 'proboscideus':
            ax.scatter(x, y, z, c='lightcoral')
        elif spec.species() == 'uniformis':
            ax.scatter(x, y, z, c='b')
        elif spec.species() == 'humeralis':
            ax.scatter(x, y, z, c='navy')
        elif spec.species() == 'sulcatulus':
            ax.scatter(x, y, z, c='dodgerblue')
        elif spec.species() == 'victoriensis':
            ax.scatter(x, y, z, c='skyblue')
        else:
            pass
        regx.append(x)
        regy.append(y)
        regz.append(z)
        spl.append(str(x)+' '+str(y)+' '+str(z)+' '+spec.asuid()+' '+spec.species())
    x = array(regx)
    y = array(regy)
    z = array(regz)
    ax.set_xlabel('Length')
    ax.set_ylabel('Strain')
    ax.set_zlabel('Stress')
    #regline = linreg(x, y, z)
    #plt.plot(x, regline[0]*x+regline[1], color='black')
    #plt.ylim(0,0.04)
    #plt.xlim(0,8)
    for  i in sorted(spl):
        print(i)
    plt.show()
    #print('m = {}'.format(regline[0]), 'b = {}'.format(regline[1]), 'r = {}'.format(regline[2]), 'p = {}'.format(regline[3]), 'std_err = {}'.format(regline[4]), sep='\n')

def write_data(specimens, filename = 'tensile.csv'):
    outfile=open(filename,'w')
    print('asuid,species,specimen,length,exo,endo,total,fmax,uts,dmax,ufs,Etanl,Etanh,Esec,Ut', file=outfile)
    humeralis = 0
    caryae = 0
    proboscideus = 0
    sulcatulus = 0
    uniformis = 0
    victoriensis = 0
    current_sp = 0
    for spec in specimens:
        if spec.species() == 'humeralis':
            humeralis += 1
            current_sp = humeralis
        elif spec.species() == 'caryae':
            caryae += 1
            current_sp = caryae
        elif spec.species() == 'proboscideus':
            proboscideus += 1
            current_sp = proboscideus
        elif spec.species() == 'sulcatulus':
            sulcatulus += 1
            current_sp = sulcatulus
        elif spec.species() == 'uniformis':
            uniformis += 1
            current_sp = uniformis
        elif spec.species() == 'victoriensis':
            victoriensis += 1
            current_sp = victoriensis
        else:
            print('unrecognized species')
        print(spec.asuid(),
              spec.species(),
              spec.species()+str(current_sp),
              spec.length(),
              spec.exo(),
              spec.endo(),
              spec.total(),
              spec.fmax(),
              spec.uts(),
              spec.dmax(),
              spec.ufs(),
              spec.Etanl(),
              spec.E(),
              spec.uts() / spec.ufs(),
              spec.U(),
              sep=',',
              file=outfile)
    outfile.close()

def main():
    specimens = spec_data(get_files())
    stress_strain(specimens, save=True)
    xyplot(specimens, save=False)
    #xyzplot(specimens)
    write_data(specimens)
    return specimens

testlib = main()

max_stars_repo_path: tex_v5/supplementary data/S1/tensile.py

max_stars_repo_name: entojansen/curculio-tensile

max_issues_repo_path: tex_v5/supplementary data/S1/tensile.py

max_issues_repo_name: entojansen/curculio-tensile

max_forks_repo_path: tex_v5/supplementary data/S1/tensile.py

max_forks_repo_name: entojansen/curculio-tensile


text:
from gen_itfc import loadGen, cloneSample
import matplotlib.pyplot as plt
import numpy as np

def qoiFn(u):
    mean = np.mean(u)
    qoi = np.mean((u-mean)**2)
    return qoi

generator = loadGen()
qoiVal = 2.5
nClone = 2
genData = cloneSample(generator,qoiVal,nClone)

print(genData.shape)
print(qoiFn(genData[0,:,0]))
print(qoiFn(genData[1,:,0]))
diff = abs(genData[1,:,0]-genData[0,:,0])
print("diff =",np.mean(diff))
print("diff std = ",np.linalg.norm(diff))
print("max diff = ",np.linalg.norm(genData[0,:,0]))

plt.plot(genData[0,:,0])
plt.plot(genData[1,:,0])
plt.show()

ext: py

max_stars_repo_name: NREL/GANISP

max_issues_repo_name: NREL/GANISP

max_forks_repo_name: NREL/GANISP

reason: import numpy


text:
# Licensed to Modin Development Team under one or more contributor license agreements.
# See the NOTICE file distributed with this work for additional information regarding
# copyright ownership.  The Modin Development Team licenses this file to you under the
# Apache License, Version 2.0 (the "License"); you may not use this file except in
# compliance with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under
# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific language
# governing permissions and limitations under the License.

import pytest
import numpy as np
import pandas
import matplotlib
from modin.config import MinPartitionSize
import modin.pandas as pd

from pandas.core.dtypes.common import is_list_like
from modin.pandas.test.utils import (
    random_state,
    df_equals,
    test_data_values,
    test_data_keys,
    query_func_keys,
    query_func_values,
    agg_func_keys,
    agg_func_values,
    agg_func_except_keys,
    agg_func_except_values,
    eval_general,
    create_test_dfs,
    udf_func_values,
    udf_func_keys,
    test_data,
    bool_arg_keys,
    bool_arg_values,
    arg_keys,
)
from modin.config import NPartitions, StorageFormat

NPartitions.put(4)

# Force matplotlib to not use any Xwindows backend.
matplotlib.use("Agg")

def test_agg_dict():
    md_df, pd_df = create_test_dfs(test_data_values[0])
    agg_dict = {pd_df.columns[0]: "sum", pd_df.columns[-1]: ("sum", "count")}
    eval_general(md_df, pd_df, lambda df: df.agg(agg_dict), raising_exceptions=True)

    agg_dict = {
        "new_col1": (pd_df.columns[0], "sum"),
        "new_col2": (pd_df.columns[-1], "count"),
    }
    eval_general(md_df, pd_df, lambda df: df.agg(**agg_dict), raising_exceptions=True)

@pytest.mark.parametrize("axis", [0, 1])
@pytest.mark.parametrize(
    "func",
    agg_func_values + agg_func_except_values,
    ids=agg_func_keys + agg_func_except_keys,
)
@pytest.mark.parametrize("op", ["agg", "apply"])
def test_agg_apply(axis, func, op):
    eval_general(
        *create_test_dfs(test_data["float_nan_data"]),
        lambda df: getattr(df, op)(func, axis),
    )

@pytest.mark.parametrize("axis", ["rows", "columns"])
@pytest.mark.parametrize(
    "func",
    agg_func_values + agg_func_except_values,
    ids=agg_func_keys + agg_func_except_keys,
)
@pytest.mark.parametrize("op", ["agg", "apply"])
def test_agg_apply_axis_names(axis, func, op):
    eval_general(
        *create_test_dfs(test_data["int_data"]),
        lambda df: getattr(df, op)(func, axis),
    )

def test_aggregate_alias():
    assert pd.DataFrame.agg == pd.DataFrame.aggregate

def test_aggregate_error_checking():
    modin_df = pd.DataFrame(test_data["float_nan_data"])

    with pytest.warns(UserWarning):
        modin_df.aggregate({modin_df.columns[0]: "sum", modin_df.columns[1]: "mean"})

    with pytest.warns(UserWarning):
        modin_df.aggregate("cumproduct")

    with pytest.raises(ValueError):
        modin_df.aggregate("NOT_EXISTS")

@pytest.mark.xfail(
    StorageFormat.get() == "Pandas",
    reason="DataFrame.apply(dict) raises an exception because of a bug in its"
    "implementation for pandas storage format, this prevents us from catching the desired"
    "exception. You can track this bug at:"
    "https://github.com/modin-project/modin/issues/3221",
)
@pytest.mark.parametrize(
    "func",
    agg_func_values + agg_func_except_values,
    ids=agg_func_keys + agg_func_except_keys,
)
def test_apply_key_error(func):
    if not (is_list_like(func) or callable(func) or isinstance(func, str)):
        pytest.xfail(
            reason="Because index materialization is expensive Modin first"
            "checks the validity of the function itself and only then the engine level"
            "checks the validity of the indices. Pandas order of such checks is reversed,"
            "so we get different errors when both (function and index) are invalid."
        )
    eval_general(
        *create_test_dfs(test_data["int_data"]),
        lambda df: df.apply({"row": func}, axis=1),
    )

@pytest.mark.parametrize("axis", [0, 1])
@pytest.mark.parametrize("level", [None, -1, 0, 1])
@pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
@pytest.mark.parametrize("func", ["kurt", "count", "sum", "mean", "all", "any"])
def test_apply_text_func_with_level(level, data, func, axis):
    func_kwargs = {"level": level, "axis": axis}
    rows_number = len(next(iter(data.values())))  # length of the first data column
    level_0 = np.random.choice([0, 1, 2], rows_number)
    level_1 = np.random.choice([3, 4, 5], rows_number)
    index = pd.MultiIndex.from_arrays([level_0, level_1])

    eval_general(
        pd.DataFrame(data, index=index),
        pandas.DataFrame(data, index=index),
        lambda df, *args, **kwargs: df.apply(func, *args, **kwargs),
        **func_kwargs,
    )

@pytest.mark.parametrize(
    "column", ["A", ["A", "C"]], ids=arg_keys("column", ["A", ["A", "C"]])
)
@pytest.mark.parametrize(
    "ignore_index", bool_arg_values, ids=arg_keys("ignore_index", bool_arg_keys)
)
def test_explode_single_partition(column, ignore_index):
    # This test data has two columns where some items are lists that
    # explode() should expand. In some rows, the columns have list-like
    # elements that must be expanded, and in others, they have empty lists
    # or items that aren't list-like at all.
    data = {
        "A": [[0, 1, 2], "foo", [], [3, 4]],
        "B": 1,
        "C": [["a", "b", "c"], np.nan, [], ["d", "e"]],
    }
    eval_general(
        *create_test_dfs(data),
        lambda df: df.explode(column, ignore_index=ignore_index),
    )

@pytest.mark.parametrize(
    "column", ["A", ["A", "C"]], ids=arg_keys("column", ["A", ["A", "C"]])
)
@pytest.mark.parametrize(
    "ignore_index", bool_arg_values, ids=arg_keys("ignore_index", bool_arg_keys)
)
def test_explode_all_partitions(column, ignore_index):
    # Test explode with enough rows to fill all partitions. explode should
    # expand every row in the input data into two rows. It's especially
    # important that the input data has list-like elements that must be
    # expanded at the boundaries of the partitions, e.g. at row 31.
    num_rows = NPartitions.get() * MinPartitionSize.get()
    data = {"A": [[3, 4]] * num_rows, "C": [["a", "b"]] * num_rows}
    eval_general(
        *create_test_dfs(data),
        lambda df: df.explode(column, ignore_index=ignore_index),
    )

@pytest.mark.parametrize("axis", ["rows", "columns"])
@pytest.mark.parametrize("args", [(1,), ("_A",)])
def test_apply_args(axis, args):
    def apply_func(series, y):
        try:
            return series + y
        except TypeError:
            return series.map(str) + str(y)

    eval_general(
        *create_test_dfs(test_data["int_data"]),
        lambda df: df.apply(apply_func, axis=axis, args=args),
    )

def test_apply_metadata():
    def add(a, b, c):
        return a + b + c

    data = {"A": [1, 2, 3], "B": [4, 5, 6], "C": [7, 8, 9]}

    modin_df = pd.DataFrame(data)
    modin_df["add"] = modin_df.apply(
        lambda row: add(row["A"], row["B"], row["C"]), axis=1
    )

    pandas_df = pandas.DataFrame(data)
    pandas_df["add"] = pandas_df.apply(
        lambda row: add(row["A"], row["B"], row["C"]), axis=1
    )
    df_equals(modin_df, pandas_df)

@pytest.mark.parametrize("func", udf_func_values, ids=udf_func_keys)
@pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
def test_apply_udf(data, func):
    eval_general(
        *create_test_dfs(data),
        lambda df, *args, **kwargs: df.apply(*args, **kwargs),
        func=func,
        other=lambda df: df,
    )

def test_eval_df_use_case():
    frame_data = {"a": random_state.randn(10), "b": random_state.randn(10)}
    df = pandas.DataFrame(frame_data)
    modin_df = pd.DataFrame(frame_data)

    # test eval for series results
    tmp_pandas = df.eval("arctan2(sin(a), b)", engine="python", parser="pandas")
    tmp_modin = modin_df.eval("arctan2(sin(a), b)", engine="python", parser="pandas")

    assert isinstance(tmp_modin, pd.Series)
    df_equals(tmp_modin, tmp_pandas)

    # Test not inplace assignments
    tmp_pandas = df.eval("e = arctan2(sin(a), b)", engine="python", parser="pandas")
    tmp_modin = modin_df.eval(
        "e = arctan2(sin(a), b)", engine="python", parser="pandas"
    )
    df_equals(tmp_modin, tmp_pandas)

    # Test inplace assignments
    df.eval("e = arctan2(sin(a), b)", engine="python", parser="pandas", inplace=True)
    modin_df.eval(
        "e = arctan2(sin(a), b)", engine="python", parser="pandas", inplace=True
    )
    # TODO: Use a series equality validator.
    df_equals(modin_df, df)

def test_eval_df_arithmetic_subexpression():
    frame_data = {"a": random_state.randn(10), "b": random_state.randn(10)}
    df = pandas.DataFrame(frame_data)
    modin_df = pd.DataFrame(frame_data)
    df.eval("not_e = sin(a + b)", engine="python", parser="pandas", inplace=True)
    modin_df.eval("not_e = sin(a + b)", engine="python", parser="pandas", inplace=True)
    # TODO: Use a series equality validator.
    df_equals(modin_df, df)

TEST_VAR = 2

@pytest.mark.parametrize("method", ["query", "eval"])
@pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
@pytest.mark.parametrize("local_var", [2])
def test_eval_and_query_with_local_and_global_var(method, data, local_var):
    modin_df, pandas_df = pd.DataFrame(data), pandas.DataFrame(data)
    op = "+" if method == "eval" else "<"
    for expr in (f"col1 {op} @local_var", f"col1 {op} @TEST_VAR"):
        df_equals(getattr(modin_df, method)(expr), getattr(pandas_df, method)(expr))

@pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
def test_filter(data):
    modin_df = pd.DataFrame(data)
    pandas_df = pandas.DataFrame(data)

    by = {"items": ["col1", "col5"], "regex": "4$|3$", "like": "col"}
    df_equals(modin_df.filter(items=by["items"]), pandas_df.filter(items=by["items"]))

    df_equals(
        modin_df.filter(regex=by["regex"], axis=0),
        pandas_df.filter(regex=by["regex"], axis=0),
    )
    df_equals(
        modin_df.filter(regex=by["regex"], axis=1),
        pandas_df.filter(regex=by["regex"], axis=1),
    )

    df_equals(modin_df.filter(like=by["like"]), pandas_df.filter(like=by["like"]))

    with pytest.raises(TypeError):
        modin_df.filter(items=by["items"], regex=by["regex"])

    with pytest.raises(TypeError):
        modin_df.filter()

@pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
def test_pipe(data):
    modin_df = pd.DataFrame(data)
    pandas_df = pandas.DataFrame(data)

    n = len(modin_df.index)
    a, b, c = 2 % n, 0, 3 % n
    col = modin_df.columns[3 % len(modin_df.columns)]

    def h(x):
        return x.drop(columns=[col])

    def g(x, arg1=0):
        for _ in range(arg1):
            x = x.append(x)
        return x

    def f(x, arg2=0, arg3=0):
        return x.drop([arg2, arg3])

    df_equals(
        f(g(h(modin_df), arg1=a), arg2=b, arg3=c),
        (modin_df.pipe(h).pipe(g, arg1=a).pipe(f, arg2=b, arg3=c)),
    )
    df_equals(
        (modin_df.pipe(h).pipe(g, arg1=a).pipe(f, arg2=b, arg3=c)),
        (pandas_df.pipe(h).pipe(g, arg1=a).pipe(f, arg2=b, arg3=c)),
    )

@pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
@pytest.mark.parametrize("funcs", query_func_values, ids=query_func_keys)
def test_query(data, funcs):
    modin_df = pd.DataFrame(data)
    pandas_df = pandas.DataFrame(data)

    try:
        pandas_result = pandas_df.query(funcs)
    except Exception as e:
        with pytest.raises(type(e)):
            modin_df.query(funcs)
    else:
        modin_result = modin_df.query(funcs)
        df_equals(modin_result, pandas_result)

def test_empty_query():
    modin_df = pd.DataFrame([1, 2, 3, 4, 5])

    with pytest.raises(ValueError):
        modin_df.query("")

def test_query_after_insert():
    modin_df = pd.DataFrame({"x": [-1, 0, 1, None], "y": [1, 2, None, 3]})
    modin_df["z"] = modin_df.eval("x / y")
    modin_df = modin_df.query("z >= 0")
    modin_result = modin_df.reset_index(drop=True)
    modin_result.columns = ["a", "b", "c"]

    pandas_df = pd.DataFrame({"x": [-1, 0, 1, None], "y": [1, 2, None, 3]})
    pandas_df["z"] = pandas_df.eval("x / y")
    pandas_df = pandas_df.query("z >= 0")
    pandas_result = pandas_df.reset_index(drop=True)
    pandas_result.columns = ["a", "b", "c"]

    df_equals(modin_result, pandas_result)
    df_equals(modin_df, pandas_df)

@pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
@pytest.mark.parametrize(
    "func",
    agg_func_values + agg_func_except_values,
    ids=agg_func_keys + agg_func_except_keys,
)
def test_transform(data, func):
    eval_general(
        *create_test_dfs(data),
        lambda df: df.transform(func),
    )

max_stars_repo_name: Rubtsowa/modin

max_issues_repo_name: Rubtsowa/modin

max_forks_repo_name: Rubtsowa/modin


text:
import cv2
import numpy as np

B2_CAMERA_CALIBRATION = load_npy_camera_calibration_file

def cam_to_world(cam_vector):
    assert cam_vector.shape[0] == 3

    tvec = B2_CAMERA_CALIBRATION['t']

    rotation_mtx = cv2.Rodrigues(B2_CAMERA_CALIBRATION['r'])[0]
    invert_rotation_mtx = np.linalg.inv(rotation_mtx)

    return invert_rotation_mtx @ (cam_vector - tvec)

def compute_zc(uv, z=0):
    assert uv.shape[0] == 3

    rotation_mtx = cv2.Rodrigues(B2_CAMERA_CALIBRATION['r'])[0]
    cross = np.cross(rotation_mtx[:, 0], rotation_mtx[:, 1]).reshape(3, 1)
    det = np.linalg.det(rotation_mtx)

    tvec = B2_CAMERA_CALIBRATION['t']

    intrinsic_mtx = B2_CAMERA_CALIBRATION['Intrinsic']
    invert_intrinsic_mtx = np.linalg.inv(intrinsic_mtx)

    return (np.dot(tvec.T, cross) + z * det) / np.dot(cross.T, invert_intrinsic_mtx @ uv)

def get_undistorted_points(coord):
    if coord.shape[0] == 3:
        coord = coord[:2, :]
    coord = coord.astype(np.float32)

    upt = cv2.undistortPoints(coord, B2_CAMERA_CALIBRATION['Intrinsic'], B2_CAMERA_CALIBRATION['dist_coeff'], None,
                              None, B2_CAMERA_CALIBRATION['Intrinsic'])
    return np.append(upt.reshape(-1, 2), np.ones([upt.shape[0], 1]), axis=1).T

def _2D_to_planar_world(uv, z=0):
    assert uv.shape[0] in [2, 3]

    uv = get_undistorted_points(uv)

    intrinsic_mtx = B2_CAMERA_CALIBRATION['Intrinsic']
    invert_intrinsic_mtx = np.linalg.inv(intrinsic_mtx)

    zc = compute_zc(uv, z)

    uvz = uv * zc

    cam_vector = invert_intrinsic_mtx @ uvz

    return cam_to_world(cam_vector)

max_stars_repo_name: vbruttin/Master-Thesis-Pipelines

max_issues_repo_name: vbruttin/Master-Thesis-Pipelines

max_forks_repo_name: vbruttin/Master-Thesis-Pipelines


text:
import pytest
from collections import OrderedDict
import numpy as np

def create_eam_r_array(r_max,N_r):
    _r = r_max*np.linspace(1,N_r,N_r)/N_r
    return _r

def create_eam_rho_array(rho_max,N_rho):
    _rho = rho_max*np.linspace(1,N_rho,N_rho)/N/rho
    return _rho

def calculate_equilibrium_volume_per_atom(lattice_info):
    for k,v in lattice_info.items():
        if v['lattice_type'] == 'fcc':
            _a0 = v['lattice_parameter']
            _V = _a0**3

        v['volume_per_atom'] = _V

def calculate_equilibrium_interatomic_distance(lattice_info):
    for k,v in lattice_info.items():
        if v['lattice_type'] == 'fcc':
            _a0 = v['lattice_parameter']
            _re = 1/(2**0.5)*_a0

        v['equilbrium_interatomic_distance'] = _re

def convert_gpa_to_atom_units(pressure_in_gpa):
    _kg_to_g_per_mole = 1. #??
    _m_to_angs = 1.e10
    _s_to_ps = 1.e12
    # GPa in SI: N m^-2 == kg m s^-2 m^-2 = kg s^-2 m^-1
    P = pressure_in_gpa * 1.e9 # kg/m/s/s
    P = P * _kg_to_g_per_mole

from pypospack.potentials.eam_eos_foiles import func_eam_embed_foiles

from pypospack.potential import EamPotential

from pypospack.potential import BornMayerPotential
from pypospack.potential import ExponentialDensityFunction
from pypospack.potential import RoseEquationOfStateEmbeddingFunction

symbols = ['Ni']

func_pair_name = 'bornmayer'
func_density_name = 'eam_dens_exp'
func_embedding_name = 'eam_embed_eos_rose'

lattice_info = OrderedDict()
lattice_info['Ni'] = OrderedDict()
lattice_info['Ni']['equlibrium_interatomic_distance']
lattice_info['Ni']['cohesive_energy']
lattice_info['Ni']['bulk_modulus']
lattice_info['Ni']['lattice_parameter']
lattice_info['Ni']['bulk_modulus']

parameters = OrderedDict()
parameters['p_NiNi_phi0'] = 1.0
parameters['p_NiNi_gamma'] = 2.0
parameters['p_NiNi_r0'] = lattice_info['Ni']['equilibrium_interatomic_distance']
parameters['d_Ni_rho0'] = 1.0
parameters['d_Ni_beta'] = 4.0
parameters['d_Ni_r0'] = lattice_info['Ni']['equilibrium_interatomic_distance']
parameters['e_Ni_ecoh'] = lattice_info['Ni']['cohesive_energy']
parameters['e_Ni_latticetype'] = lattice_info['Ni']['lattice_type']
parameters['e_Ni_B'] = lattice_info['Ni']['bulk_modulus']
parameters['e_Ni_a0'] = lattice_info['Ni']['lattice_parameter']

r_max = 10.
N_r = 1000

rho_max = 100000.
N_rho = 100
r = create_eam_r_array(r_max,N_r)
rho = create_eam_rho_array(rho_max,N_rho)

parameter_names_pair = ['p_NiNi_D0', 'p_NiNi_a', 'p_NiNi_r0']
parameter_names_density = ['d_Ni_rho0', 'd_Ni_beta', 'd_Ni_r0']
parameter_names_embedding = ['e_Ni_ecoh', 'e_Ni_latticetype', 'e_Ni_B', 'e_Ni_a0']
parameter_names = \
        parameter_names_pair\
        + parameter_names_density\
        + parameter_names_embedding

lattice_info = OrderedDict()
lattice_info['Ni'] = OrderedDict()
lattice_info['Ni']['lattice_type'] = 'fcc'
lattice_info['Ni']['cohesive_energy'] = -4.5
lattice_info['Ni']['bulk_modulus'] = 162
lattice_info['Ni']['lattice_parameter'] = 3.52

calculate_equilibrium_volume_per_atom(lattice_info)
calculate_equilibrium_interatomic_distance(lattice_info)

def test__init__():

    pot = EamPotential(symbols=symbols,
            func_pair=func_pair_name,
            func_density=func_density_name,
            func_embedding=func_embedding_name)

    assert pot.potential_type == 'eam'
    assert pot.is_charge is False

    assert pot.symbols == symbols

    assert type(pot.obj_pair) is BornMayerPotential
    assert type(pot.obj_embedding) is RoseEquationOfStateEmbeddingFunction
    assert type(pot.obj_density) is ExponentialDensityFunction

    assert pot.obj_pair.parameter_names == parameter_names_pair
    assert pot.obj_embedding.parameter_names == parameter_names_embedding
    assert pot.obj_density.parameter_names == parameter_names_density

def test__evaluate():

    pot = EamPotential(symbols=symbols,
            func_pair=func_pair_name,
            func_density=func_density_name,
            func_embedding=func_embedding_name)
    pot.evaluate(
            r = r,
            rho = rho,
            rcut = r_max,
            parameters = parameters)

max_stars_repo_name: eragasa/pypospack

max_issues_repo_name: eragasa/pypospack

max_forks_repo_name: eragasa/pypospack


text:
import sys
import numpy as np
import pandas as pd
from PyQt5.QtCore import *
from PyQt5.QtGui import *
from PyQt5.QtWidgets import *
from PyQt5.QtWidgets import QMessageBox

from fotf import *
from fotf import MAX_LAMBDA,MIN_COEF,MAX_COEF,MIN_EXPO,MAX_EXPO
from matplotlib import pyplot as plt
from pyGui import loaddatagui, fotftidgui, trimdatagui
from pyGui.fomconoptimizegui import *

#region Constants
STATUSBAR_TIME = 7000

#endregion

class loadDataClass(QDialog, loaddatagui.Ui_LoadDataForm):
    def __init__(self):
        QDialog.__init__(self)
        loaddatagui.Ui_LoadDataForm.__init__(self)
        self.setupUi(self)
        self.setWindowIcon(QIcon('index.png'))
        self.pushButtonBrowse.clicked.connect(self._browse)
        self.pushButtonOK.clicked.connect(self.close)
        self.pushButtonCancel.clicked.connect(self.close)
        self.lineEditDataPath.textChanged.connect(self._checkText)
        self.lineEditSysName.textChanged.connect(self._checkText)
        self.show()

    def _browse(self):
        fname = QFileDialog.getOpenFileName(self, 'Open Data', QDir.currentPath(), "Excel Files (*.xls *.xlsx)")
        self.lineEditDataPath.setText(fname[0])
        # self._checkText()

    def _checkText(self):
        if (self.lineEditSysName.text() is not (None or "")) and self.lineEditDataPath.text() is not (None or ""):
            self.pushButtonOK.setEnabled(True)
        else:
            self.pushButtonOK.setEnabled(False)

    def closeEvent(self, event):
        sender = self.sender().text()

        close = QMessageBox.question(self, "{0}?".format(sender),
                                     "Are you sure you would like to '{0}' this form?".format(sender),
                                     QMessageBox.Yes | QMessageBox.No)
        if close == QMessageBox.Yes:
            if sender == self.pushButtonOK.text():
                pass
            else:
                self.lineEditSysName.clear()
                self.lineEditDataPath.clear()
            # event.accept()
        else:
            event.ignore()

class trimDataClass(QDialog, trimdatagui.Ui_TrimDataForm):
    def __init__(self):
        QDialog.__init__(self)
        trimdatagui.Ui_TrimDataForm.__init__(self)
        self.setupUi(self)
        self.setWindowIcon(QIcon('index.png'))
        self.lineEditT2.textChanged.connect(self.editedT2)
        self.lineEditT1.textChanged.connect(self.editedT1)
        self.lineEditNewName.textChanged.connect(self.editedName)
        self.pushButtonOK.clicked.connect(self.close)
        self.pushButtonCancel.clicked.connect(self.close)
        self.t1edited = False
        self.t2edited = False
        self.nameEdited = False
        self.initialt1 = -1
        self.initialt2 = -1
        self.initialName = ""

        self.show()

    def editedT1(self):
        try:
            if self.initialt1 <= float(self.lineEditT1.text()) < self.initialt2:
                if float(self.lineEditT1.text()) < float(self.lineEditT2.text()):
                    self.t1edited = True
                else:
                    self.t1edited = False
            else:
                self.t1edited = False
            self.edited()
        except:
            self.t1edited = False

    def editedT2(self):
        try:
            if self.initialt1 < float(self.lineEditT2.text()) <= self.initialt2:
                if float(self.lineEditT1.text()) < float(self.lineEditT2.text()):
                    self.t2edited = True
                else:
                    self.t2edited = False
            else:
                self.t2edited = False
            self.edited()
        except:
            self.t2edited = False

    def editedName(self):
        try:
            if self.initialName == self.lineEditNewName.text():
                self.nameEdited = False
            else:
                self.nameEdited = True
            self.edited()
        except:
            self.nameEdited = False

    def edited(self):
        if self.nameEdited and self.t1edited and self.t2edited:
            self.pushButtonOK.setEnabled(True)
        else:
            self.pushButtonOK.setEnabled(False)

    def closeEvent(self, event):
        sender = self.sender().text()

        close = QMessageBox.question(self, "{0}?".format(sender), "Are you sure you would like to '{0}' this form?".format(sender),
                                     QMessageBox.Yes | QMessageBox.No)
        if close == QMessageBox.Yes:
            if sender == self.pushButtonOK.text():
                pass
            else:
                self.lineEditNewName.clear()
                self.lineEditT1.clear()
                self.lineEditT2.clear()
            event.accept()
        else:
            event.ignore()

class fotftidguiclass(QMainWindow, fotftidgui.Ui_MainWindow_fotftid):
    def __init__(self):
        QMainWindow.__init__(self)
        fotftidgui.Ui_MainWindow_fotftid.__init__(self)
        self.setupUi(self)
        self.setWindowIcon(QIcon('index.png'))
        self.IdentifiedModel = None

        self.comboBoxData.currentIndexChanged.connect(self._comboBoxDataEmpty)
        self.pushButtonGeneratGuess.clicked.connect(self._GeneratePolynomials)
        self.pushButtonAddData.clicked.connect(self._addData)
        self.pushButtonDeleteData.clicked.connect(self._deleteData)
        self.pushButtonPlotData.clicked.connect(self._plot)
        self.pushButtonTrimData.clicked.connect(self._trim)
        self.checkBoxFixZeros.clicked.connect(self._fixedZero)
        self.checkBoxFixPoles.clicked.connect(self._fixedPole)
        self.checkBoxUseDelay.clicked.connect(self._useDelay)
        self.checkBoxUseCoefLimits.clicked.connect(self._useCoeffLim)
        self.checkBoxUseExpoLimits.clicked.connect(self._useExpoLim)
        self.pushButtonModelStability.clicked.connect(self._stabilityCheck)
        self.pushButtonRoundOff.clicked.connect(self._roundOff)
        self.pushButtonModelValidate.clicked.connect(self._validate)
        self.pushButtonIdentify.clicked.connect(self._identify)
        self.pushButton_GetFOFOPDT.clicked.connect(self._GetFOFOPDT)
        self.lineEditCommesurateOder.textChanged.connect(self._qChanged)
        self.lineEditFOTFOrder.textChanged.connect(self._fotfOrderChanged)
        self.lineEditCoefLimitLower.textChanged.connect(self._coeffLowerChanger)
        self.lineEditCoefLimitUpper.textChanged.connect(self._coeffUpperChanged)
        self.lineEditExpLimitLower.textChanged.connect(self._expLowerChanged)
        self.lineEditExpLimitUpper.textChanged.connect(self._expUpperChanged)
        self.lineEditOrder.textChanged.connect(self._OustaOrderChanged)
        self.lineEdit_StartFreq.textChanged.connect(self._OustaStartFreqChanged)
        self.lineEdit_StopFreq.textChanged.connect(self._OustaStopFreqChanged)
        self.lineEditLamda.textChanged.connect(self._LamdaChanged)
        self.textEdit_Zeros.textChanged.connect(self._LamdaChanged)
        self.textEdit_Poles.textChanged.connect(self._LamdaChanged)
        self.comboBoxOptTypeMethod.currentIndexChanged.connect(self._oustaloopSelected)
        self._isexpUpperok = self._isexpLowerok = self._iscoeffUpperok = self._iscoeffLowerok = True
        self._isOustaStartFreq = self._isOustaStopFreqOK = self._isOustaOrderOk = self._isLamdaOk = True
        self._isqok = self._isnOk = True
        self._reloadAllFOTransFunc()
        self.show()

    #region Button Checks
    def _oustaloopSelected(self):
        if self.comboBoxOptTypeMethod.currentData() is simMethod.grunwaldLetnikov:
            self.lineEdit_StartFreq.setEnabled(False)
            self.lineEdit_StopFreq.setEnabled(False)
            self.lineEditOrder.setEnabled(False)
            self._isOustaStartFreq = self._isOustaStopFreqOK = self._isOustaOrderOk = True
            try:
                self._ok2Identify()
            except:
                pass
        else:
            self.lineEdit_StartFreq.setEnabled(True)
            self.lineEdit_StopFreq.setEnabled(True)
            self.lineEditOrder.setEnabled(True)
            try:
                self._ok2Identify()
            except:
                pass

    def _LamdaChanged(self):
        try:
            x = int(self.lineEditLamda.text())
            z = self.textEdit_Zeros.toPlainText()
            p = self.textEdit_Poles.toPlainText()

            if 0 < x <= MAX_LAMBDA and p != "" and z !=  "":
                self._isLamdaOk = True
                self.pushButtonRoundOff.setEnabled(True)
                self.pushButtonModelStability.setEnabled(True)
            else:
                self._isLamdaOk = False
                self.pushButtonRoundOff.setEnabled(False)
                self.pushButtonModelStability.setEnabled(False)
            self._ok2Identify()
            self._ok2GetFOFOPDT()

        except:
            self._isLamdaOk = False
            self.pushButtonRoundOff.setEnabled(False)
            self.pushButtonModelStability.setEnabled(False)

    def _OustaStartFreqChanged(self):
        try:
            x = int(self.lineEdit_StartFreq.text())
            if x < 0:
                self._isOustaStartFreq = True
            else:
                self._isOustaStartFreq = False
            self._ok2Identify()
        except:
            self._isOustaStartFreq = False

    def _OustaStopFreqChanged(self):
        try:
            x = int(self.lineEdit_StopFreq.text())
            if x > 0:
                self._isOustaStopFreqOK = True
            else:
                self._isOustaStopFreqOK = False
            self._ok2Identify()
        except:
            self._isOustaStopFreqOK = False

    def _OustaOrderChanged(self):
        try:
            x = int(self.lineEditOrder.text())
            if x > 0:
                self._isOustaOrderOk = True
            else:
                self._isOustaOrderOk = False
            self._ok2Identify()
        except:
            self._isOustaOrderOk = False

    def _qChanged(self):
        try:
            x = float(self.lineEditCommesurateOder.text())
            if 0.01 <= x <= 2:
                self._isqok = True
            else:
                self._isqok = False
            self._generateOk()
        except:
            self._isqok = False

    def _fotfOrderChanged(self):
        try:
            x = int(self.lineEditFOTFOrder.text())
            if 1 <= x <= 10:
                self._isnOk = True
            else:
                self._isnOk = False
            self._generateOk()
        except:
            self._isnOk = False

    def _coeffLowerChanger(self):
        try:
            lower = int(self.lineEditCoefLimitLower.text())
            higher = int(self.lineEditCoefLimitUpper.text())
            if MIN_COEF <= lower <= MAX_COEF and lower < higher:
                self._iscoeffLowerok = True
            else:
                self._iscoeffLowerok = False
            self._ok2Identify()
        except:
            self._iscoeffLowerok = False

    def _coeffUpperChanged(self):
        try:
            lower = int(self.lineEditCoefLimitLower.text())
            higher = int(self.lineEditCoefLimitUpper.text())
            if MIN_COEF <= higher <= MAX_COEF and lower < higher:
                self._iscoeffUpperok = True
            else:
                self._iscoeffUpperok = False
            self._ok2Identify()
        except:
            self._iscoeffUpperok = False

    def _expLowerChanged(self):
        try:
            lower = int(self.lineEditExpLimitLower.text())
            higher = int(self.lineEditExpLimitUpper.text())
            if MIN_EXPO <= lower <= MAX_EXPO and lower < higher:
                self._isexpLowerok = True
            else:
                self._isexpLowerok = False
            self._ok2Identify()
        except:
            self._isexpLowerok = False

    def _expUpperChanged(self):
        try:
            lower = int(self.lineEditExpLimitLower.text())
            higher = int(self.lineEditExpLimitUpper.text())
            if MIN_EXPO <= higher <= MAX_EXPO and lower < higher:
                self._isexpUpperok = True
            else:
                self._isexpUpperok = False
            self._ok2Identify()
        except:
            self._isexpUpperok = False

    def _generateOk(self):
        try:
            if self._isqok and self._isnOk:
                self.pushButtonGeneratGuess.setEnabled(True)
            else:
                self.pushButtonGeneratGuess.setEnabled(False)
        except:
            self.pushButtonGeneratGuess.setEnabled(False)

    def _ok2Identify(self):
        x = self.comboBoxData.count() > 0
        try:
            if x and self._iscoeffLowerok and self._iscoeffUpperok and self._isexpLowerok and self._isexpUpperok \
                    and self._isOustaStartFreq and self._isOustaStopFreqOK and self._isOustaOrderOk and self._isLamdaOk:
                self.pushButtonIdentify.setEnabled(True)
            else:
                self.pushButtonIdentify.setEnabled(False)
        except:
            self.pushButtonIdentify.setEnabled(False)

    def _ok2GetFOFOPDT(self):
        identifiedSytem = None
        isstabledata = False
        epsi = int(self.lineEditLamda.text())
        try:
            if epsi > 3:
                epsi = 3
            if self.checkBoxUseDelay.isChecked():
                _dt = float(self.lineEdit_Delay.text())
            else:
                _dt = 0

            if self.IdentifiedModel is None:
                _zero = self.textEdit_Zeros.toPlainText()
                _poles = self.textEdit_Poles.toPlainText()
                # semizero = str2poly(_zero)
                # semipole = str2poly(_poles)
                #
                # newZero = poly2str(semizero[0], semizero[1], eps=epsi)
                # newPole = poly2str(semipole[0], semipole[1], eps=epsi)
                identifiedSytem = newfotf(_zero, _poles, _dt)
                identifiedSytem.numberOfDecimal = epsi

            elif isinstance(self.IdentifiedModel, FOTransFunc):
                self.IdentifiedModel.numberOfDecimal = epsi
                identifiedSytem = self.IdentifiedModel

            num,nnum,den, nden,dt = fotfparam(identifiedSytem)
            if nnum.size ==1 and nnum[-1] <= 10**-MAX_EXPO and nden.size == 2 and nden[-1] <= 10**-MAX_EXPO:# and self.pushButtonIdentify.isEnabled():
                self.pushButton_GetFOFOPDT.setEnabled(True)
            else:
                self.pushButton_GetFOFOPDT.setEnabled(False)
        except:
            pass
    #endregion

    #region Checkboxes event handlers
    def _fixedZero(self):
        if self.checkBoxFixZeros.isChecked():
            self.textEdit_Zeros.setEnabled(False)
        else:
            self.textEdit_Zeros.setEnabled(True)

    def _fixedPole(self):
        if self.checkBoxFixPoles.isChecked():
            self.textEdit_Poles.setEnabled(False)
        else:
            self.textEdit_Poles.setEnabled(True)

    def _useDelay(self):
        if self.checkBoxUseDelay.isChecked():
            self.lineEdit_Delay.setEnabled(True)
        else:
            self.lineEdit_Delay.setEnabled(False)

    def _useCoeffLim(self):
        if self.checkBoxUseCoefLimits.isChecked():
            self.lineEditCoefLimitLower.setEnabled(True)
            self.lineEditCoefLimitUpper.setEnabled(True)
        else:
            self.lineEditCoefLimitLower.setEnabled(False)
            self.lineEditCoefLimitUpper.setEnabled(False)

    def _useExpoLim(self):
        if self.checkBoxUseExpoLimits.isChecked():
            self.lineEditExpLimitLower.setEnabled(True)
            self.lineEditExpLimitUpper.setEnabled(True)
        else:
            self.lineEditExpLimitLower.setEnabled(False)
            self.lineEditExpLimitUpper.setEnabled(False)
    #endregion

    #region Initialization Functions
    def _reloadAllFOTransFunc(self):
        # Startup Config
        fix = {"Free Identification": optFix.Free, "Fix Coefficient": optFix.Coeff, "Fix Exponents": optFix.Exp}
        algo = {"Levenberg Marquardt": optAlgo.LevenbergMarquardt,
                "Trust Region Reflective": optAlgo.TrustRegionReflective,
                "Cauchy Robust Loss": optAlgo.RobustLoss,
                "softl1 Robust Loss": optAlgo.Softl1}
        method = {"Grunwald Letnikov": simMethod.grunwaldLetnikov, "Oustaloop": simMethod.oustaloop}
        # method = {"Grunwald Letnikov": simMethod.grunwaldLetnikov}
        for i in fix:
            self.comboBoxOptFix.addItem(i, fix[i])

        for i in algo:
            self.comboBoxAlgorithm.addItem(i, algo[i])

        for i in method:
            self.comboBoxOptTypeMethod.addItem(i, method[i])

        self.comboBoxPolesOrZeros.addItems(['Zero Polynomial', 'Pole Polynomial'])
        self.comboBoxPolesOrZeros.setCurrentIndex(1)
        self._GeneratePolynomials()

    def _GeneratePolynomials(self):
        accu = int(self.lineEditLamda.text())
        q = float(self.lineEditCommesurateOder.text())
        order = int(self.lineEditFOTFOrder.text())
        poleOrZero = self.comboBoxPolesOrZeros.currentText()

        firstgen = np.arange(order, dtype=float) * q
        nnum = firstgen[::-1]
        num = np.ones_like(nnum)

        # Get fractional polynomial
        poly = poly2str(num, nnum, eps=accu)

        if poleOrZero == "Zero Polynomial":
            self.textEdit_Zeros.setPlainText(poly)
        elif poleOrZero == 'Pole Polynomial':
            self.textEdit_Poles.setPlainText(poly)
    #endregion

    #region Button Clicked Functions
    def _addData(self):
        _loadData = loadDataClass()
        _loadData.exec_()
        _pandasData = ""

        try:
            _sysname = _loadData.lineEditSysName.text()
            _datapath = _loadData.lineEditDataPath.text()
            if _datapath != "" and _sysname != "":
                _pandasD = pd.read_excel(_datapath)
                _pandasData = idData()
                _pandasData.y = np.array(_pandasD.y.values)
                _pandasData.u = np.array(_pandasD.u.values)
                _pandasData.t = np.array(_pandasD.t.values)

                del _pandasD
                self.comboBoxData.addItem(_sysname, _pandasData)
                self.comboBoxData.setCurrentIndex(int(self.comboBoxData.count()) - 1)
            else:
                self.statusbar.showMessage('Data Addition Failed', STATUSBAR_TIME)
                print('\nData Addition Failed\n')

        except:
            self.statusbar.showMessage('Data Addition Failed, wrong Data type', STATUSBAR_TIME)
            print('\nData Addition Failed, wrong Data type\n')

    def _deleteData(self):
        self.comboBoxData.removeItem(self.comboBoxData.currentIndex())

    def _comboBoxDataEmpty(self):
        if self.comboBoxData.count() == 0:
            self.pushButtonDeleteData.setEnabled(False)
            self.pushButtonPlotData.setEnabled(False)
            self.pushButtonTrimData.setEnabled(False)
            self.pushButtonModelValidate.setEnabled(False)
            self.pushButtonIdentify.setEnabled(False)
        else:
            self.pushButtonDeleteData.setEnabled(True)
            self.pushButtonPlotData.setEnabled(True)
            self.pushButtonTrimData.setEnabled(True)
            self.pushButtonModelValidate.setEnabled(True)
            self.pushButtonIdentify.setEnabled(True)

    def _plot(self):
        try:
            currentText = self.comboBoxData.currentText()
            currentData = self.comboBoxData.currentData()
            y = currentData.y
            u = currentData.u
            t = currentData.t

            if y.size != u.size or u.size != t.size or t.size != y.size:
                raise IOError("fotftidguiclass._plot: size of data idd are not the same. Kindly Fix your data")

            plt.figure(dpi=128)
            plt.subplot(2, 1, 1)
            plt.plot(t, y, 'b-')
            plt.title(currentText)
            plt.ylabel('output')
            plt.grid(True, axis='both', which='both')

            plt.subplot(2, 1, 2)
            plt.plot(t, u, 'r-')
            plt.xlabel('time (sec)')
            plt.ylabel('input')
            plt.grid(True, axis='both', which='both')
            plt.show()

            if self.pushButtonTrimData.isEnabled() == False:
                self.pushButtonTrimData.setEnabled(True)
        except:
            print("fotftidguiclass._plot: size of data idd are not the same. Kindly Fix your data")
            self.statusbar.showMessage("fotftidguiclass._plot: size of data idd are not the same. Kindly Fix your data",
                                       STATUSBAR_TIME)

    def _trim(self):
        # read current name and Data
        currentText = self.comboBoxData.currentText()
        currentData = self.comboBoxData.currentData()

        # get datavalues for u,t,y
        y = currentData.y
        u = currentData.u
        t = currentData.t

        # create trim data class
        _trista = trimDataClass()

        t0 = t[0]
        t1 = t[-1]
        # save initial value in  trimdata class as a variable
        _trista.initialName = currentText
        _trista.initialt1 = t0
        _trista.initialt2 = t1

        # fill in data value in trim data class
        _trista.lineEditNewName.setText(currentText)
        _trista.lineEditT1.setText(str(t0))
        _trista.lineEditT2.setText(str(t1))

        # exec the trimdata classs
        _trista.setFocus()
        _trista.exec_()

        # now trimdata is exited get new values
        newText = _trista.lineEditNewName.text()
        if newText != "":
            newt1 = float(_trista.lineEditT1.text())
            newt2 = float(_trista.lineEditT2.text())

            tlower = t >= newt1
            thigher = t <= newt2
            truncy = np.nonzero(tlower == thigher)

            newy = y[truncy]
            newu = u[truncy]
            newt = t[truncy]

            newdata = idData()
            newdata.y = newy
            newdata.u = newu
            newdata.t = newt-newt1

            if currentText == newText or newt.size == t.size:
                print("Data not Trimmed becasue there were no changes in Name or Time")
                self.statusbar.showMessage("Data not Trimmed because there were no changes in '{0}'".format(currentText),
                                           STATUSBAR_TIME)

            else:
                self.comboBoxData.addItem(newText, newdata)
                print("Trimmed '{}' to '{}' successfully".format(currentText, newText))
                self.statusbar.showMessage("Trimmed '{}' to '{}' successfully".format(currentText, newText), STATUSBAR_TIME)
                self.comboBoxData.setCurrentIndex(self.comboBoxData.count() - 1)

    def _stabilityCheck(self):
        try:
            epsi = int(self.lineEditLamda.text())
            if epsi > 3:
                epsi = 3
            if self.checkBoxUseDelay.isChecked():
                _dt = float(self.lineEdit_Delay.text())
            else:
                _dt = 0
            identifiedSytem = None
            isstabledata = False

            if self.IdentifiedModel is None:
                _zero = self.textEdit_Zeros.toPlainText()
                _poles = self.textEdit_Poles.toPlainText()
                identifiedSytem = newfotf(_zero, _poles, _dt)
                identifiedSytem.numberOfDecimal = epsi
                isstabledata = identifiedSytem.isstable(True)

            elif isinstance(self.IdentifiedModel, FOTransFunc):
                self.IdentifiedModel.numberOfDecimal = epsi
                identifiedSytem = self.IdentifiedModel
                isstabledata = self.IdentifiedModel.isstable(doPlot=True)

            if self.checkBoxUseDelay.isChecked():
                self.lineEdit_Delay.setText(_dt)
            if isstabledata[0]:
                print("{}\nSTABLE".format(identifiedSytem))
            else:
                print("{}\nUNSTABLE".format(identifiedSytem))
        except Exception as inst:
            print("An exception occurred. Kindly Check 'Number of Decimals'")
            self.statusbar.showMessage("An exception occurred. Kindly Check 'Number of Decimals'", STATUSBAR_TIME)
            print(type(inst))
            print(inst.args)

    def _roundOff(self):
        epsi = int(self.lineEditLamda.text())
        newZero = None
        newPole = None
        if self.IdentifiedModel is None:
            _zero = self.textEdit_Zeros.toPlainText()
            _poles = self.textEdit_Poles.toPlainText()

            semizero = str2poly(_zero)
            semipole = str2poly(_poles)

            newZero = poly2str(semizero[0], semizero[1], eps=epsi)
            newPole = poly2str(semipole[0], semipole[1], eps=epsi)

        elif isinstance(self.IdentifiedModel,FOTransFunc):
            self.IdentifiedModel.numberOfDecimal = epsi
            num,nnum,den,nden,dt = fotfparam(self.IdentifiedModel)

            newZero = poly2str(num, nnum, eps=epsi)
            newPole = poly2str(den, nden, eps=epsi)

        self.textEdit_Zeros.setText(newZero)
        self.textEdit_Poles.setText(newPole)

    def _validate(self):
        # get verification Data
        currentIndex = self.comboBoxData.currentIndex()
        currentText = self.comboBoxData.currentText()
        verificationData = self.comboBoxData.currentData()

        # simulate verification data
        vy = verificationData.y
        vu = verificationData.u
        vt = verificationData.t

        # get identified system and step
        _zero = self.textEdit_Zeros.toPlainText()
        _poles = self.textEdit_Poles.toPlainText()

        if self.checkBoxUseDelay.isChecked():
            _dt = float(self.lineEdit_Delay.text())
        else:
            _dt = 0

        IdentifiedG = newfotf(_zero, _poles, _dt)
        IdentifiedG.numberOfDecimal = int(self.lineEditLamda.text())

        # simulate with the data from current combobox
        lsimG = lsim(IdentifiedG, vu, vt)

        # plot identified system output vs Data from initial system
        plt.figure(dpi=128)
        plt.subplot(2, 1, 1)
        plt.plot(vt, vy, 'b-', vt, lsimG, 'g-')
        plt.title("Validation Data '{0}' vs Identified Model".format(currentText))
        plt.ylabel('output')
        plt.legend(['Validation data', 'Identified Model'], loc='best')
        plt.grid(True, axis='both', which='both')

        # Fitness measure
        err = vy - lsimG
        fitness = 100 * (1 - (np.linalg.norm(err) / np.linalg.norm(vy - np.mean(lsimG))))
        print("{}\nFitness: {}%".format(IdentifiedG,round(fitness, 2)))

        plt.subplot(2, 1, 2)
        plt.plot(vt, vy - lsimG, 'r-')
        plt.title("Model error with {}% fitness".format(round(fitness, 2)))
        plt.xlabel('time (sec)')
        plt.ylabel('error')
        plt.grid(True, axis='both', which='both')
        plt.show()

    def _identify(self):
        try:
            accu = int(self.lineEditLamda.text())
            # inital Guess
            initialGuess = newfotf(self.textEdit_Zeros.toPlainText(), self.textEdit_Poles.toPlainText())
            initialGuess.numberOfDecimal = accu
            # Similation method from combobox
            optimethod = self.comboBoxOptTypeMethod.currentData()

            if optimethod is simMethod.oustaloop:
                freqlower = int(self.lineEdit_StartFreq.text())
                freqhigher = int(self.lineEdit_StopFreq.text())
                freqOrder = int(self.lineEditOrder.text())
                oustoptions = np.array([freqlower, freqhigher, freqOrder], dtype=int)
            else:
                oustoptions = None

            # Identification algorithm from combobox
            optialg = self.comboBoxAlgorithm.currentData()

            # fix Coef or Expo or Free
            optiFix = self.comboBoxOptFix.currentData()

            # fix Zeros or Poles or None
            polyfix = [self.checkBoxFixZeros.isChecked(), self.checkBoxFixPoles.isChecked()]

            # optimize with delay?
            optiDelay = [self.checkBoxUseDelay.isChecked(), float(self.lineEdit_Delay.text())]

            findDelay = self.checkBoxUseDelay.isChecked()

            # generate optimization parameter class
            optiset = opt(initialGuess, optimethod, optialg, optiFix, polyfix, findDelay, oustaOption=oustoptions,
                          accuracy=10.0 ** -(accu+MAX_LAMBDA))

            # Get Currrent Data
            data = self.comboBoxData.currentData()

            # Get limits settings
            if self.checkBoxUseExpoLimits.isChecked():
                expLim = [int(self.lineEditExpLimitLower.text()), int(self.lineEditExpLimitUpper.text())]
            else:
                # expLim = [10.0 ** -accu, 10]
                expLim = [MIN_EXPO, MAX_EXPO]

            if self.checkBoxUseCoefLimits.isChecked():
                coefLim = [int(self.lineEditCoefLimitLower.text()), int(self.lineEditCoefLimitUpper.text())]
            else:
                coefLim = [MIN_COEF, MAX_COEF]

            # run Identification
            res = fid(data, optiset, [coefLim, expLim], plot=[False, False])
            res.numberOfDecimal = accu+MAX_LAMBDA
            print(res)
            # store identified object as a varable for future use
            self.IdentifiedModel = res

            res.numberOfDecimal = accu          # set roundoff for model
            zero = res.poly2str(res.num[0][0], res.nnum[0][0])
            pole = res.poly2str(res.den[0][0], res.nden[0][0])
            dt = str(res.dt)

            # Update Text Boxes with identififed results
            self.textEdit_Zeros.setPlainText(zero)
            self.textEdit_Poles.setPlainText(pole)
            if self.lineEdit_Delay.isEnabled():
                self.lineEdit_Delay.setPlainText(dt)
        except:
            print("An exception occurred. Try using another limit/ initial guess settings")
            self.statusbar.showMessage("An exception occurred. Try using another limit/ initial guess settings", STATUSBAR_TIME)

    def _GetFOFOPDT(self):
        identifiedSytem = None
        _dt = 0
        epsi = int(self.lineEditLamda.text())
        try:
            if epsi > 4:
                epsi = 4
            if self.checkBoxUseDelay.isChecked():
                _dt = float(self.lineEdit_Delay.text())

            if self.IdentifiedModel is None:
                _zero = self.textEdit_Zeros.toPlainText()
                _poles = self.textEdit_Poles.toPlainText()
                identifiedSytem = newfotf(_zero, _poles, _dt)
                identifiedSytem.numberOfDecimal = epsi

            elif isinstance(self.IdentifiedModel, FOTransFunc):
                self.IdentifiedModel.numberOfDecimal = epsi
                identifiedSytem = self.IdentifiedModel
                identifiedSytem.dt = _dt

            num, nnum, den, nden, dt = fotfparam(identifiedSytem)
            if num.size == nnum.size == 1 and den.size == nden.size == 2:# and self.pushButtonIdentify.isEnabled():
                num = num / den[-1]     #dont change the order, divide num before den
                den = den / den[-1]
                FOFOPDT = fotf(num,nnum,den,nden,dt)
                FOFOPDT.numberOfDecimal = epsi
                print("FOFOPDT = {0}".format(FOFOPDT))
                self.statusbar.showMessage("FO-FOPDT Model is printed in python interpreter", STATUSBAR_TIME)
            else:
                self.pushButton_GetFOFOPDT.setEnabled(False)
        except:
            print("An exception occurred but was caught to avoid a crash")
            self.statusbar.showMessage("An exception occurred but was caught to avoid a crash", STATUSBAR_TIME)

    def closeEvent(self, event):
        reply = QMessageBox.question(self, "Exit?", "Are you sure about Exit?", QMessageBox.Yes | QMessageBox.No,
                                     QMessageBox.Yes)
        if reply == QMessageBox.Yes:
            event.accept()
        else:
            event.ignore()
    #endregion

if __name__ == "__main__":
    app = QApplication(sys.argv)
    fomcon = fotftidguiclass()
    app.exec_()

max_stars_repo_name: outstandn/fomcon

max_issues_repo_name: outstandn/fomcon

max_forks_repo_name: outstandn/fomcon


text:
# This library is designed to read Gadget format I, format II and hdf5 files
import numpy as np
import readsnap
import sys,os,h5py

# find snapshot name and format
def fname_format(snapshot):
    if os.path.exists(snapshot):
        if snapshot[-4:]=='hdf5':
            filename, fformat = snapshot, 'hdf5'
        else:
            filename, fformat = snapshot, 'binary'
    elif os.path.exists(snapshot+'.0'):
        filename, fformat = snapshot+'.0', 'binary'
    elif os.path.exists(snapshot+'.hdf5'):
        filename, fformat = snapshot+'.hdf5', 'hdf5'
    elif os.path.exists(snapshot+'.0.hdf5'):
        filename, fformat = snapshot+'.0.hdf5', 'hdf5'
    else:  raise Exception('File not found!')
    return filename,fformat

# This class reads the header of the gadget file
class header:
    def __init__(self, snapshot):

        filename, fformat = fname_format(snapshot)

        if fformat=='hdf5':
            f             = h5py.File(filename, 'r')

            self.time     = f['Header'].attrs[u'Time']
            self.redshift = f['Header'].attrs[u'Redshift']
            self.npart    = (f['Header'].attrs[u'NumPart_ThisFile']).astype(np.int64)
            self.nall     = (f['Header'].attrs[u'NumPart_Total']).astype(np.int64)
            self.filenum  = int(f['Header'].attrs[u'NumFilesPerSnapshot'])
            self.massarr  = f['Header'].attrs[u'MassTable']
            self.boxsize  = f['Header'].attrs[u'BoxSize']

            # check if it is a SWIFT snapshot
            if '/Cosmology' in f.keys():
                self.omega_m  = f['Cosmology'].attrs[u'Omega_m']
                self.omega_l  = f['Cosmology'].attrs[u'Omega_lambda']
                self.hubble   = f['Cosmology'].attrs[u'h']

            # check if it is a Gadget-4 snapshot
            elif '/Parameters' in f.keys():
                self.omega_m  = f['Parameters'].attrs[u'Omega0']
                self.omega_l  = f['Parameters'].attrs[u'OmegaLambda']
                self.hubble   = f['Parameters'].attrs[u'HubbleParam']
                #self.cooling  = f['Parameters'].attrs[u'Flag_Cooling']

            # if it is a traditional Gadget-1/2/3 snapshot
            else:
                self.omega_m  = f['Header'].attrs[u'Omega0']
                self.omega_l  = f['Header'].attrs[u'OmegaLambda']
                self.hubble   = f['Header'].attrs[u'HubbleParam']
                #self.cooling  = f['Header'].attrs[u'Flag_Cooling']

            self.format   = 'hdf5'
            f.close()

        else:
            head = readsnap.snapshot_header(filename)
            self.time     = head.time
            self.redshift = head.redshift
            self.boxsize  = head.boxsize
            self.filenum  = head.filenum
            self.omega_m  = head.omega_m
            self.omega_l  = head.omega_l
            self.hubble   = head.hubble
            self.massarr  = head.massarr
            self.npart    = head.npart
            self.nall     = head.nall
            self.cooling  = head.cooling
            self.format   = head.format

        # km/s/(Mpc/h)
        self.Hubble = 100.0*np.sqrt(self.omega_m*(1.0+self.redshift)**3+self.omega_l)

# This function reads a block of an individual file of a gadget snapshot
def read_field(snapshot, block, ptype):

    filename, fformat = fname_format(snapshot)
    head              = header(filename)
    Masses            = head.massarr*1e10 #Msun/h
    Npart             = head.npart        #number of particles in the subfile
    Nall              = head.nall         #total number of particles in the snapshot

    if fformat=="binary":
        return readsnap.read_block(filename, block, parttype=ptype)
    else:
        prefix = 'PartType%d/'%ptype
        f = h5py.File(filename, 'r')
        if   block=="POS ":  suffix = "Coordinates"
        elif block=="MASS":  suffix = "Masses"
        elif block=="ID  ":  suffix = "ParticleIDs"
        elif block=="VEL ":  suffix = "Velocities"
        else: raise Exception('block not implemented in readgadget!')

        if '%s%s'%(prefix,suffix) not in f.keys():
            if Masses[ptype] != 0.0:
                array = np.ones(Npart[ptype], np.float32)*Masses[ptype]
            else:
                raise Exception('Problem reading the block %s'%block)
        else:
            array = f[prefix+suffix][:]
        f.close()

        if block=="VEL ":  array *= np.sqrt(head.time)
        if block=="POS " and array.dtype==np.float64:
            array = array.astype(np.float32)

        return array

# This function reads a block from an entire gadget snapshot (all files)
# it can read several particle types at the same time.
# ptype has to be a list. E.g. ptype=[1], ptype=[1,2], ptype=[0,1,2,3,4,5]
def read_block(snapshot, block, ptype, verbose=False):

    # find the format of the file and read header
    filename, fformat = fname_format(snapshot)
    head    = header(filename)
    Nall    = head.nall
    filenum = head.filenum

    # find the total number of particles to read
    Ntotal = 0
    for i in ptype:
        Ntotal += Nall[i]

    # find the dtype of the block
    if   block=="POS ":  dtype=np.dtype((np.float32,3))
    elif block=="VEL ":  dtype=np.dtype((np.float32,3))
    elif block=="MASS":  dtype=np.float32
    elif block=="ID  ":  dtype=read_field(filename, block, ptype[0]).dtype
    else: raise Exception('block not implemented in readgadget!')

    # define the array containing the data
    array = np.zeros(Ntotal, dtype=dtype)

    # do a loop over the different particle types
    offset = 0
    for pt in ptype:

        # format I or format II Gadget files
        if fformat=="binary":
            array[offset:offset+Nall[pt]] = \
                readsnap.read_block(snapshot, block, pt, verbose=verbose)
            offset += Nall[pt]

        # single files (either binary or hdf5)
        elif filenum==1:
            array[offset:offset+Nall[pt]] = read_field(snapshot, block, pt)
            offset += Nall[pt]

        # multi-file hdf5 snapshot
        else:

            # do a loop over the different files
            for i in range(filenum):

                # find the name of the file to read
                filename = '%s.%d.hdf5'%(snapshot,i)

                # read number of particles in the file and read the data
                npart = header(filename).npart[pt]
                array[offset:offset+npart] = read_field(filename, block, pt)
                offset += npart

    if offset!=Ntotal:  raise Exception('not all particles read!!!!')

    return array

max_stars_repo_name: modichirag/sbi-scattering

max_issues_repo_name: modichirag/sbi-scattering

max_forks_repo_name: modichirag/sbi-scattering


text:
from fpb.base import numpy

class Runner(numpy.BaseNumpy1dRunner):
    def run(self, data):
        output = data.max()
        return output

max_stars_repo_name: cloudmercato/python-fpb

max_issues_repo_name: cloudspectatordevelopment/python-fpb

max_forks_repo_name: cloudspectatordevelopment/python-fpb


text:
import numpy as np
import os, sys
from torch.utils.data import Dataset
from torch.utils.data import DataLoader
import torch.nn as nn
import torch

## Locations
FALCON_DIR = os.environ.get('FALCON_DIR')
BASE_DIR = os.environ.get('base_dir')
DATA_DIR = os.environ.get('data_dir')
assert ( all (['FALCON_DIR', 'BASE_DIR', 'DATA_DIR']) is not None)

FEATS_DIR = BASE_DIR + '/feats/world_feats_20msec'
ETC_DIR = BASE_DIR + '/etc'

sys.path.append(FALCON_DIR)
import src.nn.layers as layers

import torch.nn.functional as F
import random
import math

print_flag = 0

# Input: Phones of Shape(B,T1) and World ccoeffs of dim 60 Shape (B,T2,C)
# Output: ccoeffs of Shape (B,T2,C)
class baseline_model(nn.Module):

      def __init__(self):
          super(baseline_model, self).__init__()

class baseline_lstm(baseline_model):

        def __init__(self, vocab_size):
          super(baseline_lstm, self).__init__()

          self.vocab_size = vocab_size
          self.embed_size = 128

          # Encoder
          self.encoder_embedding = nn.Embedding(self.vocab_size, self.embed_size)
          self.seq_model = nn.LSTM(128, 64, 1, batch_first=True)

          # Decoder
          self.decoder_initialfc = nn.Linear(64, 128)
          self.decoder_lstm = nn.LSTM(128, 66, 1, batch_first=True)
          self.final_fc = nn.Linear(66, 128)

        def encoder(self, x):

           # Do something about the phones
           x = self.encoder_embedding(x)

           # Pass through some sequence model
           x, (c,h) = self.seq_model(x, None)

           return c[0]

        def decoder(self, encoder_output, ccoeffs):

            # Loop through the length of decoder
            max_steps = ccoeffs.shape[1]
            outputs = []
            initial_input = encoder_output.new(encoder_output.shape[0],1)
            x = initial_input.zero_()
            x = self.encoder_embedding(x.long())
            assert len(x.shape) == 3
            hidden = None
            for i in range(max_steps):

                assert x.shape[1] == 1
                x, hidden = self.decoder_lstm(x, hidden)
                o = x.squeeze(1)
                outputs.append(o)

                # Modify shape for next input
                x = self.final_fc(x)

            decoder_outputs = torch.stack(outputs, 1)
            return decoder_outputs

        def forward(self, x, c):
            encoder_output = self.encoder(x)
            if print_flag:
                print("Shape of encoder lstm output: ", encoder_output.shape)

            decoder_output =  self.decoder(encoder_output, c)
            if print_flag:
                print("Shape of decoder lstm output: ", decoder_output.shape)

            return decoder_output

class attentionlstm(baseline_lstm):

        def __init__(self, vocab_size):
           super(attentionlstm, self).__init__(vocab_size)

           # Attention
           self.attention_w = nn.Linear(64, 32)
           self.attention_u = nn.Parameter(torch.randn((32,16)))
           self.attention_fc = nn.Linear(128, 64)

           self.vocab_size = vocab_size
           self.embed_size = 128

           # Encoder
           self.encoder_embedding = nn.Embedding(self.vocab_size, self.embed_size)
           self.seq_model = nn.LSTM(128, 64, 1, batch_first=True)

           # Decoder
           self.decoder_initialfc = nn.Linear(64, 128)
           self.decoder_lstm = nn.LSTM(32, 66, 1, batch_first=True)
           self.final_fc = nn.Linear(66, 128)

        def encoder(self, x):

           # Do something about the phones
           x = self.encoder_embedding(x)

           # Pass through some sequence model
           x, (c,h) = self.seq_model(x, None)

           return x

        def decoder(self, encoder_output, ccoeffs):

            # Loop through the length of decoder
            max_steps = ccoeffs.shape[1]
            outputs = []
            initial_input = encoder_output.new(encoder_output.shape[0],1)
            input = initial_input.zero_()
            x = self.encoder_embedding(input.long())
            assert len(x.shape) == 3
            hidden = None
            for i in range(max_steps):

                assert x.shape[1] == 1
                decoder_lstm_input = self.attend(x, encoder_output)
                decoder_lstm_input = decoder_lstm_input.unsqueeze(1)
                #print("Shape of decoder lstm input is ", decoder_lstm_input.shape)

                assert decoder_lstm_input.shape[1] == 1
                x, hidden = self.decoder_lstm(decoder_lstm_input, hidden)

                o = x.squeeze(1)
                outputs.append(o)

                # Modify shape for next input
                x = self.final_fc(x)

            decoder_outputs = torch.stack(outputs, 1)
            return decoder_outputs

        def attend(self, a, A):

           a = self.attention_fc(a)
           if print_flag:
               print("Shapes of a and A are: ", a.shape, A.shape)

           A = a * A
           assert len(A.shape) == 3
           batch_size = A.shape[0]
           #print("Shape of A: ", A.shape)

           # Multiply
           alpha = torch.tanh(self.attention_w(A))
           alpha = F.softmax(alpha, dim = -1)
           #print("Shape of alpha: ", alpha.shape)

           # Sum
           #beta = torch.bmm(alpha, self.attention_u)
           beta = torch.sum(alpha, dim = 1)

           # Return
           #print("Shape of beta is ", beta.shape)
           return beta

        def forward(self, x, c):
            encoder_output = self.encoder(x)
            if print_flag:
                print("Shape of encoder lstm output: ", encoder_output.shape)

            decoder_output =  self.decoder(encoder_output, c)
            if print_flag:
                print("Shape of decoder lstm output: ", decoder_output.shape)

            return decoder_output

max_stars_repo_name: APMplusplus/falkon

max_issues_repo_name: saikrishnarallabandi/falkon

max_forks_repo_name: saikrishnarallabandi/falkon


text:
import matplotlib.pylab as plt
import numpy as np
import soundfile as sf
import torch

from config import sampling_rate
from models.layers import STFT
from text import text_to_sequence

class Denoiser(torch.nn.Module):
    """ Removes model bias from audio produced with waveglow """

    def __init__(self, waveglow, filter_length=1024, n_overlap=4,
                 win_length=1024, mode='zeros'):
        super(Denoiser, self).__init__()
        self.stft = STFT(filter_length=filter_length,
                         hop_length=int(filter_length / n_overlap),
                         win_length=win_length).cuda()
        if mode == 'zeros':
            mel_input = torch.zeros(
                (1, 80, 88),
                dtype=waveglow.upsample.weight.dtype,
                device=waveglow.upsample.weight.device)
        elif mode == 'normal':
            mel_input = torch.randn(
                (1, 80, 88),
                dtype=waveglow.upsample.weight.dtype,
                device=waveglow.upsample.weight.device)
        else:
            raise Exception("Mode {} if not supported".format(mode))

        with torch.no_grad():
            bias_audio = waveglow.infer(mel_input, sigma=0.0).float()
            bias_spec, _ = self.stft.transform(bias_audio)

        self.register_buffer('bias_spec', bias_spec[:, :, 0][:, :, None])

    def forward(self, audio, strength=0.1):
        audio_spec, audio_angles = self.stft.transform(audio.cuda().float())
        audio_spec_denoised = audio_spec - self.bias_spec * strength
        audio_spec_denoised = torch.clamp(audio_spec_denoised, 0.0)
        audio_denoised = self.stft.inverse(audio_spec_denoised, audio_angles)
        return audio_denoised

def plot_data(data, figsize=(16, 4)):
    fig, axes = plt.subplots(1, len(data), figsize=figsize)
    for i in range(len(data)):
        axes[i].imshow(data[i], aspect='auto', origin='bottom',
                       interpolation='none')

if __name__ == '__main__':
    checkpoint = 'BEST_checkpoint.tar'
    checkpoint = torch.load(checkpoint)
    model = checkpoint['model']
    model.eval()

    waveglow_path = 'waveglow_256channels.pt'
    waveglow = torch.load(waveglow_path)['model']
    waveglow.cuda().eval().half()
    for k in waveglow.convinv:
        k.float()
    denoiser = Denoiser(waveglow)

    text = "You, me, or nobody is gonna hit as hard as life. But it is not about how hard you hit. It’s about how hard you can get hit and keep moving forward."
    sequence = np.array(text_to_sequence(text, ['english_cleaners']))[None, :]
    sequence = torch.autograd.Variable(torch.from_numpy(sequence)).cuda().long()

    mel_outputs, mel_outputs_postnet, _, alignments = model.inference(sequence)
    plot_data((mel_outputs.float().data.cpu().numpy()[0],
               mel_outputs_postnet.float().data.cpu().numpy()[0],
               alignments.float().data.cpu().numpy()[0].T))
    plt.savefig('images/mel_spec.jpg')

    np.save('mel_outputs.npy', mel_outputs_postnet.float().data.cpu().numpy()[0])

    mel_outputs_postnet = mel_outputs_postnet.type(torch.float16)
    with torch.no_grad():
        audio = waveglow.infer(mel_outputs_postnet, sigma=0.666)

    audio = audio[0].data.cpu().numpy()
    audio = audio.astype(np.float32)

    print('audio.shape: ' + str(audio.shape))
    print(audio)

    sf.write('output.wav', audio, sampling_rate, 'PCM_24')


text:
import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))
#
# opel_corsa_01 data
#
nRowsRead = 1000 # specify 'None' if want to read whole file
# opel_corsa_01.csv has 7392 rows in reality, but we are only loading/previewing the first 1000 rows
df1 = pd.read_csv('../input/mldrivingstyles/opel_corsa_01.csv', delimiter=';', nrows = nRowsRead)
df1.dataframeName = 'opel_corsa_01.csv'
nRow, nCol = df1.shape
print(f'There are {nRow} rows and {nCol} columns')

#
# opel_corsa_02 data
#
nRowsRead = 1000
df2 = pd.read_csv('../input/mldrivingstyles/opel_corsa_02.csv', delimiter=';', nrows = nRowsRead)
df2.dataFrame = 'opel_corsa_02.csv'
nrow, nCol = df2.shape
print(f'There are {nRow} rows and {nCol} columns')

#
# peugeot_207_01 data
#
nRowsRead = 1000
df3 = pd.read_csv('../input/mldrivingstyles/peugeot_207_01.csv', delimiter=';', nrows = nRowsRead)
df3.dataFrame = 'peugeot_207_01'
nRow, nCol = df3.shape
print(f'There are {nRow} rowsand {nCol} columns')

#
# peugeot_207_02.csv data
#
nRowsRead = 1000
df4 = pd.read_csv('../input/mldrivingstyles/peugeot_207_02.csv', delimiter=';', nrows = nRowsRead)
df4.dataFrame = 'peugeot_207_02.csv'
nRow, nCol = df4.shape
print(f'There are {nRow} rows and {nCol} columns')

max_stars_repo_name: anthonymeagher/machinelearning

max_issues_repo_name: anthonymeagher/machinelearning

max_forks_repo_name: anthonymeagher/machinelearning


text:
import argparse
import random

import logging
import tensorflow as tf
from optimization.optimize import build_tensorflow
from common import settings_reader, io, model_builder, optimizer_parameter_parser, evaluation, auxilliaries
from model import Model
import numpy as np
import pandas as pd

logging.basicConfig(
    format="%(asctime)s| %(message)s", datefmt="%d-%m-%y %H:%M:%S", level=logging.INFO
)
logger = logging.getLogger("split_data_to_sets")

parser = argparse.ArgumentParser(description="Train a model on a given dataset.")
parser.add_argument("--settings", help="Filepath for settings file.", required=True)
parser.add_argument("--dataset", help="Filepath for dataset.", required=True)
args = parser.parse_args()

settings = settings_reader.read(args.settings)

'''
Load datasets:
'''

dataset = args.dataset

relations_path = dataset + '/relations.dict'
entities_path = dataset + '/entities.dict'
train_path = dataset + '/formatted/train_triplets.csv'
valid_path = dataset + '/formatted/valid_triplets.csv'
test_path = dataset + '/formatted/test_triplets.csv'

# Extend paths for accuracy evaluation:
if settings['Evaluation']['Metric'] == 'Accuracy':
    valid_path = dataset + '/valid_accuracy.txt'
    test_path = dataset + '/test_accuracy.txt'

train_triplets = pd.read_csv(train_path, header=0)
valid_triplets = pd.read_csv(valid_path, header=0)
test_triplets = pd.read_csv(test_path, header=0)

train_triplets = train_triplets.values
valid_triplets = valid_triplets.values
test_triplets = test_triplets.values

entities = io.read_dictionary(entities_path)
relations = io.read_dictionary(relations_path)

logger.info("Entities: {}".format(len(entities)))
logger.info("Relations: {}".format(len(relations)))
logger.info("Splits count: {train}(train), {valid}(valid), {test}(test)".format(
    train=train_triplets.shape[0], valid=valid_triplets.shape[0],
    test=test_triplets.shape[0]))

'''
Load general settings
'''

encoder_settings = settings['Encoder']
decoder_settings = settings['Decoder']
shared_settings = settings['Shared']
general_settings = settings['General']
optimizer_settings = settings['Optimizer']
evaluation_settings = settings['Evaluation']

general_settings.put('EntityCount', len(entities))
general_settings.put('RelationCount', len(relations))
general_settings.put('EdgeCount', len(train_triplets))

encoder_settings.merge(shared_settings)
encoder_settings.merge(general_settings)
decoder_settings.merge(shared_settings)
decoder_settings.merge(general_settings)

optimizer_settings.merge(general_settings)
evaluation_settings.merge(general_settings)

'''
Construct the encoder-decoder pair:
'''
encoder = model_builder.build_encoder(encoder_settings, train_triplets)
model = model_builder.build_decoder(encoder, decoder_settings)

'''
Construct the optimizer with validation MRR as early stopping metric:
'''

opp = optimizer_parameter_parser.Parser(optimizer_settings)
opp.set_save_function(model.save)

scorer = evaluation.Scorer(evaluation_settings)
scorer.register_data(train_triplets)
scorer.register_data(valid_triplets)
scorer.register_data(test_triplets)
scorer.register_degrees(train_triplets)
scorer.register_model(model)
scorer.finalize_frequency_computation(np.concatenate((train_triplets, valid_triplets,
                                                      test_triplets), axis=0))

def score_validation_data(validation_data):
    score_summary = scorer.compute_scores(validation_data, verbose=False).get_summary()
    # score_summary.dump_degrees('dumps/degrees.in', 'dumps/degrees.out')
    # score_summary.dump_frequencies('dumps/near.freq', 'dumps/target.freq')
    # score_summary.pretty_print()

    lookup_string = ''
    if evaluation_settings['Metric'] == 'MRR':
        lookup_string = score_summary.mrr_string()
    elif evaluation_settings['Metric'] == 'Accuracy':
        lookup_string = score_summary.accuracy_string()

    early_stopping = score_summary.results['Filtered'][lookup_string]

    score_summary = scorer.compute_scores(test_triplets, verbose=False).get_summary()
    score_summary.pretty_print()

    return early_stopping

opp.set_early_stopping_score_function(score_validation_data)

adj_list = [[] for _ in entities]
for i, triplet in enumerate(train_triplets):
    adj_list[triplet[0]].append([i, triplet[2]])
    adj_list[triplet[2]].append([i, triplet[0]])

degrees = np.array([len(a) for a in adj_list])
adj_list = [np.array(a) for a in adj_list]

def sample_edge_neighborhood(triplets, sample_size):

    edges = np.zeros(sample_size, dtype=np.int32)

    # initialize
    sample_counts = np.array([d for d in degrees])
    picked = np.array([False for _ in triplets])
    seen = np.array([False for _ in degrees])

    for idx in range(0, sample_size):
        weights = sample_counts * seen

        if np.sum(weights) == 0:
            weights = np.ones_like(weights)
            weights[np.where(sample_counts == 0)] = 0

        probabilities = weights / np.sum(weights)
        chosen_vertex = np.random.choice(np.arange(degrees.shape[0]), p=probabilities)
        chosen_adj_list = adj_list[chosen_vertex]
        seen[chosen_vertex] = True

        chosen_edge = np.random.choice(np.arange(len(chosen_adj_list)))
        chosen_edge = chosen_adj_list[chosen_edge]
        edge_number = chosen_edge[0]

        while picked[edge_number]:
            chosen_edge = np.random.choice(np.arange(len(chosen_adj_list)))
            chosen_edge = chosen_adj_list[chosen_edge]
            edge_number = chosen_edge[0]

        edges[idx] = edge_number
        other_vertex = chosen_edge[1]
        picked[edge_number] = True
        sample_counts[chosen_vertex] -= 1
        sample_counts[other_vertex] -= 1
        seen[other_vertex] = True

    return edges

if 'NegativeSampleRate' in general_settings:
    ns = auxilliaries.NegativeSampler(int(general_settings['NegativeSampleRate']),
                                      general_settings['EntityCount'])
    ns.set_known_positives(train_triplets)

    def t_func(x):  # horrible hack!!!
        arr = np.array(x)
        if not encoder.needs_graph():
            return ns.transform(arr)
        else:
            if 'GraphBatchSize' in general_settings:
                graph_batch_size = int(general_settings['GraphBatchSize'])

                '''
                n = np.zeros(100)
                for i in range(100):
                    if i % 20 == 0:
                        print(i)
                    n[i] = sample_TIES(arr, 1000).shape[0]

                print(n.mean())
                print(n.std())
                exit()
                '''

                # graph_batch_ids = sample_TIES(arr, 1000) #sample_edge_neighborhood(
                # arr, graph_batch_size)
                graph_batch_ids = sample_edge_neighborhood(arr, graph_batch_size)
            else:
                graph_batch_size = arr.shape[0]
                graph_batch_ids = np.arange(graph_batch_size)

            graph_batch = np.array(train_triplets)[graph_batch_ids]

            # Apply dropouts:
            graph_percentage = float(general_settings['GraphSplitSize'])
            split_size = int(graph_percentage * graph_batch.shape[0])
            graph_split_ids = np.random.choice(graph_batch_ids, size=split_size,
                                               replace=False)
            graph_split = np.array(train_triplets)[graph_split_ids]

            t = ns.transform(graph_batch)

            flag_data = 'StoreEdgeData'
            if flag_data in encoder_settings and encoder_settings[flag_data] == "Yes":
                return graph_split, graph_split_ids, t[0], t[1]
            else:
                return graph_split, t[0], t[1]

    opp.set_sample_transform_function(t_func)

'''
Initialize for training:
'''

# Hack for validation evaluation:
model.preprocess(train_triplets)
model.register_for_test(train_triplets)

model.initialize_train()

optimizer_weights = model.get_weights()
optimizer_input = model.get_train_input_variables()
loss = model.get_loss(mode='train') + model.get_regularization()

'''
Clean this up:
'''

for add_op in model.get_additional_ops():
    opp.additional_ops.append(add_op)

optimizer_parameters = opp.get_parametrization()

'''
Train with Converge:
'''
print(optimizer_parameters)
model.session = tf.Session()
optimizer = build_tensorflow(loss, optimizer_weights, optimizer_parameters,
                             optimizer_input)
optimizer.set_session(model.session)

optimizer.fit(train_triplets, validation_data=valid_triplets)
# scorer.dump_all_scores(valid_triplets, 'dumps/subjects.valid', 'dumps/objects.valid')
# scorer.dump_all_scores(test_triplets, 'dumps/subjects.test', 'dumps/objects.test')

max_stars_repo_name: vamships/RelationPrediction

max_issues_repo_name: vamships/RelationPrediction

max_forks_repo_name: vamships/RelationPrediction