text:
import pyomo.environ as pe
import pandas as pd
import numpy as np

def network_precondition(net):
    """Auxiliary funtion that add extra columns in the net model,
    in order to maje useful for optimization model."""
    all_tables = ['bus', 'load','sgen', 'motor', 'asymmetric_load', 'asymmetric_sgen', 'storage',
                        'gen', 'switch', 'shunt','ext_grid', 'line', 'trafo', 'trafo3w', 'impedance',
                        'dcline', 'ward', 'xward', 'measurement', 'pwl_cost', 'poly_cost', 'controller']

    for t in all_tables:
        if not hasattr(net[t], 'model'):
            net[t]['model'] = None

class BaseModelWriter:

    def __init__(self, net = None, scenes = None):
        self.net = net
        self.scenes = scenes
        self.model = None
        self.results = None
        self.tables = []
        self.max_investement = None
        self.load_tables = ['load']

        if net:
            #self._add_extra_columns(self.net)
            network_precondition(net)
    """
    def _add_extra_columns(self, net):
        all_tables = ['bus', 'load','sgen', 'motor', 'asymmetric_load', 'asymmetric_sgen', 'storage',
                        'gen', 'switch', 'shunt','ext_grid', 'line', 'trafo', 'trafo3w', 'impedance',
                        'dcline', 'ward', 'xward', 'measurement', 'pwl_cost', 'poly_cost', 'controller']

        for t in all_tables:
            net[t]['model'] = None
    """

    def create_model(self):
        self.model = pe.ConcreteModel()
        self.model.scenes = pe.Set(initialize = range(len(self.scenes)), doc = 'Scene Set')
        return self.model

    def additional_constraint(self, name, expression):
        """
        Adds a constraint of the simple (not  indexed) type.
        Arguments:
            name: unique name
            expression: a Pyomo expression (relation between Pyomo vars and operators)
        """
        constraint = pe.Constraint(expr = expression)
        setattr(self.model, name, constraint)
        setattr(self, name, constraint)

    def remove_if_has_constraint(self, name):
        if hasattr(self.model, name):
            delattr(self.model, name)
            return True
        return False

    def remove_constraint(self, name):
        if hasattr(self.model, name):
            delattr(self.model, name)
        else:
            raise Exception("Constraint not found")

    def backconfigure_network(self):
        """
        Modifies network tables based on results of the optimization.
        Works only on sgen's.

        Returns:
            A reference to the network description.
        """
        for table in self.tables:
            for element in range(len(table)):
                m = table['model'][element]
                if m:
                    if m.decide_construction:
                        vn = m.name + '_create'
                        var = getattr(self.model, vn).value
                        if var:
                            table['in_service'][element] = True
                            if hasattr(table, 'max_p_mw'): table['max_p_mw'][element] = getattr(self.model, m.name + '_pr_mw').value
                            if hasattr(table, 'max_q_mvar'): table['max_q_mvar'][element] = 0.5*getattr(self.model, m.name + '_pr_mw').value
                            if hasattr(table, 'max_e_mwh'): table['max_e_mwh'][element] = getattr(self.model, m.name + '_er_mwh').value
                        else:
                            table['in_service'][element] = False
                            if hasattr(table, 'max_p_mw'): table['max_p_mw'][element] = 0.0
                            if hasattr(table, 'max_q_mvar'): table['max_q_mvar'][element] = 0.0
                            if hasattr(table, 'max_e_mwh'): table['max_e_mwh'][element] = 0.0

    def get_scenes_results(self):
        """
        Returns a new Pandas DataFrame with the simulation results based on the
        optimization results.
        """
        #Extracción de resultados en forma de Pandas Dataframe
        self.results =  pd.DataFrame([], index = range(len(self.scenes)))

        for table in self.tables:
            for element in range(len(table)):
                m = table['model'][element]
                if m and hasattr(m, 'get_scenes_results'):
                    m.get_scenes_results(self.results)

        return self.results

    def total_suministred_energy(self):

        load_tables = [self.net.load]
        energy = 0.0

        for t in self.load_tables:
            table = self.net[t]
            for element in range(len(table)):
                m = table['model'][element]
                if m and hasattr(m, 'p_mw'):
                    for s in self.model.scene_set:
                        scene = self.scenes.iloc[s]
                        energy += m.p_mw[s].value*scene['dt']*scene['dd']

        return -energy

    def total_discounted_energy(self):

        load_tables = [self.net.load]
        energy = 0.0

        for t in self.load_tables:
            table = self.net[t]
            for element in range(len(table)):
                m = table['model'][element]
                if m and hasattr(m, 'p_mw'):
                    for s in self.model.scene_set:
                        scene = self.scenes.iloc[s]
                        energy += m.p_mw[s].value*scene['dt']*scene['dd']*scene['discount']

        return -energy

    def total_cost_function(self):

        return pe.value(model_writer.model.value)

    def npc(self):
        """
        NPC can be the objective function, in this case is more efficient
        to call total_cost_funtion. A true implementation is needed.
        """
        return pe.value(self.model.value)

    def npv(self):
        """
        NPV can be the objective function, in this case is more efficient
        to call total_cost_funtion. A true implementation is needed.
        """
        return pe.value(self.model.value)

    def lcoe(self):
        return self.npc()/self.total_discounted_energy()

    def lvoe(self):
        return self.npv()/self.total_discounted_energy()

max_stars_repo_name: ExoticAnt/MMGO

reason: import numpy,import pyomo


text:
import numpy as np
import gym
from collections import namedtuple
from gym.utils import seeding

ObsInfo = namedtuple('ObsInfo', ('names', 'dimensions', 'idx'))
StateInfo = namedtuple('StateInfo', ('names', 'dimensions', 'idx'))
ObsType = namedtuple('ObsType', ('name', 'idx'))
StateType = namedtuple('StateType', ('name', 'idx'))

class RobolearnEnv(gym.Env):
    def __init__(self, seed=None):
        self.step_counter = 0
        self.action_space = None
        self.observation_space = None
        self.state_space = None
        self.done = False

        # Set the seed
        self.seed(seed)

    def _set_action(self, action):
        """Applies the given action to the environment.
        """
        raise NotImplementedError()

    def _update_env_obs(self):
        """Updates the environment observations.
        """
        raise NotImplementedError()

    def _update_env_state(self):
        """Updates the environment state.
        """
        raise NotImplementedError()

    def _compute_reward(self, state, action, next_state=None):
        """Compute the reward, based on state, action,
        (and possible next_state)
        """
        raise NotImplementedError()

    def _check_termination(self):
        """Check if some conditions are met.
        """
        raise NotImplementedError()

    def _reset_env(self, *args, **kwargs):
        """Reset everything in the environment"""
        raise NotImplementedError()

    def reset(self, *args, **kwargs):
        """Reset the environment.

        Args:
            *args:
            **kwargs:

        Returns:
            np.ndarray: Initial environment observation

        """
        self.step_counter = 0

        self._reset_env(*args, **kwargs)

        return self._update_env_obs()

    def step(self, action):
        """Run one timestep of the environment's dynamics.

        Accepts an action and returns a tuple (observation, reward, done, info).

        Args:
            action (float[Adim]): action provided by the environment

        Returns:
            next_observation (float[Odim]): agent's observation of the current
                environment
            reward (float): amount of reward returned after previous action
            done (boolean): whether the episode has ended, in which case
                further step() calls will return undefined results
            info (dict): contains auxiliary diagnostic information (helpful
                for debugging, and sometimes learning)

        """
        # Get current environment state: s_t
        state = self._update_env_state()

        # Apply action to environment: a_t
        action = np.clip(action, self.action_space.low, self.action_space.high)
        self._set_action(action)

        # Update env observation and state: s_t+1, o_t+1
        next_observation = self._update_env_obs()
        next_state = self._update_env_state()

        # Calculate if it is done
        self.done, done_info = self._check_termination()

        # Calculate the reward
        reward, reward_info = self._compute_reward(state, action, next_state)

        # Environment information dictionary
        info = {
            'next_state': next_state,
        }
        info.update(done_info)
        info.update(reward_info)

        self.step_counter += 1

        return next_observation, reward, self.done, info

    def seed(self, seed=None, hard=True):
        if hard:
            self.np_random, seed = seeding.np_random(seed)
            # TODO: use the same np_randomizer for robot as for env
            # self.robot.np_random = self.np_random
        else:
            raise NotImplementedError
        return [seed]

    def stop(self):
        """
        Something useful to do. E.g. stop command to robot.
        :return:
        """
        pass

    # ########## #
    # Attributes #
    # ########## #
    @property
    def step_counter(self):
        return self._step_counter

    @step_counter.setter
    def step_counter(self, num_step):
        self._step_counter = num_step

    @property
    def done(self):
        return self._is_done

    @done.setter
    def done(self, done):
        self._is_done = done

    @property
    def obs_dim(self):
        if self.observation_space is None:
            raise AttributeError("No obs_space has been defined for this "
                                 "environment.")
        else:
            return np.prod(self.observation_space.shape)

    @property
    def action_dim(self):
        if self.action_space is None:
            raise AttributeError("No action_space has been defined for this "
                                 "environment.")
        else:
            return np.prod(self.action_space.shape)

    @property
    def state_dim(self):
        if self.state_space is None:
            raise AttributeError("No state_space has been defined for this "
                                 "environment.")
        else:
            return np.prod(self.state_space.shape)

    @property
    def action_bounds(self):
        if self.action_space is None:
            raise AttributeError("No action_space has been defined for this "
                                 "environment.")
        else:
            return np.vstack((self.action_space.low, self.action_space.high))

    @property
    def low_action_bounds(self):
        if self.action_space is None:
            raise AttributeError("No action_space has been defined for this "
                                 "environment.")
        else:
            return self.action_space.low

    @property
    def high_action_bounds(self):
        if self.action_space is None:
            raise AttributeError("No action_space has been defined for this "
                                 "environment.")
        else:
            return self.action_space.high

    @property
    def name(self):
        return type(self).__name__.lower()

    @property
    def np_random(self):
        return self._np_random

    @np_random.setter
    def np_random(self, random_state):
        if not isinstance(random_state, np.random.RandomState):
            raise ValueError("It is not a numpy random state!.")
        self._np_random = random_state

ext: py

reason: import numpy


text:
import torch.nn as nn
import torch
import torch.nn.functional as F
import numpy as np
import matplotlib.pyplot as plt
from torch import optim
# print(torch.__version__)
from torch.cuda.amp import autocast

class Flatten(nn.Module):
    def forward(self, input):
        return input.view(input.size(0), -1)

class Squeeze_Layer(nn.Module):
    def forward(self, input):
        return input[:, 0, 0, :, :]

class Permute_Layer(nn.Module):
    def forward(self, input):
        return input.permute(0, 2, 1, 3, 4)

class Height_pool(nn.Module):
    def forward(self, input):
        if input.shape[2] == 1:
            return input
        else:
            return F.adaptive_avg_pool3d(input, (1, input.shape[3], input.shape[4]))

class ChannelPool(nn.Module):
    def forward(self, input):
        return F.adaptive_avg_pool2d(input.permute(0, 2, 1, 3), (1, 1)).permute(0, 2, 1, 3)

class Tstep_combine(nn.Module):
    def __init__(self, channels):
        super().__init__()
        self.conv = nn.Sequential(nn.Conv2d(3, 3, (1, 1)),
                                  nn.LeakyReLU(),
                                  nn.Conv2d(3, 1, (1, 1)),
                                  nn.LeakyReLU())
        # self.conv_2 = nn.Conv2d(channels, channels, (1, 1))
        # self.relu = nn.LeakyReLU()

    def forward(self, input):
        if input.shape[2] == 1:
            return input
        else:
            min_ten, _ = input.min(dim=2, keepdim=True)
            max_ten, _ = input.max(dim=2, keepdim=True)
            # std_ten = input.std(dim=2, keepdim=True)
            avg_ten = input.mean(dim=2, keepdim=True)

            new_tensor = torch.cat((min_ten, max_ten, avg_ten), dim=2)

            output = self.conv(new_tensor.permute(
                0, 2, 1, 3)).permute(0, 2, 1, 3)

            return output

class Height_conv(nn.Module):
    def __init__(self, height):
        super().__init__()
        if height > 1:
            # self.conv = nn.Sequential(nn.Conv1d(height, 10, 1),
            #     nn.LeakyReLU(),
            #     nn.Conv1d(10, 5, 1),
            #     nn.LeakyReLU(),
            #     nn.Conv1d(5, 1, 1))
            self.conv = nn.Sequential(nn.Conv2d(height, 2 * height, (1, 1)),
                                      nn.LeakyReLU(),
                                      nn.Conv2d(2 * height, 2 *
                                                height, (1, 1)),
                                      nn.LeakyReLU(),
                                      nn.Conv2d(2 * height, 1, (1, 1)))

    def forward(self, input):
        if input.shape[2] == 1:
            return input
        else:
            output = self.conv(input[:, 0, :, :]).unsqueeze(1)

            return output

def compute_stft(x, length=0, pool_size=0, crop=1000, hop_length=0, norm=0, harmonics=0, harmonic_downsample=False, no_adding=False, pad_factor=0,
                 bin_comparison=False):
    x = x - x.mean(dim=2, keepdim=True)
    switch = 0

    # if pad_factor!=0:
    #     pad_val = int(x.shape[2] * pad_factor)
    #     x = F.pad(x, (0, pad_val))

    if length == 0:
        length = x.shape[2]
    if hop_length == 0:
        hop_length = length

    for j in range(x.shape[1]):
        added_harmonics = 0
        stft_count = 0
        switch_harm = 0
        if pad_factor == 0:
            stft = torch.stft(x[:, j, :], length, hop_length=hop_length, win_length=None,
                              window=None, center=False, normalized=True, onesided=True, return_complex=False)
        else:
            nfft = int(length + pad_factor * length)
            hop_length += 1
            win_length = length

            stft = torch.stft(x[:, j, :], nfft, hop_length=hop_length, win_length=nfft,
                              window=None, center=True, normalized=True, onesided=True, return_complex=False, pad_mode='constant')
            # print(stft.shape)

        if not bin_comparison:
            power_stft = stft[:, :crop, :, 0] ** 2 + \
                stft[:, :crop, :, 1] ** 2
        else:
            ini_power_stft = stft[:, :crop, :, 0] ** 2 + \
                stft[:, :crop, :, 1] ** 2
            bin_stft = stft + torch.roll(stft, 1, 1)
            bin_power_stft = (bin_stft[:, :crop, :, 0] ** 2 +
                              bin_stft[:, :crop, :, 1] ** 2) / 2
            stack_stft = torch.stack(
                (ini_power_stft, bin_power_stft, torch.roll(bin_power_stft, 1, -1)), -1)

            power_stft, _ = torch.max(stack_stft, -1)

        for harm_counter in range(2 ** harmonics):
            if harm_counter == 0:
                added = power_stft[:, :, :]
            else:
                if not harmonic_downsample:
                    upsampled = F.interpolate(power_stft.transpose(
                        2, 1), scale_factor=harm_counter + 1, mode='nearest').transpose(2, 1)
                    new_harm_series = upsampled[:, :crop, :]
                    # added = added[:, :, :] + upsampled[:, :crop, :]
                    # print(added, upsampled)
                else:
                    downsampled = F.max_pool1d(power_stft.transpose(
                        2, 1), kernel_size=(harm_counter + 1), stride=(harm_counter + 1)).transpose(2, 1)[:, :crop, :]
                    pad_val = added.shape[1] - downsampled.shape[1]
                    new_harm_series = F.pad(downsampled, (0, 0, 0, pad_val))
                    # added = added[:, :, :] + downsampled
                if not no_adding:
                    added = added[:, :, :] + new_harm_series
                else:
                    if switch_harm == 0:
                        # print(x.shape, power_stft.shape, downsampled.shape)
                        # out_harm = downsampled.unsqueeze(3).unsqueeze(4)
                        out_harm = torch.cat((power_stft.unsqueeze(3).unsqueeze(
                            4), new_harm_series.unsqueeze(3).unsqueeze(4)), dim=4)
                        switch_harm = 1
                    else:
                        # print(x.shape, power_stft.shape, downsampled.shape)
                        out_harm = torch.cat(
                            (out_harm, new_harm_series.unsqueeze(3).unsqueeze(4)), dim=4)
                        # print(out_harm.shape)
                    # print(added, downsampled, pad_val)
                # print(added.device, added.shape)
            added_harmonics += 1
            # if no_adding:
            #     if harmonic_downsample:
            #         pad_val = power_stft.shape[1] - downsampled.shape[1]
            #         downsampled = F.pad(downsampled, (0,0,0,0,0,pad_val))
            #         added = torch.cat((added[:, :, :, :], downsampled.permute(0,1, 3, 2)), dim=3)
            if added_harmonics == 2 ** stft_count and not no_adding:
                if pool_size:
                    single_out = F.adaptive_max_pool1d(
                        added.transpose(2, 1), pool_size).transpose(2, 1)
                else:
                    single_out = added
                if not switch_harm:
                    out_harm = single_out.unsqueeze(3)
                    switch_harm = 1
                else:
                    out_harm = torch.cat((out_harm, single_out.unsqueeze(
                        3) / np.sqrt(added_harmonics)), dim=3)
                stft_count += 1
        power_stft = out_harm
        # if pool_size:
        #     power_stft = F.adaptive_max_pool1d(power_stft.transpose(2, 1), pool_size)
        # else:
        power_stft = power_stft.transpose(2, 1)
        if not switch:
            out_stft = power_stft
            switch = 1
        else:
            out_stft = torch.cat((out_stft, power_stft), dim=3)
    if norm:
        std = out_stft.view(out_stft.shape[0], -1).std(dim=1)
        out_stft = out_stft / std[:, None, None]
    # if not harmonics:
    #     out_stft = out_stft[:, :, :, :]
    # print(out_stft.shape, no_adding)

    return out_stft

class classifier_stft(nn.Module):
    def __init__(self, input_length, input_resolution, class_para, name='', dm0_class=False, channel_classification=False):
        super().__init__()
        self.input_length = input_length
        self.pad_factor = class_para.pad_factor
        # if class_para.stft_count==1:
        #     self.pad_factor = class_para.pad_factor
        # else:
        #     self.pad_factor = 0
        # if self.pad_factor!=0:
        #     self.input_length += int(self.input_length * self.pad_factor)
        #self.crop = int(crop_factor * (self.input_length // 2))
        self.final_output = 2
        self.crop_factor = class_para.crop_factor
        self.channels = class_para.channels
        self.kernel = class_para.kernel
        self.norm = class_para.norm
        self.harmonics = class_para.harmonics
        self.nn_layers = class_para.nn_layers
        self.dm0_class = dm0_class
        self.input_resolution = input_resolution
        self.height_dropout = class_para.height_dropout

        self.use_center = False

        self.stft_count = class_para.stft_count
        self.bin_comparison = class_para.bin_comparison

        self.channel_classification = channel_classification

        if hasattr(class_para, 'norm_significance'):
            self.norm_significance = class_para.norm_significance
        else:
            norm_significance = False
        # print(input_length)

        max_height = 2 ** (self.stft_count - 1)
        self.lengths = []
        self.total_height = 0
        self.heights = []

        self.name = name
        self.harmonic_downsample = class_para.harmonic_downsample
        self.train_harmonic = class_para.train_harmonic
        self.combine_harm = (class_para.combine_harm if hasattr(
            class_para, 'combine_harm') else False)

        if self.train_harmonic:
            self.harmonic_net = nn.Sequential(nn.Conv3d(self.harmonics**2, self.harmonics + 1, (1, 1, 1)),
                                              nn.LeakyReLU())

        for j in range(self.stft_count):
            current_in = 1
            current_out = 0
            layers = []
            height = 2 ** j
            self.total_height += height
            current_length = int(np.floor(self.input_length / height))
            self.min_length = current_length
            self.lengths.append(current_length)
            self.heights.append(height)
            pool = max_height // height
            setattr(self, f"pool_{current_length}", nn.AvgPool3d(
                (1, pool, 1), stride=(1, pool, 1), padding=(0, pool // 2, 0)))
            # print(height)

        current_in = 1
        if self.total_height > 1:
            current_out = self.channels
        else:
            current_out = 0
        layers = []
        if self.height_dropout:
            layers += [nn.Dropout2d(self.height_dropout)]

        if self.nn_layers >= 0:
            current_out += self.channels
            pool = max_height // height
            if not self.combine_harm:
                ini_harm_kernel = 1
                ini_harm_stride = 1
            else:
                ini_harm_kernel = (self.harmonics + 1)
                ini_harm_stride = (self.harmonics + 1)
            layers += [nn.Conv2d(self.total_height, current_out, (self.kernel, ini_harm_kernel), stride=(
                1, ini_harm_stride), padding=(self.kernel // 2, 0)),
            ]
            for i in range(self.nn_layers):
                current_in = current_out
                current_out += self.channels
                dilation = 2 ** (i + 1)
                pad = (self.kernel // 2) * dilation

                layers += [nn.LeakyReLU(),
                           nn.Conv2d(current_in, current_out, (self.kernel, 1), stride=(
                               1, 1), padding=(pad, 0), dilation=(dilation, 1))]
            layers += [nn.LeakyReLU(),
                       nn.Conv2d(current_out, 1, (1, 1), stride=(
                           1, 1)),
                       ]
        self.conv = nn.Sequential(*layers)

        self.glob_pool = nn.Sequential(nn.AdaptiveMaxPool2d(
            (1, 1), return_indices=True),
        )
        final_channels = 1
#        self.conv = nn.Sequential(*layers)
        self.final = nn.Sequential(
            nn.Linear(final_channels, self.final_output))

        self.final_cands = nn.Sequential(
            nn.Linear(final_channels, self.final_output))

        if self.harmonics:
            self.final_combine = nn.AdaptiveMaxPool1d(1, return_indices=True)

        ini_conv = 0
        if ini_conv:
            self.ini_conv(1, 0.1)
        self.ini_final()

        pretrain_conv = 0
        if pretrain_conv:
            self.pretrain_conv()

        self.fft_res = 1 / (self.input_resolution *
                            int(self.min_length * (1 + self.pad_factor)))

    def forward(self, x):

        combined_pool = torch.zeros(
            (x.shape[0], self.total_height, self.min_length // 2, x.shape[1] * (self.harmonics + 1))).to(x.device)
        j = 0
        k = 0
        if not hasattr(self, 'train_harmonic'):
            self.train_harmonic = False
        if not hasattr(self, 'bin_comparison'):
            self.bin_comparison = False
        for length in self.lengths:
            with autocast(enabled=False):
                stft = compute_stft(x.float(), length, hop_length=length, norm=self.norm, crop=int(length * self.crop_factor),
                                    harmonics=self.harmonics, harmonic_downsample=self.harmonic_downsample,
                                    no_adding=self.train_harmonic, pad_factor=self.pad_factor, bin_comparison=self.bin_comparison)
            # if len(stft.shape)==4:
            #     stft = stft.unsqueeze(1)
            if self.train_harmonic:
                # print(stft.shape, 'before')
                stft = self.harmonic_net(stft.transpose(1, 4)).transpose(1, 4)
                # print(stft.shape, 'before')
                stft = stft.reshape(
                    stft.shape[0], stft.shape[1], stft.shape[2], stft.shape[3] * stft.shape[4])
                # print(stft.shape, 'after')

            # else:
            #     stft = stft.unsqueeze(1)

            out_pool = getattr(self, f"pool_{length}")(stft.unsqueeze(1))

            out_pool = out_pool[:, 0, :, :self.min_length // 2, :]
            # if self.train_harmonic:
            #     out_pool = self.harmonic_net(out_pool.transpose(2,4)).transpose(2,4)
            #     print(out_pool.shape, 'before')
            #     out_pool = out_pool.reshape(-1,-1,-1, out_pool.shape[3]*out_pool.shape[4])[:, :, :self.min_length // 2, :]
            #     print(out_pool.shape)
            # else:
            #     out_pool = out_pool[:, 0, :, :self.min_length // 2, :]
            current_height = out_pool.shape[1]
            combined_pool[:, k:k + current_height, :, :] = out_pool[:, :, :, :]
            j += 1
            k += current_height

        out_conv = self.conv(combined_pool)

        if getattr(self, 'dm0_class', 0):
            total_harms = (self.harmonics + 1)
            total_channels = out_conv.shape[3] // (self.harmonics + 1) - 1
            out_conv = out_conv[:, :, :, :-total_harms] - out_conv[:,
                                                                   :, :, -total_harms:].repeat(1, 1, 1, total_channels)

        # if getattr(self, 'dm0_class', 0):
        #    out_conv_combined = out_conv_combined[:,:,:,:,:-1] - out_conv_combined[:,:,:,:,-1][:,:,:,:,None]

        #out_conv = self.conv(stft.unsqueeze(1))

        # Normalise the output based on the standard deviation and median
        if not hasattr(self, 'norm_significance'):
            self.norm_significance = False

        if self.norm_significance:
            out_conv = (out_conv - out_conv.median(dim=2, keepdim=True)
                        [0]) / out_conv.std(dim=2, keepdim=True)

        reduce_edges = 1
        if reduce_edges:
            out_conv[:, :, :50, :] = -100
            out_conv[:, :, -50:, :] = -100

        #out_pool, max_pos = self.glob_pool(out_conv)
        if self.channel_classification:
            pool_target = [1, x.shape[1]]
        else:
            pool_target = [1, 1]

        # out_pool, max_pos = self.glob_pool(out_conv)
        # print(out_pool.shape, max_pos.shape)
        out_pool, max_pos = F.adaptive_max_pool2d(out_conv,
                                                  output_size=pool_target, return_indices=True)

        # print(out_pool)
        out_pool = out_pool.reshape(
            out_pool.shape[0] * pool_target[-1], out_pool.shape[1], 1, 1)
        # print(out_pool)
        max_pos = max_pos.reshape(
            max_pos.shape[0] * max_pos.shape[-1], max_pos.shape[1], 1, 1)
        # print(max_pos, self.channel_classification)

        max_pos = max_pos[:, :1, 0, 0].float()

        max_pos_freq = max_pos // out_conv.shape[3] % out_conv.shape[2]

        max_pos_chan_raw = max_pos % out_conv.shape[3]
        # if not self.combine_harm:
        #     max_pos_input_chan = max_pos_chan_raw // (self.harmonics+1)
        #     max_pos_harm = max_pos_chan_raw % (self.harmonics+1)
        #     harm_factor = 2 ** max_pos_harm
        #     if self.harmonic_downsample:
        #         harm_correction = harm_factor
        #     else:
        #         harm_correction = 1 / harm_factor
        # else:
        #     harm_correction = 1

        harm_correction = 1
        # print(max_pos_chan_raw, max_pos_input_chan, max_pos_harm, harm_factor)

        out_pool_final = out_pool[:, :, 0, 0]

        output = self.final(out_pool_final)

        output_period = 1 / \
            (max_pos_freq * harm_correction * self.fft_res + 0.0000001)
        # print(output)
        #output_freq = output_freq.clamp(0, 5)
        #output_freq = torch.ones((x.shape[0], 1)).to(x.device)
        output = torch.cat((output, output_period), dim=1)

        #output = output.reshape(x.shape[0], 3, pool_target[-1])
        output = output.reshape(x.shape[0], pool_target[-1],3).transpose(1,2)
        # print('after reshape in class', output[:,:2,:])

        periods = torch.arange(
            out_conv.shape[2], dtype=torch.float).to(out_conv.device)
        periods[0] = 0.001
        periods = 1 / (periods * self.fft_res)

        self.channel_correction = None
        # self.channel_correction = torch.zeros(out_conv.shape[3]).to(x.device)
        # for i in range(x.shape[1]):
        #     for j in range(self.harmonics+1):
        #         current_channel = i*(self.harmonics+1) + j
        #         if self.harmonic_downsample:
        #             correction = 1 / (2**j)
        #         else:
        #             correction = 2 **j
        #         self.channel_correction[current_channel] = correction
        # print(self.channel_correction)
        # print(output.shape, out_conv.shape, periods.shape)

        return output, (out_conv, periods)

    def ini_conv(self, mean=0, std=1):
        for child in self.conv.modules():
            if isinstance(child, nn.Conv3d):
                torch.nn.init.normal_(child.weight, mean=mean, std=std)
                torch.nn.init.zeros_(child.bias)

    def ini_final(self, weight=0.5):

        final_para = torch.nn.Parameter(torch.Tensor([[-weight], [weight]]))
        self.final[0].weight = final_para
        final_para_cands = torch.nn.Parameter(
            torch.Tensor([[-weight], [weight]]))
        self.final_cands[0].weight = final_para_cands
        # self.final[0].weight[0,:] = - weight
        # self.final[0].weight[1,:] = + weight

    def pretrain_conv(self):
        print('Pretraining stft classifier')
        batch = 1
        length = 100
        middle = 50
        steps = 3000
        optimizer = optim.Adam(self.conv.parameters(), lr=0.001)
        loss_function = nn.MSELoss()
        input_tensor = torch.zeros((batch, 1, self.total_height, length, 1))
        input_tensor[:, :, :, middle, :] = 1
        success = 0
        for i in range(steps):
            optimizer.zero_grad()
            input_slight_noise = input_tensor.clone() + torch.rand_like(input_tensor) * 0.05
            outputs = self.conv(input_slight_noise)
            loss = loss_function(outputs, input_tensor)
            loss.backward()
            optimizer.step()

            max_pos = torch.argmax(outputs)
            if np.abs(max_pos - middle) < 5 and i > 200:
                success = 1
                # plt.plot(input_slight_noise.detach().numpy()[0,0,0,:,0])
                # plt.plot(outputs.squeeze().detach().numpy())
                # plt.show()
                break
        if success:
            print(f'Pretraining stft classifier successfull after {i} steps')
        else:
            print(
                f'Pretraining stft classifier failed even after after {steps} steps')

max_stars_repo_name: SauravMaheshkar/DeepPulsarNet

max_issues_repo_name: SauravMaheshkar/DeepPulsarNet

max_forks_repo_name: SauravMaheshkar/DeepPulsarNet


text:
'''
Testing lattice models
'''

import sys
from pyscf import gto, scf, mcscf
import numpy as np
import pytest
from mdmet import orthobasis, schmidtbasis, qcsolvers, dmet
from mdmet.latticeHamiltonian import hubbard_1D, hubbard_2D_rectangular
import time

def test_hubbard1D_FCI():

	num_sites = 4
	filling = 0.5
	t = -1.0
	U = 8.0
	boundary_conditions = 'open'
	site_energy = [0.3, 0, 0.3, 0]
	no_hopping = [[1,2]] #[1]
	mf_hubbard = hubbard_1D(num_sites, filling, t, U, boundary_conditions, site_energy, no_hopping)

	mc = mcscf.CASCI(mf_hubbard, 4,4)
	EFCI = mc.kernel()[0]
	#assert np.isclose(np.round(EFCI/4,decimals=3), −0.086) #J. Chem. Phys. 143, 024107 (2015)
	return EFCI, mf_hubbard

def test_hubbard2D():

	num_sites = [8,8]
	filling = 0.25
	t = -1.0
	U = 1.0
	mf_hubbard = hubbard_2D_rectangular(num_sites, filling, t, U, boundary_conditions = 'antipbc', site_energy = None, no_hopping = None)
	return mf_hubbard

'''def test_hubbard1D_DMET():
	EFCI, mf_hubbard = test_hubbard1D_FCI()
	impClusters = [[1, 1, 0, 0],[0, 0, 1, 1]]
	symmetry = None
	solverlist = 'DMRG'
	runDMET = dmet.DMET(mf_hubbard, impClusters, symmetry, orthogonalize_method = 'lattice', schmidt_decomposition_method = 'OED', OEH_type = 'FOCK', SC_CFtype = 'FB', solver = solverlist)
	runDMET.self_consistent()
	assert np.isclose(runDMET.Energy_total, EFCI)'''


text:
from tensorflow.keras.models import Model
from tensorflow.keras.layers import Input, LSTM, Dense, Concatenate, Embedding
from tensorflow.keras.utils import plot_model
import numpy as np

def seq2seq_ext(latent_dim, num_encoder_tokens, num_decoder_tokens, ext_quant_dim, ext_cate_dim_list):
    # Define an input sequence and process it.
    encoder_inputs = []
    outputs = []
    inp_main = Input(shape=(None, num_encoder_tokens), name='en_prev_flow')
    encoder_inputs.append(inp_main)
    outputs.append(inp_main)
    inp_quant = Input(shape=(None, ext_quant_dim), name='en_ext_quant')
    encoder_inputs.append(inp_quant)
    outputs.append(inp_quant)
    i = 0
    for ext_cate_dim in ext_cate_dim_list:
        inp_cate = Input(shape=(None,), name='en_ext_cate_{}'.format(i))
        encoder_inputs.append(inp_cate)
        output = Embedding(input_dim=ext_cate_dim, output_dim=int(np.sqrt(ext_cate_dim)))(inp_cate)
        outputs.append(output)
        i += 1
    out = Concatenate()(outputs)
    fused = Dense(units=8, activation='relu')(out)
    encoder = LSTM(latent_dim, return_state=True)
    encoder_outputs, state_h, state_c = encoder(fused)
    # We discard `encoder_outputs` and only keep the states.
    encoder_states = [state_h, state_c]

    decoder_inputs = []
    outputs = []
    inp_main = Input(shape=(None, num_decoder_tokens), name='de_prev_flow')
    decoder_inputs.append(inp_main)
    outputs.append(inp_main)
    inp_quant = Input(shape=(None, ext_quant_dim), name='de_external_quant')
    decoder_inputs.append(inp_quant)
    outputs.append(inp_quant)
    i = 0
    for ext_cate_dim in ext_cate_dim_list:
        inp_cate = Input(shape=(None,), name='de_external_cate_{}'.format(i))
        decoder_inputs.append(inp_cate)
        output = Embedding(ext_cate_dim, int(np.sqrt(ext_cate_dim)))(inp_cate)
        outputs.append(output)
        i += 1
    out = Concatenate()(outputs)
    fused = Dense(units=8, activation='relu')(out)

    # Set up the decoder, using `encoder_states` as initial state.
    # decoder_inputs = Input(shape=(None, num_decoder_tokens))
    # We set up our decoder to return full output sequences,
    # and to return internal states as well. We don't use the
    # return states in the training model, but we will use them in inference.
    decoder_lstm = LSTM(latent_dim, return_sequences=True, return_state=True)
    decoder_outputs, _, _ = decoder_lstm(fused,
                                         initial_state=encoder_states)
    decoder_dense = Dense(num_decoder_tokens)
    decoder_outputs = decoder_dense(decoder_outputs)

    # Define the model that will turn
    # `encoder_input_data` & `decoder_input_data` into `decoder_target_data`
    seq2seq_model = Model(encoder_inputs + decoder_inputs, decoder_outputs)

    encoder_model = Model(encoder_inputs, encoder_states)

    decoder_state_input_h = Input(shape=(latent_dim,))
    decoder_state_input_c = Input(shape=(latent_dim,))
    decoder_states_inputs = [decoder_state_input_h, decoder_state_input_c]
    decoder_outputs, state_h, state_c = decoder_lstm(
        fused, initial_state=decoder_states_inputs)
    decoder_states = [state_h, state_c]
    decoder_outputs = decoder_dense(decoder_outputs)
    decoder_model = Model(
        decoder_inputs + decoder_states_inputs,
        [decoder_outputs] + decoder_states)
    return seq2seq_model, encoder_model, decoder_model

if __name__ == '__main__':
    import os
    os.environ["PATH"] += os.pathsep + 'C:/Program Files (x86)/Graphviz2.38/bin'

    seq2seq_model, encoder_model, decoder_model = seq2seq_ext(8, 1, 1, 5, [8, 8, 24])
    plot_model(seq2seq_model, to_file='seq2seq_ext_model.png', show_shapes=True)
    plot_model(encoder_model, to_file='encoder_ext_model.png', show_shapes=True)
    plot_model(decoder_model, to_file='decoder_ext_model.png', show_shapes=True)
    seq2seq_model.summary()

max_stars_repo_name: MALMCS/MALMCS

max_issues_repo_name: MALMCS/MALMCS

max_forks_repo_name: MALMCS/MALMCS


text:
from z3 import *
import cost as ch
import uuid
import numpy as np
import cost_model
from size_domains import MAX_HEIGHT, MAX_WIDTH, MIN_HEIGHT, MIN_WIDTH

CANVAS_WIDTH = 360
CANVAS_HEIGHT = 640

class Solution(object):
	""" Solution class keeps track of the variables and assigned values for a given design solution """
	def __init__(self):
		self.variables = []
		self.id = uuid.uuid4().hex

	def add_assigned_variable(self, variable):
		""" Add variable to the list of assigned variables """
		self.variables.append(variable)

	def parse_position_values(self, shape, element, variables, model):
		""" Parse values out of the Z3 model for x, y position and store them on the element object """
		f_x = model[variables[shape.variables.x.id]]
		f_y = model[variables[shape.variables.y.id]]
		adj_x = f_x.as_string()
		adj_y = f_y.as_string()
		adj_x = int(adj_x)
		adj_y = int(adj_y)
		element["x"] = adj_x
		element["y"] = adj_y

	def parse_grid_values(self, shape, element, variables, model):
		""" Parse values out of the Z3 model for layout grid variables and store them on the element object """
		margin = int(model[variables[shape.variables.margin.id]].as_string())
		baseline_grid = int(model[variables[shape.variables.baseline_grid.id]].as_string())
		gutter_width = int(model[variables[shape.variables.gutter_width.id]].as_string())
		column_width = int(model[variables[shape.variables.column_width.id]].as_string())
		columns = int(model[variables[shape.variables.columns.id]].as_string())

		element["margin"] = margin
		element["baseline_grid"] = baseline_grid
		element["columns"] = columns
		element["column_width"] = column_width
		element["gutter_width"] = gutter_width
		element["grid_layout"] = [margin, columns, gutter_width, column_width]

	def parse_size_values(self, shape, element, variables, model):
		""" Parse values out of the Z3 model for height,width and store them on the element object """
		height = model[variables[shape.computed_height()]].as_string()
		width = model[variables[shape.computed_width()]].as_string()
		height = int(height)
		width = int(width)
		element["width"] = width
		element["height"] = height
		element["size"] = [width, height]

	def parse_container_values(self, shape, element, variables, model):
		""" Parse values out of the Z3 model for container properties and store them on the element object """
		arrangement = model[variables[shape.variables.arrangement.id]].as_string()
		alignment = model[variables[shape.variables.alignment.id]].as_string()
		padding = model[variables[shape.variables.padding.id]].as_string()
		group_alignment = model[variables[shape.variables.group_alignment.id]].as_string()

		element["arrangement"] = int(arrangement)
		element["alignment"] = int(alignment)
		element["padding"] = int(padding)
		element["group_alignment"] = int(group_alignment)

	def parse_column_values(self, shape, element, variables, model):
		""" Parse values out of the Z3 model for left_column, right_column, and canvas_alignment
		 and store them on the element object """
		left_column = model[variables[shape.variables.left_column.id]].as_string()
		element["left_column"] = int(left_column)

		right_column = model[variables[shape.variables.right_column.id]].as_string()
		element["right_column"] = int(right_column)

		canvas_alignment = model[variables[shape.variables.canvas_alignment.id]].as_string()
		element["canvas_alignment"] = int(canvas_alignment)

	def parse_alternate_values(self, shape, element, variables, model):
		""" Parse values out of the Z3 model for alternate variable and store it on the element object """
		alternate = model[variables[shape.variables.alternate.id]].as_string()
		element["alternate"] = alternate

	def parse_size_multiplier_values(self, shape, element, variables, model):
		""" Parse values out of the Z3 model for size_factor multiplier variable and store it on the element object """
		size_factor = model[variables[shape.variables.size_factor.id]].as_string()
		size_factor = int(size_factor)
		element["size_factor"] = size_factor

	def parse_values(self, shape, element, variables, model):
		""" Parses values out of the stored Z3 model and stores them on the element object.
			What gets parsed depends on the element/shape type """
		self.parse_position_values(shape, element, variables, model)

		if shape.type == "canvas":
			self.parse_grid_values(shape, element, variables, model)
		elif shape.type == "container" or shape.type == "leaf":
			self.parse_size_values(shape, element, variables, model)

			if shape.has_columns:
				self.parse_column_values(shape, element, variables, model)
				element["canvas_child"] = True

			if shape.type == "container":
				self.parse_container_values(shape, element, variables, model)
			elif shape.type == "leaf":
				self.parse_size_multiplier_values(shape, element, variables, model)
				element["size_combo"] = [element["width"], element["height"], element["size_factor"]]

				if shape.is_alternate:
					self.parse_alternate_values(shape, element, variables, model)

	def parse_variables_from_model(self, model):
		""" Return a collection of variables parsed out of the Z3 model """
		variables = dict()
		num_vars = len(model)

		decls = model.decls()
		for i in range(num_vars):
			var = decls[i]
			name = var.name()
			if name != 'div0' and name != 'mod0':
				variables[var.name()] = var()
		return variables

	def process_hierarchy(self, tree, variables, model, cost_matrix, cost_metrics):
		""" Process element tree to get the solved variable values out of the model into corresponding elements
			And compute corresponding cost metrics """
		if tree.element is not None:
			element = copy.deepcopy(tree.element)

			# Get the computed values from the model
			self.parse_values(tree, element, variables, model)

			# Emphasis cost function calculations
			if tree.type != "canvas":
				height = element["height"]
				width = element["width"]
				x = element["x"]
				y = element["y"]
				if tree.importance == "high":
					# Cost will be the distance from the maximum size
					cost_metrics['importance_change'] += (height - tree.orig_height)
					cost_metrics['importance_change'] += (width - tree.orig_width)
					cost_metrics['importance_max'] += (MAX_HEIGHT - tree.orig_height)
					cost_metrics['importance_max'] += (MAX_WIDTH - tree.orig_width)

					# Compute the distance of the shape from the center of the canvas
					cost_metrics['distance_cost'] += ch.compute_distance_from_center(x, y, width, height)
				elif tree.importance == "low":
					# Used for computing importance cost
					cost_metrics['importance_change'] += (tree.orig_height - height)
					cost_metrics['importance_change'] += (tree.orig_width - width)
					cost_metrics['importance_max'] += (tree.orig_height - MIN_HEIGHT)
					cost_metrics['importance_max'] += (tree.orig_width - MIN_WIDTH)

					cost_metrics['distance_cost'] += ch.compute_inverse_distance_from_center(x, y, width, height)

				# Only the locations of leaf level shapes to compute the symmetry cost
				cost_matrix[y:(y+height-1),x:(x+width-1)] = 1

			if hasattr(tree, "children") and len(tree.children):
				element['children'] = []
				for child in tree.children:
					child_element = self.process_hierarchy(child, variables, model, cost_matrix, cost_metrics)
					if child_element is not None:
						element['children'].append(child_element)

			return element

	def convert_to_json(self, tree, model):
		""" Given the element tree and the Z3 model, parse the assigned values out of the model and store them
			in a JSON object for the solution

			Parameters:
				tree: Element
		"""
		sln = dict()
		cost_matrix = np.zeros((CANVAS_HEIGHT, CANVAS_WIDTH), dtype=np.uint8)
		new_elements = dict()

		# Cost variables
		cost_metrics = dict()
		cost_metrics['importance_change'] = 0
		cost_metrics['distance_cost'] = 0
		cost_metrics['importance_max'] = 0

		variables = self.parse_variables_from_model(model)

		element_tree = self.process_hierarchy(tree, variables, model, cost_matrix, cost_metrics)

		# Old cost function metrics.
		# symmetry_cost = ch.compute_symmetry_cost(cost_matrix)
		# importance_change = cost_metrics['importance_change']
		# importance_max = cost_metrics['importance_max']
		# distance_cost = cost_metrics['distance_cost']

		# importance_cost = ch.compute_importance_cost(importance_change, importance_max)
		# cost = ch.compute_weighted_cost(symmetry_cost, importance_cost, distance_cost)

		# Compute the cost model score for the element tree.
		new_cost = cost_model.compute_cost(element_tree)
		cost = new_cost

		print("Total cost: " + str(cost))
		sln["elements"] = element_tree
		sln["id"] = self.id
		sln["cost"] = cost # Assign the computed cost here.
		sln["valid"] = True
		sln["saved"] = 0
		sln["conflicts"] = []

		return sln

max_stars_repo_name: uwplse/scout

max_issues_repo_name: uwplse/scout

max_forks_repo_name: uwplse/scout


text:
import torch
import os
import glob
import numpy as np
from torch import nn
from PIL import Image
from torch.utils.data import Dataset
from torchvision import transforms
from utils import *

def round_nearest(x, p):
    return ((x - 1) // p + 1) * p

class ADE20K_Dataset(Dataset):
    """
    ADE20K dataset with anime style transfer
    """
    def __init__(self, root, transform=None, status="Train"):
        super(ADE20K_Dataset, self).__init__()
        self.root = root
        self.transform = transform
        self.label_downsample_rate = 256 / 21
        if status == "Train":
            self.dir = os.path.join(self.root, "images", "training")
            self.styles = ("Hayao", "Hosoda")
        else:
            self.dir = os.path.join(self.root, "images", "validation")
            self.styles = ("Hayao", "Shinkai")
        self.classes_path = concat_subfolder(get_subfolder(self.dir))
        # save mask as a dict because one mask corresponds to two images
        self.images_path, self.masks_path = [], {}
        for folder in self.classes_path:
            for style in self.styles:
                style = "*_" + style + ".jpg"
                for img in glob.glob(os.path.join(folder, style)):
                    self.images_path.append(img)
            for mask in glob.glob(os.path.join(folder, "*_seg.png")):
                img_name = mask.split("/")[-1].replace("_seg.png", "")
                self.masks_path[img_name] = mask

    def __getitem__(self, index):
        """
        get image and mask,
        :param index:
        :return:
        """
        img_path = self.images_path[index]
        img_name = img_path.split("/")[-1]
        for style in self.styles:
            style = "_" + style + ".jpg"
            img_name = img_name.replace(style, "")
        mask_path = self.masks_path[img_name]
        img = Image.open(img_path).convert("RGB")
        mask = Image.open(mask_path).convert("L")
        # resize mask into the same dim with CartoonGAN outputs
        mask = mask.resize((224, 224), resample=Image.NEAREST)
        # convert mask img into labels
        mask = np.array(mask) // self.label_downsample_rate
        mask = torch.from_numpy(mask).int().long()
        # apply transform
        if self.transform is None:
            self.transform = transforms.Compose([
                transforms.Resize((224, 224)),
                transforms.ToTensor(),
                # compute normalizing terms in prepare_dataset.py
                transforms.Normalize(mean=[0.5169, 0.4734, 0.4078], std=[0.2075, 0.2059, 0.1907])
            ])
        img = self.transform(img)
        return img, mask

    def __len__(self):
        return len(self.images_path)


text:
# -*- coding: utf-8 -*-

# %% [markdown]
# # Wavelets
#
# ## Importando bibliotecas

# %%
import os
import pywt
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import statsmodels.robust.scale as robust
from scipy import signal
from scipy import stats as sc

# %%

def getListOfFiles(dirName):
    '''
    create a list of file and sub directories names in the given directory
    '''
    listOfFile = os.listdir(dirName)
    allFiles = list()
    # Iterate over all the entries
    for entry in listOfFile:
        # Create full path
        fullPath = os.path.join(dirName, entry)
        # If entry is a directory then get the list of files in this directory
        if os.path.isdir(fullPath):
            allFiles = allFiles + getListOfFiles(fullPath)
        else:
            allFiles.append(fullPath)

    return allFiles

# %%

def EnergyPercent(Ac, Dc, Original):

    E = np.empty((1, np.size(Dc, 1)+1), float)
    Ep = np.zeros((1, np.size(Dc, 1)+1), float)

    Et = np.sum(np.square(Original))

    for i in range(0, np.size(Dc, 1)):
        E[0, i] = np.sum(np.square(Dc[0, i]))

    E[0, i+1] = np.sum(np.square(Ac))

    Ep = E/Et

    return Ep

# %%
# Ler amostra do arquivo
path = 'C:\Renomeado2'  # Confgurar caminho para pasta contendo as amostras

Files = getListOfFiles(path)

amostra = np.empty((100000, 4), float)

N_decomp_levels = 5  # Configurar níveis de decomposição aqui

n = N_decomp_levels+2

nCols = (n-1)*6+3

DataBase = np.zeros((len(Files), nCols), float)

# Configurar quantas amostras devem ser "saltadas" para subamostragem
x_array = [1]

for x_idx in range(0, len(x_array)):

    for i in range(0, len(Files)):
        fp = open(Files[i])

        content = fp.readlines()
        x = np.array(content[21:])

        for j in range(0, 100000):
            amostra_strings = x[j].split(',')
            amostra[j] = np.array(amostra_strings)

        #  Remover a media
        mean = np.mean(amostra[:, 1:5], 0)
        amostra[:, 1:5] = amostra[:, 1:5]-mean

        x = x_array[x_idx]
        # Filtragem digital
        fs = 10000
        fc = fs/(2*x)-300  # Cut-off frequency of the filter
        w = fc / (fs / 2)  # Normalize the frequency
        b, a = signal.butter(5, w, 'low')
        output = signal.filtfilt(b, a, amostra[:, 3])
        # output = amostra[:,3]

        # Subamostragem
        t = 10  # Configurar tempo total em segundos da amostra cortada

        n_pontos = int(t*10000)

        fluxo = output[0:n_pontos:x]

        input = fluxo

        Desvio = np.zeros((1, n-1), float)
        MeanAD = np.zeros((1, n-1), float)
        MedianAD = np.zeros((1, n-1), float)
        Energia = np.zeros((1, n-1), float)
        Kurtosis = np.zeros((1, n-1), float)
        Skewness = np.zeros((1, n-1), float)
        cD = np.empty((1, n-2), object)

        for m in range(0, n-2):
            cA, cD[0, m] = pywt.dwt(input, 'db2')
            Desvio[0, m] = np.std(cD[0, m])
            MeanAD[0, m] = pd.DataFrame(cD[0, m]).mad()
            MedianAD[0, m] = robust.mad(cD[0, m])
            Kurtosis[0, m] = sc.kurtosis(cD[0, m])
            Skewness[0, m] = sc.skew(cD[0, m])
            input = cA

        Desvio[0, m+1] = np.std(cA)
        MeanAD[0, m+1] = pd.DataFrame(cA).mad()
        MedianAD[0, m+1] = robust.mad(cA)
        Kurtosis[0, m+1] = sc.kurtosis(cA)
        Skewness[0, m+1] = sc.skew(cA)

        Energia = EnergyPercent(cA, cD, fluxo)

        # sObject=slice(31,33);

        # Freq=int(Files[i][sObject]);
        Freq = 0

        # sObject=slice(34,37);

        # Load=int(Files[i][sObject]);
        Load = 0

        # sObject=slice(23,24);

        # Classe=int(Files[i][sObject]);
        Classe = 7

        # sObject=slice(26,30);

        # Index=int(Files[i][sObject]);

        Pattern = np.zeros((1, nCols-3), float)
        np.concatenate(
            (Desvio, MeanAD, MedianAD, Kurtosis, Skewness, Energia),
            out=Pattern,
            axis=1
        )

        DataBase[i, 0:nCols-3] = Pattern

        DataBase[i, nCols-3:] = [Freq, Classe, Load]

        fp.close()
print(i)

# %%
# FFT=np.fft.fft(fluxo);
#
# FFT_abs=abs(FFT)[0:50000]
#
# f1=np.linspace(0,5000,50000);
#
# FFT_a2=np.fft.fft(cA)
#
# FFT_abs_a2=abs(FFT_a2)[0:12500];
#
# f2=np.linspace(0,1250,12500);
#
#
# FFT_d2=np.fft.fft(cD[0,1])
#
# FFT_abs_d2=abs(FFT_d2)[0:12500];
#
# f3=np.linspace(1251,2500,12500);
#
# FFT_d1=np.fft.fft(cD[0,0])
#
# FFT_abs_d1=abs(FFT_d1)[0:25000];
#
# f4=np.linspace(2500,5000,25000);
#
#
#
#
# plt.figure()
# plt.plot(f1,FFT_abs)
#
# plt.figure()
# plt.plot(f2,FFT_abs_a2)
#
# plt.figure()
# plt.plot(f3,FFT_abs_d2)
#
# plt.figure()
# plt.plot(f4,FFT_abs_d1)
#
# new_freq=0.001*fs/x
#np.savetxt('Estudo Redução da frequencia 3//'+str(new_freq)+"kHz-10s-db2-5niveis.csv", DataBase, delimiter=",")

#
#
# fig=plt.figure();
#    #plt.plot(fluxo)
#ax = fig.add_subplot(n, 1, 1)
# ax.plot(fluxo)
#ax.set_title('Original', fontsize=10)
#ax = fig.add_subplot(n, 1, 2)
# ax.plot(cD[0,0])
#ax.set_title('Detalhe 1', fontsize=10)
#ax = fig.add_subplot(n, 1, 3)
# ax.plot(cD[0,1])
#ax.set_title('Detalhe 2', fontsize=10)
#ax = fig.add_subplot(n, 1, 4)
#    ax.plot(cD3)
#    ax.set_title('Detalhe 3', fontsize=10)
#    ax = fig.add_subplot(n, 1, 5)
#    ax.plot(cD4)
#    ax.set_title('Detalhe 4', fontsize=10)
#    ax = fig.add_subplot(n, 1, 6)
#    ax.plot(cD5)
#    ax.set_title('Detalhe 5', fontsize=10)
#    ax = fig.add_subplot(n, 1, 7)
#    ax.plot(cD6)
#    ax.set_title('Detalhe 6', fontsize=10)
#    ax = fig.add_subplot(n, 1, 8)
# ax.plot(cA)
#ax.set_title('Aproximação 2', fontsize=10)
#
#    plt.show()

max_stars_repo_name: newtmagalhaes/Machine-Learning

max_issues_repo_name: newtmagalhaes/Machine-Learning

max_forks_repo_name: newtmagalhaes/Machine-Learning

reason: import numpy,from scipy,import statsmodels


text:
"""This module contains any tests which cause memory leaks."""
import weakref

import numpy as np

def test_pyvistandarray_assign(sphere):
    sphere.point_data['data'] = range(sphere.n_points)

    # this might leave a reference behind if we don't properly use the pointer
    # to the vtk array.
    sphere.point_data['data'] = sphere.point_data['data']

def test_pyvistandarray_strides(sphere):
    sphere['test_scalars'] = sphere.points[:, 2]
    assert np.allclose(sphere['test_scalars'], sphere.points[:, 2])

def test_complex_collection(plane):
    name = 'my_data'
    data = np.random.random((plane.n_points, 2)).view(np.complex128).ravel()
    plane.point_data[name] = data

    # ensure shallow copy
    assert np.shares_memory(plane.point_data[name], data)

    # ensure data remains but original numpy object does not
    ref = weakref.ref(data)
    data_copy = data.copy()
    del data
    assert np.allclose(plane.point_data[name], data_copy)

    assert ref() is None

max_stars_repo_name: sthagen/pyvista

max_issues_repo_name: sthagen/pyvista

max_forks_repo_name: sthagen/pyvista


text:
import os
import torch
import torch.utils.data as data
import data.utils as data_utils
from data import single_dataset
from data.custom_dataset_dataloader import CustomDatasetDataLoader
from data.class_aware_dataset_dataloader import ClassAwareDataLoader
from data.image_folder import make_dataset_with_labels
from config.config import cfg
import numpy as np
from sklearn.model_selection import train_test_split
from scipy.io import loadmat

# +
def prepare_data_Logit(seed=None):
    dataloaders = {}
    train_transform = data_utils.get_transform(True)
    test_transform = data_utils.get_transform(False)

    source = cfg.DATASET.SOURCE_NAME
    target = cfg.DATASET.TARGET_NAME
    dataroot_S = os.path.join(cfg.DATASET.DATAROOT, source, "train_images")
    dataroot_T = os.path.join(cfg.DATASET.DATAROOT, target)

    classes = list(range(10))

    item_S = data_utils.read_image_list(dataroot_S)
    data_paths_S = [ item[0] for item in item_S ]
    data_labels_S = [ item[1] for item in item_S ]
    batch_size = cfg.TRAIN.SOURCE_BATCH_SIZE
    dataset_type = 'SingleDataset'
    print('Building %s dataloader...' % source)

    X_train, X_test, y_train, y_test = train_test_split( data_paths_S, data_labels_S,
                                                        test_size=0.025, random_state=seed,
                                                        stratify=data_labels_S)
    dataloaders[source] = CustomDatasetDataLoader(dataset_root=dataroot_S, dataset_type=dataset_type,
                                                  batch_size=batch_size, transform=train_transform,
                                                  data_paths=X_test, data_labels=y_test,
                                                  train=True, num_workers=cfg.NUM_WORKERS,
                                                  classnames=classes)
    '''
    dataloaders[source] = CustomDatasetDataLoader(dataset_root=dataroot_S, dataset_type=dataset_type,
                                                  batch_size=batch_size, transform=train_transform,
                                                  data_paths=data_paths_S, data_labels=data_labels_S,
                                                  train=True, num_workers=cfg.NUM_WORKERS,
                                                  classnames=classes)
    '''
    item_T = data_utils.read_image_list(os.path.join(dataroot_T, "train_images"))
    data_paths_T = [ item[0] for item in item_T ]
    data_labels_T = [ item[1] for item in item_T ]
    batch_size = cfg.TRAIN.TARGET_BATCH_SIZE
    dataset_type = 'SingleDataset'
    print('Building %s dataloader...' % target)
    '''
    X_train, X_test, y_train, y_test = train_test_split( data_paths_T, data_labels_T,
                                                        test_size=0.025, random_state=seed,
                                                        stratify=data_labels_T)
    dataloaders[target] = CustomDatasetDataLoader(dataset_root=dataroot_T, dataset_type=dataset_type,
                                                  batch_size=batch_size, transform=train_transform,
                                                  data_paths=X_test, data_labels=y_test,
                                                  train=True, num_workers=cfg.NUM_WORKERS,
                                                  classnames=classes)

    '''
    dataloaders[target] = CustomDatasetDataLoader(dataset_root=dataroot_T, dataset_type=dataset_type,
                                                  batch_size=batch_size, transform=train_transform,
                                                  data_paths=data_paths_T, data_labels=data_labels_T,
                                                  train=True, num_workers=cfg.NUM_WORKERS,
                                                  classnames=classes)

    dataloaders[target+'_labeled'] = None

    item_test = data_utils.read_image_list(os.path.join(dataroot_T, "test_images"))
    data_paths_test = [ item[0] for item in item_test ]
    data_labels_test = [ item[1] for item in item_test ]
    batch_size = cfg.TEST.BATCH_SIZE
    dataset_type = cfg.TEST.DATASET_TYPE
    test_domain = cfg.TEST.DOMAIN if cfg.TEST.DOMAIN != "" else target
    dataroot_test = os.path.join(cfg.DATASET.DATAROOT, test_domain)
    print('Building %s test dataloader...' % test_domain)
    '''
    X_train, X_test, y_train, y_test = train_test_split( data_paths_test, data_labels_test,
                                                        test_size=0.025, random_state=seed,
                                                        stratify=data_labels_test)
    dataloaders['test'] = CustomDatasetDataLoader(dataset_root=dataroot_test, dataset_type=dataset_type,
                                                  batch_size=batch_size, transform=test_transform,
                                                  data_paths=X_test, data_labels=y_test,
                                                  train=False, num_workers=cfg.NUM_WORKERS,
                                                  classnames=classes)

    '''
    dataloaders['test'] = CustomDatasetDataLoader(dataset_root=dataroot_test, dataset_type=dataset_type,
                                                  batch_size=batch_size, transform=test_transform,
                                                  data_paths=data_paths_test, data_labels=data_labels_test,
                                                  train=False, num_workers=cfg.NUM_WORKERS,
                                                  classnames=classes)

    idxs_lb = np.zeros(len(data_labels_T), dtype=bool)

    print('number of labeled pool: {}'.format(len(dataloaders[source].dataset)))
    print('number of unlabeled pool: {}'.format(len(dataloaders[target].dataset)))
    print('number of testing pool: {}'.format(len(dataloaders['test'].dataset)))
    return dataloaders, data_paths_T, data_labels_T, idxs_lb, classes

def prepare_data_Office31(seed=None):
    dataloaders = {}
    train_transform = data_utils.get_transform(True)
    test_transform = data_utils.get_transform(False)

    source = cfg.DATASET.SOURCE_NAME
    target = cfg.DATASET.TARGET_NAME
    dataroot_S = os.path.join(cfg.DATASET.DATAROOT, source)
    dataroot_T = os.path.join(cfg.DATASET.DATAROOT, target)

    with open(os.path.join(cfg.DATASET.DATAROOT, 'category.txt'), 'r') as f:
        classes = f.readlines()
        classes = [c.strip() for c in classes]
    assert(len(classes) == cfg.DATASET.NUM_CLASSES)

    batch_size = cfg.TRAIN.SOURCE_BATCH_SIZE
    dataset_type = 'SingleDataset'
    print('Building %s dataloader...' % source)
    dataloaders[source] = CustomDatasetDataLoader(dataset_root=dataroot_S, dataset_type=dataset_type,
                                                  batch_size=batch_size, transform=train_transform,
                                                  train=True, num_workers=cfg.NUM_WORKERS,
                                                  classnames=classes)
    # split target dataset
    target_data_paths, target_data_labels = make_dataset_with_labels(dataroot_T, classes)
    X_train, X_test, y_train, y_test = train_test_split( target_data_paths, target_data_labels,
                                                        test_size=cfg.TEST.TEST_PERCENTAGE, random_state=seed,
                                                        stratify=target_data_labels)
    batch_size = cfg.TRAIN.TARGET_BATCH_SIZE
    dataset_type = 'SingleDataset'
    print('Building %s dataloader...' % target)
    dataloaders[target] = CustomDatasetDataLoader(dataset_root=dataroot_T, dataset_type=dataset_type,
                                                  batch_size=batch_size, transform=train_transform,
                                                  data_paths=X_train, data_labels=y_train,
                                                  train=True, num_workers=cfg.NUM_WORKERS,
                                                  classnames=classes)

    dataloaders[target+'_labeled'] = None

    batch_size = cfg.TEST.BATCH_SIZE
    dataset_type = cfg.TEST.DATASET_TYPE
    test_domain = cfg.TEST.DOMAIN if cfg.TEST.DOMAIN != "" else target
    dataroot_test = os.path.join(cfg.DATASET.DATAROOT, test_domain)
    print('Building %s test dataloader...' % test_domain)
    dataloaders['test'] = CustomDatasetDataLoader(dataset_root=dataroot_test, dataset_type=dataset_type,
                                                  batch_size=batch_size, transform=test_transform,
                                                  data_paths=X_test, data_labels=y_test,
                                                  train=False, num_workers=cfg.NUM_WORKERS,
                                                  classnames=classes)

    idxs_lb = np.zeros(len(y_train), dtype=bool)

    print('number of labeled pool: {}'.format(len(dataloaders[source].dataset)))
    print('number of unlabeled pool: {}'.format(len(dataloaders[target].dataset)))
    print('number of testing pool: {}'.format(len(dataloaders['test'].dataset)))
    return dataloaders, X_train, y_train, idxs_lb, classes

def prepare_data_DomainNet(seed=None):
    dataloaders = {}
    train_transform = data_utils.get_transform(True)
    test_transform = data_utils.get_transform(False)

    source = cfg.DATASET.SOURCE_NAME
    target = cfg.DATASET.TARGET_NAME
    dataroot_S = os.path.join(cfg.DATASET.DATAROOT, source)
    dataroot_T = os.path.join(cfg.DATASET.DATAROOT, target)
    with open(os.path.join(cfg.DATASET.DATAROOT, 'category.txt'), 'r') as f:
        classes = f.readlines()
        classes = [c.strip() for c in classes]
    assert(len(classes) == cfg.DATASET.NUM_CLASSES)

    batch_size = cfg.TRAIN.SOURCE_BATCH_SIZE
    dataset_type = 'SingleDataset'
    print('Building %s dataloader...' % source)
    dataloaders[source] = CustomDatasetDataLoader(dataset_root=dataroot_S, dataset_type=dataset_type,
                                                  batch_size=batch_size, transform=train_transform,
                                                  train=True, num_workers=cfg.NUM_WORKERS,
                                                  classnames=classes)

    with open(os.path.join(dataroot_T, target+'_train.txt'), 'r') as f:
        image_index = [os.path.join(cfg.DATASET.DATAROOT, x.split(' ')[0]) for x in f.readlines()]
    with open(os.path.join(dataroot_T, target+'_train.txt'), 'r') as f:
        label_list = []
        selected_list = []
        for ind, x in enumerate(f.readlines()):
            label = x.split(' ')[1].strip()
            label_list.append(int(label))
            selected_list.append(ind)
        image_index = np.array(image_index)
        y_train = np.array(label_list)
    X_train = image_index[selected_list]

    batch_size = cfg.TRAIN.TARGET_BATCH_SIZE
    dataset_type = 'SingleDataset'
    print('Building %s dataloader...' % target)
    dataloaders[target] = CustomDatasetDataLoader(dataset_root=dataroot_T, dataset_type=dataset_type,
                                                  batch_size=batch_size, transform=train_transform,
                                                  data_paths=X_train, data_labels=y_train,
                                                  train=True, num_workers=cfg.NUM_WORKERS,
                                                  classnames=classes)

    with open(os.path.join(dataroot_T, target+'_test.txt'), 'r') as f:
        image_index = [os.path.join(cfg.DATASET.DATAROOT, x.split(' ')[0]) for x in f.readlines()]
    with open(os.path.join(dataroot_T, target+'_test.txt'), 'r') as f:
        label_list = []
        selected_list = []
        for ind, x in enumerate(f.readlines()):
            label = x.split(' ')[1].strip()
            label_list.append(int(label))
            selected_list.append(ind)
        image_index = np.array(image_index)
        y_test = np.array(label_list)
    X_test = image_index[selected_list]

    dataloaders[target+'_labeled'] = None

    batch_size = cfg.TEST.BATCH_SIZE
    dataset_type = cfg.TEST.DATASET_TYPE
    test_domain = cfg.TEST.DOMAIN if cfg.TEST.DOMAIN != "" else target
    dataroot_test = os.path.join(cfg.DATASET.DATAROOT, test_domain)
    print('Building %s test dataloader...' % test_domain)
    dataloaders['test'] = CustomDatasetDataLoader(dataset_root=dataroot_test, dataset_type=dataset_type,
                                                  batch_size=batch_size, transform=test_transform,
                                                  data_paths=X_test, data_labels=y_test,
                                                  train=False, num_workers=cfg.NUM_WORKERS,
                                                  classnames=classes)

    print('number of labeled pool: {}'.format(len(dataloaders[source].dataset)))
    print('number of unlabeled pool: {}'.format(len(dataloaders[target].dataset)))
    print('number of testing pool: {}'.format(len(dataloaders['test'].dataset)))

    # generate initial labeled pool
    idxs_lb = np.zeros(len(y_train), dtype=bool)

    return dataloaders, X_train, y_train, idxs_lb, classes

def prepare_data_AADA(seed=None):
    dataloaders = {}
    train_transform = data_utils.get_transform(True)
    test_transform = data_utils.get_transform(False)

    source = cfg.DATASET.SOURCE_NAME
    target = cfg.DATASET.TARGET_NAME
    dataroot_S = os.path.join(cfg.DATASET.DATAROOT, source)
    dataroot_T = os.path.join(cfg.DATASET.DATAROOT, target)

    with open(os.path.join(cfg.DATASET.DATAROOT, 'category.txt'), 'r') as f:
        classes = f.readlines()
        classes = [c.strip() for c in classes]
    assert(len(classes) == cfg.DATASET.NUM_CLASSES)

    batch_size = cfg.TRAIN.SOURCE_BATCH_SIZE
    dataset_type = 'SingleDataset'
    dataloaders[source] = CustomDatasetDataLoader(
                dataset_root=dataroot_S, dataset_type=dataset_type,
                batch_size=batch_size, transform=train_transform,
                train=True, num_workers=cfg.NUM_WORKERS,
                classnames=classes)

    target_dataset = getattr(single_dataset, dataset_type)(root=dataroot_T)
    target_dataset.initialize(root=dataroot_T, classnames=classes)
    X_train, X_test, y_train, y_test = train_test_split( target_dataset.data_paths, target_dataset.data_labels, test_size=0.33, random_state=0, stratify=target_dataset.data_labels)
    batch_size = cfg.TRAIN.TARGET_BATCH_SIZE
    dataset_type = 'SingleDataset'
    target_test_dataset = getattr(single_dataset, dataset_type)(root=dataroot_T)
    target_test_dataset.initialize_data(data_paths=X_test, data_labels=y_test, transform=test_transform)
    dataset_type = 'SingleDataset'
    target_train_dataset = getattr(single_dataset, dataset_type)()
    target_train_dataset.initialize_data(data_paths=X_train, data_labels=y_train, transform=train_transform)

    dataloaders[target] = CustomDatasetDataLoader2(
                dataset=target_train_dataset, batch_size=batch_size,
                train=True, num_workers=cfg.NUM_WORKERS,
                classnames=classes)

    dataloaders[target+'_label'] = None

    source_x_train, source_y_train = make_dataset_with_labels(dataroot_S, classes)
    X_tr = [*source_x_train, *X_train]
    Y_tr = [*source_y_train, *y_train]
    # start experiment
    NUM_INIT_LB = len(source_y_train)
    n_pool = len(Y_tr)
    n_test = len(y_test)
    print('number of labeled pool: {}'.format(NUM_INIT_LB))
    print('number of unlabeled pool: {}'.format(n_pool - NUM_INIT_LB))
    print('number of testing pool: {}'.format(n_test))
    X_tr = X_train
    Y_tr = y_train

    # generate initial labeled pool
    idxs_lb = np.zeros(len(y_train), dtype=bool)
    handler = get_handler()

    batch_size = cfg.TEST.BATCH_SIZE
    dataset_type = cfg.TEST.DATASET_TYPE
    test_domain = cfg.TEST.DOMAIN if cfg.TEST.DOMAIN != "" else target
    dataroot_test = os.path.join(cfg.DATASET.DATAROOT, test_domain)
    dataloaders['test'] = CustomDatasetDataLoader2(
                    dataset=target_test_dataset, batch_size=batch_size,
                    train=False, num_workers=cfg.NUM_WORKERS,
                    classnames=classes)

    return dataloaders, X_tr, Y_tr, idxs_lb, handler, train_transform, test_transform
def prepare_data_ACAN_improve_clustering(seed=None):
    dataloaders = {}
    train_transform = data_utils.get_transform(True)
    test_transform = data_utils.get_transform(False)

    source = cfg.DATASET.SOURCE_NAME
    target = cfg.DATASET.TARGET_NAME
    dataroot_S = os.path.join(cfg.DATASET.DATAROOT, source)
    dataroot_T = os.path.join(cfg.DATASET.DATAROOT, target)

    with open(os.path.join(cfg.DATASET.DATAROOT, 'category.txt'), 'r') as f:
        classes = f.readlines()
        classes = [c.strip() for c in classes]
    assert(len(classes) == cfg.DATASET.NUM_CLASSES)

    # for clustering
    batch_size = cfg.CLUSTERING.SOURCE_BATCH_SIZE
    dataset_type = 'SingleDataset'
    print('Building clustering_%s dataloader...' % source)
    dataloaders['clustering_' + source] = CustomDatasetDataLoader(
                dataset_root=dataroot_S, dataset_type=dataset_type,
                batch_size=batch_size, transform=train_transform,
                train=False, num_workers=cfg.NUM_WORKERS,
                classnames=classes)

    batch_size = cfg.CLUSTERING.TARGET_BATCH_SIZE
    print('Building clustering_%s dataloader...' % target)
    # dataset_type = cfg.CLUSTERING.TARGET_DATASET_TYPE
    dataset_type = 'SingleDataset'
    # Random split
    target_dataset = getattr(single_dataset, dataset_type)(root=dataroot_T)
    target_dataset.initialize(root=dataroot_T, classnames=classes)
    X_train, X_test, y_train, y_test = train_test_split( target_dataset.data_paths, target_dataset.data_labels, test_size=0.33, random_state=0, stratify=target_dataset.data_labels)
    dataset_type = 'SingleDataset'
    target_test_dataset = getattr(single_dataset, dataset_type)(root=dataroot_T)
    target_test_dataset.initialize_data(data_paths=X_test, data_labels=y_test, transform=test_transform)
    # dataset_type = 'SingleDatasetWithoutLabel'
    dataset_type = 'SingleDataset'
    target_train_dataset = getattr(single_dataset, dataset_type)()
    target_train_dataset.initialize_data(data_paths=X_train, data_labels=y_train, transform=train_transform)

    dataloaders['clustering_' + target] = CustomDatasetDataLoader2(
                dataset=target_train_dataset, batch_size=batch_size,
                train=False, num_workers=cfg.NUM_WORKERS,
                classnames=classes)

    source_x_train, source_y_train = make_dataset_with_labels(dataroot_S, classes)
    X_tr = [*source_x_train, *X_train]
    Y_tr = [*source_y_train, *y_train]
    # start experiment
    NUM_INIT_LB = len(source_y_train)
    n_pool = len(Y_tr)
    n_test = len(y_test)
    print('number of labeled pool: {}'.format(NUM_INIT_LB))
    print('number of unlabeled pool: {}'.format(n_pool - NUM_INIT_LB))
    print('number of testing pool: {}'.format(n_test))
    X_tr = X_train
    Y_tr = y_train

    # generate initial labeled pool
    idxs_lb = np.zeros(len(y_train), dtype=bool)
    handler = get_handler()

    # class-agnostic source dataloader for supervised training
    batch_size = cfg.TRAIN.SOURCE_BATCH_SIZE
    dataset_type = 'SingleDataset'
    print('Building %s dataloader...' % source)
    dataloaders[source] = CustomDatasetDataLoader(
                dataset_root=dataroot_S, dataset_type=dataset_type,
                batch_size=batch_size, transform=train_transform,
                train=True, num_workers=cfg.NUM_WORKERS,
                classnames=classes)

    # initialize the categorical dataloader
    dataset_type = 'CategoricalSTDataset'
    source_batch_size = cfg.TRAIN.SOURCE_CLASS_BATCH_SIZE
    target_batch_size = cfg.TRAIN.TARGET_CLASS_BATCH_SIZE
    print('Building categorical dataloader...')
    dataloaders['categorical'] = ClassAwareDataLoader(
                dataset_type=dataset_type,
                source_batch_size=source_batch_size,
                target_batch_size=target_batch_size,
                source_dataset_root=dataroot_S,
                transform=train_transform,
                classnames=classes,
                num_workers=cfg.NUM_WORKERS, seed=seed,
                drop_last=True, sampler='RandomSampler')

    batch_size = cfg.TEST.BATCH_SIZE
    dataset_type = cfg.TEST.DATASET_TYPE
    test_domain = cfg.TEST.DOMAIN if cfg.TEST.DOMAIN != "" else target
    dataroot_test = os.path.join(cfg.DATASET.DATAROOT, test_domain)
    dataloaders['test'] = CustomDatasetDataLoader2(
                    dataset=target_test_dataset, batch_size=batch_size,
                    train=False, num_workers=cfg.NUM_WORKERS,
                    classnames=classes)

    return dataloaders, X_tr, Y_tr, idxs_lb, handler, train_transform, test_transform

# -

def prepare_data_ACAN(seed=None):
    dataloaders = {}
    train_transform = data_utils.get_transform(True)
    test_transform = data_utils.get_transform(False)

    source = cfg.DATASET.SOURCE_NAME
    target = cfg.DATASET.TARGET_NAME
    dataroot_S = os.path.join(cfg.DATASET.DATAROOT, source)
    dataroot_T = os.path.join(cfg.DATASET.DATAROOT, target)

    with open(os.path.join(cfg.DATASET.DATAROOT, 'category.txt'), 'r') as f:
        classes = f.readlines()
        classes = [c.strip() for c in classes]
    assert(len(classes) == cfg.DATASET.NUM_CLASSES)

    # for clustering
    batch_size = cfg.CLUSTERING.SOURCE_BATCH_SIZE
    dataset_type = 'SingleDataset'
    print('Building clustering_%s dataloader...' % source)
    dataloaders['clustering_' + source] = CustomDatasetDataLoader(
                dataset_root=dataroot_S, dataset_type=dataset_type,
                batch_size=batch_size, transform=train_transform,
                train=False, num_workers=cfg.NUM_WORKERS,
                classnames=classes)

    batch_size = cfg.CLUSTERING.TARGET_BATCH_SIZE
    print('Building clustering_%s dataloader...' % target)
    # dataset_type = cfg.CLUSTERING.TARGET_DATASET_TYPE
    dataset_type = 'SingleDataset'
    # Random split
    target_dataset = getattr(single_dataset, dataset_type)(root=dataroot_T)
    target_dataset.initialize(root=dataroot_T, classnames=classes)
    X_train, X_test, y_train, y_test = train_test_split( target_dataset.data_paths, target_dataset.data_labels, test_size=0.33, random_state=0, stratify=target_dataset.data_labels)
    dataset_type = 'SingleDataset'
    target_test_dataset = getattr(single_dataset, dataset_type)(root=dataroot_T)
    target_test_dataset.initialize_data(data_paths=X_test, data_labels=y_test, transform=test_transform)
    # dataset_type = 'SingleDatasetWithoutLabel'
    dataset_type = 'SingleDataset'
    target_train_dataset = getattr(single_dataset, dataset_type)()
    target_train_dataset.initialize_data(data_paths=X_train, data_labels=y_train, transform=train_transform)

    dataloaders['clustering_' + target] = CustomDatasetDataLoader2(
                dataset=target_train_dataset, batch_size=batch_size,
                train=False, num_workers=cfg.NUM_WORKERS,
                classnames=classes)
    # dataloaders['clustering_' + target] = CustomDatasetDataLoader(
                # dataset_root=dataroot_T, dataset_type=dataset_type,
                # batch_size=batch_size, transform=train_transform,
                # train=False, num_workers=cfg.NUM_WORKERS,
                # classnames=classes)

    source_x_train, source_y_train = make_dataset_with_labels(dataroot_S, classes)
    X_tr = [*source_x_train, *X_train]
    Y_tr = [*source_y_train, *y_train]
    # start experiment
    NUM_INIT_LB = len(source_y_train)
    n_pool = len(Y_tr)
    n_test = len(y_test)
    print('number of labeled pool: {}'.format(NUM_INIT_LB))
    print('number of unlabeled pool: {}'.format(n_pool - NUM_INIT_LB))
    print('number of testing pool: {}'.format(n_test))
    X_tr = X_train
    Y_tr = y_train

    # generate initial labeled pool
    # idxs_lb = np.zeros(n_pool, dtype=bool)
    # idxs_lb[:NUM_INIT_LB] = True
    idxs_lb = np.zeros(len(y_train), dtype=bool)
    # idxs_lb[:n_pool] = True
    handler = get_handler()

    # class-agnostic source dataloader for supervised training
    batch_size = cfg.TRAIN.SOURCE_BATCH_SIZE
    dataset_type = 'SingleDataset'
    print('Building %s dataloader...' % source)
    dataloaders[source] = CustomDatasetDataLoader(
                dataset_root=dataroot_S, dataset_type=dataset_type,
                batch_size=batch_size, transform=train_transform,
                train=True, num_workers=cfg.NUM_WORKERS,
                classnames=classes)

    # initialize the categorical dataloader
    dataset_type = 'CategoricalSTDataset'
    source_batch_size = cfg.TRAIN.SOURCE_CLASS_BATCH_SIZE
    target_batch_size = cfg.TRAIN.TARGET_CLASS_BATCH_SIZE
    print('Building categorical dataloader...')
    dataloaders['categorical'] = ClassAwareDataLoader(
                dataset_type=dataset_type,
                source_batch_size=source_batch_size,
                target_batch_size=target_batch_size,
                source_dataset_root=dataroot_S,
                transform=train_transform,
                classnames=classes,
                num_workers=cfg.NUM_WORKERS, seed=seed,
                drop_last=True, sampler='RandomSampler')

    batch_size = cfg.TEST.BATCH_SIZE
    dataset_type = cfg.TEST.DATASET_TYPE
    test_domain = cfg.TEST.DOMAIN if cfg.TEST.DOMAIN != "" else target
    dataroot_test = os.path.join(cfg.DATASET.DATAROOT, test_domain)
    dataloaders['test'] = CustomDatasetDataLoader2(
                    dataset=target_test_dataset, batch_size=batch_size,
                    train=False, num_workers=cfg.NUM_WORKERS,
                    classnames=classes)

    return dataloaders, X_tr, Y_tr, idxs_lb, handler, train_transform, test_transform

def prepare_data_CAN():
    dataloaders = {}
    train_transform = data_utils.get_transform(True)
    test_transform = data_utils.get_transform(False)

    source = cfg.DATASET.SOURCE_NAME
    target = cfg.DATASET.TARGET_NAME
    dataroot_S = os.path.join(cfg.DATASET.DATAROOT, source)
    dataroot_T = os.path.join(cfg.DATASET.DATAROOT, target)

    with open(os.path.join(cfg.DATASET.DATAROOT, 'category.txt'), 'r') as f:
        classes = f.readlines()
        classes = [c.strip() for c in classes]
    assert(len(classes) == cfg.DATASET.NUM_CLASSES)

    # for clustering
    batch_size = cfg.CLUSTERING.SOURCE_BATCH_SIZE
    dataset_type = 'SingleDataset'
    print('Building clustering_%s dataloader...' % source)
    dataloaders['clustering_' + source] = CustomDatasetDataLoader(
                dataset_root=dataroot_S, dataset_type=dataset_type,
                batch_size=batch_size, transform=train_transform,
                train=False, num_workers=cfg.NUM_WORKERS,
                classnames=classes)

    batch_size = cfg.CLUSTERING.TARGET_BATCH_SIZE
    print('Building clustering_%s dataloader...' % target)
    # dataset_type = cfg.CLUSTERING.TARGET_DATASET_TYPE
    dataset_type = 'SingleDataset'
    # Random split
    target_dataset = getattr(single_dataset, dataset_type)(root=dataroot_T)
    target_dataset.initialize(root=dataroot_T, classnames=classes)
    X_train, X_test, y_train, y_test = train_test_split( target_dataset.data_paths, target_dataset.data_labels, test_size=0.33, random_state=0, stratify=target_dataset.data_labels)
    dataset_type = 'SingleDataset'
    target_test_dataset = getattr(single_dataset, dataset_type)(root=dataroot_T)
    target_test_dataset.initialize_data(data_paths=X_test, data_labels=y_test, transform=test_transform)
    dataset_type = 'SingleDataset'
    target_train_dataset = getattr(single_dataset, dataset_type)()
    target_train_dataset.initialize_data(data_paths=X_train, data_labels=y_train, transform=train_transform)

    dataloaders['clustering_' + target] = CustomDatasetDataLoader2(
                dataset=target_train_dataset, batch_size=batch_size,
                train=False, num_workers=cfg.NUM_WORKERS,
                classnames=classes)
    # dataloaders['clustering_' + target] = CustomDatasetDataLoader(
                # dataset_root=dataroot_T, dataset_type=dataset_type,
                # batch_size=batch_size, transform=train_transform,
                # train=False, num_workers=cfg.NUM_WORKERS,
                # classnames=classes)

    # generate initial labeled pool
    n_pool = len(y_train)
    idxs_lb = np.zeros(n_pool, dtype=bool)

    # class-agnostic source dataloader for supervised training
    batch_size = cfg.TRAIN.SOURCE_BATCH_SIZE
    dataset_type = 'SingleDataset'
    print('Building %s dataloader...' % source)
    dataloaders[source] = CustomDatasetDataLoader(
                dataset_root=dataroot_S, dataset_type=dataset_type,
                batch_size=batch_size, transform=train_transform,
                train=True, num_workers=cfg.NUM_WORKERS,
                classnames=classes)

    # initialize the categorical dataloader
    dataset_type = 'CategoricalSTDataset'
    source_batch_size = cfg.TRAIN.SOURCE_CLASS_BATCH_SIZE
    target_batch_size = cfg.TRAIN.TARGET_CLASS_BATCH_SIZE
    print('Building categorical dataloader...')
    dataloaders['categorical'] = ClassAwareDataLoader(
                dataset_type=dataset_type,
                source_batch_size=source_batch_size,
                target_batch_size=target_batch_size,
                source_dataset_root=dataroot_S,
                transform=train_transform,
                classnames=classes,
                num_workers=cfg.NUM_WORKERS,
                drop_last=True, sampler='RandomSampler')

    batch_size = cfg.TEST.BATCH_SIZE
    dataset_type = cfg.TEST.DATASET_TYPE
    test_domain = cfg.TEST.DOMAIN if cfg.TEST.DOMAIN != "" else target
    dataroot_test = os.path.join(cfg.DATASET.DATAROOT, test_domain)
    dataloaders['test'] = CustomDatasetDataLoader2(
                    dataset=target_test_dataset, batch_size=batch_size,
                    train=False, num_workers=cfg.NUM_WORKERS,
                    classnames=classes)

    return dataloaders

def prepare_data_finetune():
    dataloaders = {}
    train_transform = data_utils.get_transform(True)
    test_transform = data_utils.get_transform(False)

    source = cfg.DATASET.SOURCE_NAME
    target = cfg.DATASET.TARGET_NAME
    dataroot_S = os.path.join(cfg.DATASET.DATAROOT, source)
    dataroot_T = os.path.join(cfg.DATASET.DATAROOT, target)
    test_domain = cfg.TEST.DOMAIN if cfg.TEST.DOMAIN != "" else target
    dataroot_test = os.path.join(cfg.DATASET.DATAROOT, test_domain)

    with open(os.path.join(cfg.DATASET.DATAROOT, 'category.txt'), 'r') as f:
        classes = f.readlines()
        classes = [c.strip() for c in classes]
    assert(len(classes) == cfg.DATASET.NUM_CLASSES)
    dataset_type = 'SingleDataset'
    # Random split
    target_dataset = getattr(single_dataset, dataset_type)(root=dataroot_T)
    target_dataset.initialize(root=dataroot_T, classnames=classes)
    target_x_train, X_te, target_y_train, Y_te = train_test_split( target_dataset.data_paths, target_dataset.data_labels, test_size=0.33, random_state=0, stratify=target_dataset.data_labels)
    source_x_train, source_y_train = make_dataset_with_labels(dataroot_S, classes)
    X_tr = [*source_x_train, *target_x_train]
    Y_tr = [*source_y_train, *target_y_train]
    # start experiment
    NUM_INIT_LB = len(source_y_train)
    n_pool = len(Y_tr)
    n_test = len(Y_te)
    print('number of labeled pool: {}'.format(NUM_INIT_LB))
    print('number of unlabeled pool: {}'.format(n_pool - NUM_INIT_LB))
    print('number of testing pool: {}'.format(n_test))

    # generate initial labeled pool
    idxs_lb = np.zeros(n_pool, dtype=bool)
    idxs_lb[:NUM_INIT_LB] = True
    # idxs_lb[:n_pool] = True
    handler = get_handler()

    return X_tr, Y_tr, np.array(X_te), torch.from_numpy(np.array(Y_te)), idxs_lb, handler, train_transform, test_transform

def prepare_data_MMD():
    dataloaders = {}
    train_transform = data_utils.get_transform(True)
    test_transform = data_utils.get_transform(False)

    source = cfg.DATASET.SOURCE_NAME
    target = cfg.DATASET.TARGET_NAME
    dataroot_S = os.path.join(cfg.DATASET.DATAROOT, source)
    dataroot_T = os.path.join(cfg.DATASET.DATAROOT, target)

    with open(os.path.join(cfg.DATASET.DATAROOT, 'category.txt'), 'r') as f:
        classes = f.readlines()
        classes = [c.strip() for c in classes]
    assert(len(classes) == cfg.DATASET.NUM_CLASSES)

    batch_size = cfg.TRAIN.SOURCE_BATCH_SIZE
    dataset_type = 'SingleDataset'
    dataloaders[source] = CustomDatasetDataLoader(
                dataset_root=dataroot_S, dataset_type=dataset_type,
                batch_size=batch_size, transform=train_transform,
                train=True, num_workers=cfg.NUM_WORKERS,
                classnames=classes)

    batch_size = cfg.TRAIN.TARGET_BATCH_SIZE
    dataset_type = 'SingleDatasetWithoutLabel'
    dataloaders[target] = CustomDatasetDataLoader(
                dataset_root=dataroot_T, dataset_type=dataset_type,
                batch_size=batch_size, transform=train_transform,
                train=True, num_workers=cfg.NUM_WORKERS,
                classnames=classes)

    batch_size = cfg.TEST.BATCH_SIZE
    dataset_type = cfg.TEST.DATASET_TYPE
    test_domain = cfg.TEST.DOMAIN if cfg.TEST.DOMAIN != "" else target
    dataroot_test = os.path.join(cfg.DATASET.DATAROOT, test_domain)
    dataloaders['test'] = CustomDatasetDataLoader(
                    dataset_root=dataroot_test, dataset_type=dataset_type,
                    batch_size=batch_size, transform=test_transform,
                    train=False, num_workers=cfg.NUM_WORKERS,
                    classnames=classes)

    return dataloaders

def prepare_data_SingleDomainSource():
    dataloaders = {}
    train_transform = data_utils.get_transform(True)
    test_transform = data_utils.get_transform(False)

    source = cfg.DATASET.SOURCE_NAME
    target = cfg.DATASET.TARGET_NAME
    dataroot_S = os.path.join(cfg.DATASET.DATAROOT, source)
    dataroot_T = os.path.join(cfg.DATASET.DATAROOT, target)

    with open(os.path.join(cfg.DATASET.DATAROOT, 'category.txt'), 'r') as f:
        classes = f.readlines()
        classes = [c.strip() for c in classes]
    assert(len(classes) == cfg.DATASET.NUM_CLASSES)

    batch_size = cfg.TRAIN.SOURCE_BATCH_SIZE
    dataset_type = 'SingleDataset'
    dataloaders[source] = CustomDatasetDataLoader(
                dataset_root=dataroot_S, dataset_type=dataset_type,
                batch_size=batch_size, transform=train_transform,
                train=True, num_workers=cfg.NUM_WORKERS,
                classnames=classes)

    batch_size = cfg.TEST.BATCH_SIZE
    dataset_type = cfg.TEST.DATASET_TYPE
    test_domain = cfg.TEST.DOMAIN if cfg.TEST.DOMAIN != "" else target
    dataroot_test = os.path.join(cfg.DATASET.DATAROOT, test_domain)
    dataloaders['test'] = CustomDatasetDataLoader(
                    dataset_root=dataroot_test, dataset_type=dataset_type,
                    batch_size=batch_size, transform=test_transform,
                    train=False, num_workers=cfg.NUM_WORKERS,
                    classnames=classes)

    return dataloaders

def prepare_data_SingleDomainTarget():
    dataloaders = {}
    train_transform = data_utils.get_transform(True)
    test_transform = data_utils.get_transform(False)

    with open(os.path.join(cfg.DATASET.DATAROOT, 'category.txt'), 'r') as f:
        classes = f.readlines()
        classes = [c.strip() for c in classes]
    assert(len(classes) == cfg.DATASET.NUM_CLASSES)

    target = cfg.DATASET.TARGET_NAME
    dataroot_T = os.path.join(cfg.DATASET.DATAROOT, target)
    batch_size = cfg.TRAIN.TARGET_BATCH_SIZE
    dataset_type = 'SingleDataset'
    dataloaders[target] = CustomDatasetDataLoader(
                dataset_root=dataroot_T, dataset_type=dataset_type,
                batch_size=batch_size, transform=train_transform,
                train=True, num_workers=cfg.NUM_WORKERS,
                classnames=classes)

    batch_size = cfg.TEST.BATCH_SIZE
    dataset_type = cfg.TEST.DATASET_TYPE
    test_domain = cfg.TEST.DOMAIN if cfg.TEST.DOMAIN != "" else target
    dataroot_test = os.path.join(cfg.DATASET.DATAROOT, test_domain)
    dataloaders['test'] = CustomDatasetDataLoader(
                    dataset_root=dataroot_test, dataset_type=dataset_type,
                    batch_size=batch_size, transform=test_transform,
                    train=False, num_workers=cfg.NUM_WORKERS,
                    classnames=classes)

    return dataloaders

reason: import numpy,from scipy