text:
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import codecs
import os
import sys

import yaml
import numpy as np
from paddle.inference import create_predictor, PrecisionType
from paddle.inference import Config as PredictConfig

LOCAL_PATH = os.path.dirname(os.path.abspath(__file__))
sys.path.append(os.path.join(LOCAL_PATH, '..', '..'))

from paddleseg.utils import logger, get_image_list, progbar
from infer import DeployConfig
"""
Load images and run the model, it collects and saves dynamic shapes,
which are used in deployment with TRT.
"""

def parse_args():
    parser = argparse.ArgumentParser(description='Test')
    parser.add_argument(
        "--config",
        help="The deploy config generated by exporting model.",
        type=str,
        required=True)
    parser.add_argument(
        '--image_path',
        help='The directory or path or file list of the images to be predicted.',
        type=str,
        required=True)

    parser.add_argument(
        '--dynamic_shape_path',
        type=str,
        default="./dynamic_shape.pbtxt",
        help='The path to save dynamic shape.')

    return parser.parse_args()

def is_support_collecting():
    return hasattr(PredictConfig, "collect_shape_range_info") \
        and hasattr(PredictConfig, "enable_tuned_tensorrt_dynamic_shape")

def collect_dynamic_shape(args):

    if not is_support_collecting():
        logger.error("The Paddle does not support collecting dynamic shape, " \
            "please reinstall the PaddlePaddle (latest gpu version).")

    # prepare config
    cfg = DeployConfig(args.config)
    pred_cfg = PredictConfig(cfg.model, cfg.params)
    pred_cfg.enable_use_gpu(1000, 0)
    pred_cfg.collect_shape_range_info(args.dynamic_shape_path)

    # create predictor
    predictor = create_predictor(pred_cfg)
    input_names = predictor.get_input_names()
    input_handle = predictor.get_input_handle(input_names[0])

    # get images
    img_path_list, _ = get_image_list(args.image_path)
    if not isinstance(img_path_list, (list, tuple)):
        img_path_list = [img_path_list]
    logger.info(f"The num of images is {len(img_path_list)} \n")

    # collect
    progbar_val = progbar.Progbar(target=len(img_path_list))
    for idx, img_path in enumerate(img_path_list):
        data = np.array([cfg.transforms(img_path)[0]])
        input_handle.reshape(data.shape)
        input_handle.copy_from_cpu(data)

        try:
            predictor.run()
        except:
            logger.info(
                "Fail to collect dynamic shape. Usually, the error is out of "
                "GPU memory, for the model and image are too large.\n")
            del predictor
            if os.path.exists(args.dynamic_shape_path):
                os.remove(args.dynamic_shape_path)

        progbar_val.update(idx + 1)

    logger.info(f"The dynamic shape is save in {args.dynamic_shape_path}")

if __name__ == '__main__':
    args = parse_args()
    collect_dynamic_shape(args)


text:
import pygame
import numpy as np
'''GUI for the Game of Mills.
Gets the Mapping of the field as an array.
Has a fixed conversion Array as an 2D Space on which the board is to be represented, which can be scaled as needed.
Scans the GUI for mouseclicks, and transforms them via indexation of the unscaled Array back into
the coordinates of the board which can be returned to commit the move of the player.
Graphical Representation of both Arrays (scaled Array has same Architecture, but differing dimensions.
Ignores the additional layer of the board-Array for meta-data about the game such as stages, counters and past moves.

Author: Philipp Petermeier, https://github.com/PPetermeier
Date: 28.02.2021

Based on the implementation of mills with alphazero from Simon Schnecko.

 2,7 ------------- 2,0 ------------- 2,1
 |                 |                 |
 |    1,7 ------- 1,0 ------- 1,1    |
 |     |           |           |     |
 |     |    0,7 - 0,0 - 0,1    |     |
 |     |     |           |     |     |
 2,6 - 1,6 - 0,6         0,2 - 1,2 - 2,2
 |     |     |           |     |     |
 |     |    0,5 - 0,4 - 0,3    |     |
 |     |           |           |     |
 |    1,5 ------- 1,4 ------- 1,3    |
 |                 |                 |
 2,5 ------------- 2,4 ------------- 2,3

 1,1 ------------- 1,7 ------------- 1,13
 |                 |                 |
 |    3,3 ------- 3,7 ------- 3,11   |
 |     |           |           |     |
 |     |    5,5 - 5,7 - 5,9    |     |
 |     |     |           |     |     |
 7,1 - 7,3 - 7,5         7,9 - 7,11 - 7,13
 |     |     |           |     |     |
 |     |    9,5 - 9,7 - 9,9    |     |
 |     |           |           |     |
 |    11,3------- 11,7-------11,11   |
 |                 |                 |
 13,1 -------------13,7-------------13,13

'''
##------------ Umwandlungstabellen zwischen Index & Koordinate------------
scale: int = 100
line = int(scale / 10)

def setscale(value):
    scale = value

def getscale():
    return scale

def testboard():
    testboard = np.zeros((3, 8))
    return testboard

# ---------------Überlegungen, wie eine Schnittstellentabelle automatisiert erstellt werden kann ----------
# def write_coordinates(board):
#    coordinates = np.copy(board)
#    dimension = 1
#    for i in range(len(coordinates)):
#        dimension + 4
#    vdistance = 2, 0
#    hdistance = 0, 2
#    cdistance = 2, 0
#    anchor1 = (dimension / 2) - 2
#    anchor2 = dimension / 2
#    anchor =np.array([anchor1, anchor2])
#    anchor[0] = int(dimension / 2) - 2), int(dimension / 2)
#    counter = anchor
#    for i in range(len(coordinates)):
#        for j in range(len(coordinates[i])):
#            if i == 0 and j == 0:
#                coordinates[i[j]] = int(anchor1), int(anchor2)
#                break
#            if i > 0 == True and j == 0:
#                counter = anchor - int(i * cdistance)
#                coordinates[i[j]] = counter
#            if j == 1:
#                counter = counter + hdistance
#                coordinates[i[j]] = counter
#                break
#            if j < 4 == True:
#                counter = counter + vdistance
#                coordinates[i[j]] = counter
#                break
#            if j < 6 == True:
#                counter = counter - hdistance
#                coordinates[i[j]] = counter
#                break
#            else:
#                counter = counter - vdistance
#        vdistance[0] + 1, hdistance[1] + 1
#    return coordinates

##--------------------------Fixe Schnittstelle zwischen board und GUI ----------------------
coordinates = np.array([[(5, 7), (5, 9), (7, 9), (9, 9), (9, 7), (9, 5), (7, 5), (5, 5)],
                       [(3, 7), (3, 11), (7, 11), (11, 11), (11, 7), (11, 3), (7, 3), (3, 3)],
                        [(1, 7), (1, 13), (7, 13), (13, 13), (13, 7), (13, 1), (7, 1), (1, 1)]])

coordinates = np.roll(np.fliplr(coordinates), 7 , 1)

scaled_coordinates = coordinates * scale

def get_index(gui_position1, gui_position2):     ## Das selbsterstellte Board hat vermute ich die Reihen und Spalten vertauscht. Sollte es zu Problemen kommen, einfach im loop rows & cols miteinander vertauschen!
    gui_position1 = int(round(gui_position1/scale))
    gui_position2 = int(round(gui_position2/scale))
    search = (gui_position1, gui_position2)
    rows = coordinates.shape[0]
    cols = coordinates.shape[1]
    failure = False
    print(search)
    for i in range(rows):
        for j in range(cols):
            if np.all(coordinates[i, j] == search):
                return i, j
    return failure

def get_gui_position(index_position1, index_position2):
    gui_position = scaled_coordinates[index_position1, index_position2]
    return gui_position

##------------Farbdefinition-----------------------------
BLACK, GREEN, BLUE, GRAY, YELLOW, WHITE = (0, 0, 0), (0, 255, 0), (0, 0, 255), (127, 127, 127), (255, 255, 0), (
    255, 255, 255)
##-------------- Darstellungserstellung ----------------
def drawbackground():
    pygame.draw.rect(screen, WHITE, (0, 0, 14 * scale, 14 * scale))  # Hintergrund

def drawlines():
    for i in range(3):
        for j in range(8):
            start, finish = scaled_coordinates[i][j], scaled_coordinates[i][((j + 1) % 8)]
            pygame.draw.line(screen, BLACK, start, finish, line)
            if j % 2 == 0 and i == 0:
                start, finish = scaled_coordinates[i][j], scaled_coordinates[(i + 2)][j]
                pygame.draw.line(screen, BLACK, start, finish, line)
    pygame.display.update()

## Soll nach jedem Zug aufgerufen werden, sofern nicht jedes Mal das Board komplett neu gezeichnet werden muss? Wenn nein, werden hier nur die Veraenderungen angezeigt
def updateboard(board):
    for i in range(3):
        for j in range(8):
            # valid = get_legal_moves(player) Hier fehlt die Einbindung von legal_moves
            # if board[i, j] == valid:
            #    pygame.draw.circle(surface=screen, color=GREEN, center=(get_gui_position(i, j)), radius=(scale / 2))
            #    pygame.display.update()
            if board[i, j] == 1:
                pygame.draw.circle(surface=screen, color=YELLOW, center=(get_gui_position(i, j)), radius=(scale / 2))
                pygame.display.update()
            if board[i, j] == -1:
                pygame.draw.circle(surface=screen, color=BLUE, center=(get_gui_position(i, j)), radius=(scale / 2))
                pygame.display.update()
            if board[i, j] == 0:
                pygame.draw.circle(surface=screen, color=GRAY, center=(get_gui_position(i, j)), radius=(scale / 2))
                pygame.display.update()

## ------------------------- Initialisierungsvariabeln -------------------------------
board = testboard()
width: int = 14 * scale
height: int = 14 * scale
size = (width, height)
pygame.init()
screen = pygame.display.set_mode(size)
myfont = pygame.font.SysFont("monospace", 75)
# board = write_coordinates(board)
drawbackground()
drawlines()
updateboard(board)
pygame.display.update()
pygame.time.wait(30000)

## --------------Eventsteuerung, entnommen und angepasst aus Uebung 3 --------------
# while board.pieces[3, 7] == 0:  ##piece [3,7]: speichert Spielzustand
player_turn = True
player = 1

def set_piece(board, player, col, row):
    place = get_index(col, row)
    if place != False:
        board[place] = player
    else:
        Message = 'Du hast eine ungültige Stelle angeklickt, versuch es nochmal auf einem leeren Feld'
        print(Message)

def scanning():
    while player_turn:
        for event in pygame.event.get():
            if event.type == pygame.MOUSEBUTTONDOWN:
                posx = event.pos[0]
                posy = event.pos[1]
                col = int(round(posx))
                row = int(round(posy))
                set_piece(board, player, col, row)
                updateboard(board)

            #position = [int(math.floor(posy / scale)), int(math.floor(posx / scale))]

            # column = int(math.floor(posy / scale))
            # row = int(math.floor(posx / scale))
            # valid = self.game.getValidMoves(board, player)
            # for i in range(valid):
            #    if position == valid(i):
            #        execute_move(position)
            #        updateboard(board)
            #        pygame.display.update()
            #        break
        pygame.display.update()

scanning()

ext: py

max_stars_repo_name: Schnetzkor/alpha-zero-general

max_issues_repo_name: Schnetzkor/alpha-zero-general

max_forks_repo_name: Schnetzkor/alpha-zero-general

reason: import numpy


text:
import pandas as pd
import numpy as np
import sklearn.linear_model
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import roc_auc_score
from sklearn.metrics import confusion_matrix
from sklearn.metrics import precision_score
from sklearn.metrics import recall_score
import seaborn as sns
from sklearn.metrics import roc_curve

df = pd.read_csv('FlightData.csv')
# print(df.head())
# print(df.isnull().sum())

df = df.drop('Unnamed: 25',axis = 1)
# print(df.isnull().sum())

df = df[["MONTH", "DAY_OF_MONTH", "DAY_OF_WEEK", "ORIGIN", "DEST", "CRS_DEP_TIME", "ARR_DEL15"]]
# print(df.isnull().sum())

# print(df[df.isnull().values.any(axis=1)].head())

df = df.fillna({'ARR_DEL15': 1})
# print(df.iloc[177:185])

import math

for index, row in df.iterrows():
    df.loc[index, 'CRS_DEP_TIME'] = math.floor(row['CRS_DEP_TIME'] / 100)
# print(df.head())

df = pd.get_dummies(df, columns=['ORIGIN', 'DEST'])
# print(df.head())

train_x, test_x, train_y, test_y = train_test_split(df.drop('ARR_DEL15', axis=1), df['ARR_DEL15'], test_size=0.2, random_state=42)
# print(train_x.shape)
# print(test_x.shape)

model = RandomForestClassifier(random_state=13)
model.fit(train_x, train_y)

predicted = model.predict(test_x)
# print(model.score(test_x,test_y))

probabilities = model.predict_proba(test_x)
# print(roc_auc_score(test_y, probabilities[:, 1]))

# print(confusion_matrix(test_y, predicted))

train_predictions = model.predict(train_x)
# print(precision_score(train_y, train_predictions))

# print(recall_score(train_y, train_predictions))

sns.set()

fpr, tpr, _ = roc_curve(test_y, probabilities[:, 1])
plt.plot(fpr, tpr)
plt.plot([0, 1], [0, 1], color='grey', lw=1, linestyle='--')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')

def predict_delay(departure_date_time, origin, destination):
    from datetime import datetime

    try:
        departure_date_time_parsed = datetime.strptime(departure_date_time, '%d/%m/%Y %H:%M:%S')
    except ValueError as e:
        return 'Error parsing date/time - {}'.format(e)

    month = departure_date_time_parsed.month
    day = departure_date_time_parsed.day
    day_of_week = departure_date_time_parsed.isoweekday()
    hour = departure_date_time_parsed.hour

    origin = origin.upper()
    destination = destination.upper()

    input = [{'MONTH': month,
              'DAY': day,
              'DAY_OF_WEEK': day_of_week,
              'CRS_DEP_TIME': hour,
              'ORIGIN_ATL': 1 if origin == 'ATL' else 0,
              'ORIGIN_DTW': 1 if origin == 'DTW' else 0,
              'ORIGIN_JFK': 1 if origin == 'JFK' else 0,
              'ORIGIN_MSP': 1 if origin == 'MSP' else 0,
              'ORIGIN_SEA': 1 if origin == 'SEA' else 0,
              'DEST_ATL': 1 if destination == 'ATL' else 0,
              'DEST_DTW': 1 if destination == 'DTW' else 0,
              'DEST_JFK': 1 if destination == 'JFK' else 0,
              'DEST_MSP': 1 if destination == 'MSP' else 0,
              'DEST_SEA': 1 if destination == 'SEA' else 0}]

    return model.predict_proba(pd.DataFrame(input))[0][0]
print(f"Chances to arrive on time: {predict_delay('1/10/2018 21:45:00', 'JFK', 'ATL')*100}%")
print(f"Chances to arrive on time: {predict_delay('1/10/2017 12:45:00', 'ATL', 'JFK')*100}%")

max_stars_repo_name: BuragaIonut/MachineLearning

max_issues_repo_name: BuragaIonut/MachineLearning

max_forks_repo_name: BuragaIonut/MachineLearning


text:
import grace
import grace.times
import grace.ols
import numpy as np
import matplotlib.pyplot as plt
import cPickle as pickle
import sklearn.mixture as skm
import sklearn.decomposition as skd
import time
import mpl_toolkits.basemap as maps
import matplotlib as mpl
import grace
import grace.times

days=grace.ols.time_vector()
number_of_pcs=10

colors_hex = [
	"a6cee3", "1f78b4", "b2df8a", "33a02c", "fb9a99", "e31a1c",
	"fdbf6f", "ff7f00", "cab2d6", "6a3d9a", "ffff99", "b15928","edf8b1"
	,"bcbddc", "636363"
]
colors_rgb = map(lambda hex: [ord(c) for c in hex.decode('hex')], colors_hex)

def plot_ica_clusters(model,ica,X):
	y=model.predict(X)
	y=y.reshape(180,360)[::-1]
	shape=y.shape
	#print y.shape
	fig = plt.figure(figsize=(12, 6))

	m = maps.Basemap(projection='cyl', lon_0=0, resolution='c')
	m.drawcoastlines(linewidth=2.5, color="white")
	m.drawparallels(np.arange(-90.,120.,30.), labels=[1,0,0,0])
	m.drawmeridians(np.arange(0.,420.,60.), labels=[0,0,0,1])

	centroids=model.means_
	covariance=model.covars_
	cmap=plt.cm.jet
	cmaplist=[cmap(i) for i in range(cmap.N)]
	cmap=cmap.from_list('Custom cmap',cmaplist,cmap.N)
	bounds=np.linspace(0,len(centroids),len(centroids)+1)
	norm=mpl.colors.BoundaryNorm(bounds,cmap.N)

	world = np.ones((shape[0], shape[1], 3)).astype('uint8') * 255

	for i in range(0,shape[0]):
		for j in range(0,shape[1]):
			world[i,j,:]=colors_rgb[y[i,j]]

	im = m.imshow(world,cmap=cmap,norm=norm)
	plt.title('KPCA Clustering')
	fig.savefig('HPC-output/ica-world.png')
	fig = plt.figure(figsize=(12, 6))
	labels=model.predict(centroids)
#	print centroids,model.weights_,labels
	for i,series in enumerate(ica.inverse_transform(centroids)):
		plt.plot(days, series.ravel(),lw=2,c='#' + colors_hex[i],label='cluster '+str(i+1))

	plt.legend(loc=3)
	plt.title('Cluster centroids (inverse transformed kpca)')

	fig.savefig('HPC-output/ica-centroids.png')
	return 0

def fit_gmm(X):
	print 'fitting full gaussian mixture model'
	start=time.time()
	model=skm.GMM(n_components=12,covariance_type='full'
			,random_state=1337,n_iter=100,thresh=0.01,n_init=1)
	model.fit(X)
	end=time.time()
	print 'Fit done in ' + str(end-start) + ' seconds.'
	return model

def ica_fit(X):
	print 'fitting ica with FASTICA-algorithm'
	start=time.time()
	model=skd.FastICA(n_components=number_of_pcs,whiten=True)
	X_ica=model.fit_transform(X)
	end=time.time()
	print 'Fit done in ' +str(end-start) + ' seconds.'
	return [X_ica,model]

if __name__=='__main__':
	print 'Clustering in the ICA space.'
#	display_SA()
#	loading ( position are on rows and days on columns)
	shape=grace.load.grids.shape
	Y=grace.load.grids.reshape(shape[0]*shape[1],shape[2])

	#[Y_encoded,v]=pca_fit(Y)
	[Y_encoded,ica]=ica_fit(Y)
	model=fit_gmm(Y_encoded)
	#plot_clusters(model,Y_encoded,v)
	plot_ica_clusters(model,ica,Y_encoded)

max_stars_repo_name: AndreasMadsen/grace

max_issues_repo_name: AndreasMadsen/grace

max_forks_repo_name: AndreasMadsen/grace


text:
import numpy as np
import keras
import random
from python.dataloader import loader

class DataGenerator(keras.utils.Sequence):
    'Generates data for Keras'
    def __init__(self, data_type='train', batch_size=10, num_batch=200, dim=(105,105), n_channels=1,
                 n_classes=30):
        'Initialization'
        self.type = data_type
        if self.type == 'train':
            self.is_training = np.array([True for _ in range(batch_size*2)])
        else:
            self.is_training = np.array([False for _ in range(batch_size*2)])
        self.dim = dim
        self.batch_size = batch_size
        self.n_channels = n_channels
        self.n_classes = n_classes
        self.num_batch = num_batch
        self.y_target = np.append(np.ones(self.batch_size), np.zeros(self.batch_size))
        self.build_data(self.type)
        self.on_epoch_end()

    def build_data(self, data_type):
        self.class_data = np.array(loader(data_type, 'python/images_background'))
        self.n_classes = len(self.class_data)

    def __len__(self):
        'Denotes the number of batches per epoch'
        return self.num_batch

    def __getitem__(self, index):
        'Generate one batch of data'
        # Generate data
        X1, X2 = self.__data_generation()

        # X_pos = self.sess.run(self.trans_img_output, feed_dict={self.trans_img_input:X_pos})
        # X_pin = self.sess.run(self.trans_img_output, feed_dict={self.trans_img_input:X_pin})
        # X_neg = self.sess.run(self.trans_img_output, feed_dict={self.trans_img_input:X_neg})

        return [X1,X2,self.is_training], self.y_target

    def on_epoch_end(self):
        'Updates indexes after each epoch'
        pass

    def __data_generation(self):
        'Generates data containing batch_size samples' # X : (n_samples, *dim, n_channels)
        # Initialization
        X1 = np.empty((self.batch_size * 2, *self.dim, self.n_channels))
        X2 = np.empty((self.batch_size * 2, *self.dim, self.n_channels))
        # print(pos, neg)
        # print(self.class_data[pos][0].shape)
        # Generate data
        for i in range(self.batch_size):
            cls = random.choice(range(self.n_classes))
            idx1, idx2 = random.sample(range(20),2)
            X1[i,], X2[i,] = self.class_data[cls][idx1], self.class_data[cls][idx2]

        for i in range(self.batch_size, self.batch_size*2):
            pos, neg = random.sample(range(self.n_classes), 2)
            idx1 = random.choice(range(20))
            idx2 = random.choice(range(20))
            X1[i,], X2[i,] = self.class_data[pos][idx1], self.class_data[neg][idx2]
        return X1, X2
        #return X, keras.utils.to_categorical(y, num_classes=self.n_classes)

max_stars_repo_name: barnrang/omniglot

max_issues_repo_name: barnrang/omniglot

max_forks_repo_name: barnrang/omniglot


text:
import numpy as np
import matplotlib.pyplot as plt
import csv
import os

import scipy.special as special

def main():

    legend_entries = []

    #plt.rc('text', usetex=True)
    #plt.rc('font', family='serif')

    def smooth(vals, N=20):
        new_vals = vals.copy()
        for n in range(0,vals.size):
            padding = (vals.size - n) - N
            if padding < 0:
                window = [n+k for k in range(padding,vals.size-n)]
            else:
                window = [n+k for k in range(0,N)]
            window_vals = vals[window]
            new_vals[n] = np.mean(window_vals)
        return new_vals

    filename = "UQ_Bounds_NEW.csv"

    stds = []
    t_uqs = []
    v_uqs = []
    with open(filename, "r") as csvfile:
        csvreader = csv.reader(csvfile, delimiter=' ', quotechar='|')
        for row in csvreader:
            std, t_uq, t_std, v_uq, v_std = row
            stds.append(std)
            t_uqs.append(t_uq)
            v_uqs.append(v_uq)

    stds = np.array(stds).astype(np.float32)
    t_uqs = 100*np.array(t_uqs).astype(np.float32)
    v_uqs = 100*np.array(v_uqs).astype(np.float32)

    """
    original_losses = losses.copy()
    original_losses = remove_outliers(original_losses)
    ## Smooth out losses
    SMOOTH = True
    if SMOOTH:
        losses = smooth(losses)
        losses = remove_outliers(losses)
    """

    plt.plot(stds, t_uqs, linewidth=3.0, color="C0", label="Training Dataset")
    plt.plot(stds, v_uqs, linewidth=3.0, color="C1", label='Validation Dataset')

    alpha = 0.1
    y1 = np.zeros(t_uqs.shape)
    plt.fill_between(stds, y1, t_uqs, where=t_uqs >= y1, facecolor="C0", alpha=alpha, interpolate=True, label=None)#, hatch="X", edgecolor="white")

    alpha = 0.1
    y1 = np.zeros(v_uqs.shape)
    plt.fill_between(stds, y1, v_uqs, where=v_uqs >= y1, facecolor="C1", alpha=alpha, interpolate=True, label=None)

    #plt.scatter(steps, losses, alpha=0.75, marker="s", edgecolor='black', s=100)
    #ax = plt.gca()
    #ax.set_yscale('log')
    #legend_entries.append('%d Training Points' %(data_per_file*k))

    ax = plt.gca()

    # Plot parameters
    linewidth = 3
    titlesize = 24
    ylabelsize = 24
    xlabelsize = 24
    xticksize = 18
    yticksize = 16
    ylabelpad = 20
    xlabelpad = 20

    ax.tick_params(axis='x', labelsize=xticksize)
    ax.tick_params(axis='y', labelsize=yticksize)

    ax.set_xlabel('Standard Deviations', fontsize=xlabelsize, labelpad=20)
    ax.set_ylabel('Percentage of Dataset', color='k', fontsize=ylabelsize, labelpad=ylabelpad)

    ticks = [n*20 for n in [0,1,2,3,4,5]]
    labels = tuple(["{0:}%".format(n) for n in ticks])
    plt.yticks(ticks, labels, fontsize=yticksize)

    ticks = [n*0.5 for n in [1,2,3,4,5,6]]
    labels = tuple([r"{0:}$\sigma$".format(n).replace(".0"," ",1) for n in ticks])
    plt.xticks(ticks, labels, fontsize=xticksize)

    training_vals = [0.73471046, 0.96861186, 0.99774161]
    validation_vals = [0.68202384, 0.94649767, 0.99434461]

    #alpha=0.85
    alpha=1.0

    textstr = '\n'.join((
        #r"3$\sigma$",
        r"Training:    99.77",
        r"Validation: 99.43"))

    props = dict(boxstyle='round', facecolor='white', alpha=alpha)
    ax.text(0.8, 1.04, textstr, transform=ax.transAxes, fontsize=18,
            verticalalignment='top', bbox=props)

    textstr = '\n'.join((
        #r"3$\sigma$",
        r"Training:    96.86",
        r"Validation: 94.65"))

    props = dict(boxstyle='round', facecolor='white', alpha=alpha)
    #ax.text(0.635, 0.855, textstr, transform=ax.transAxes, fontsize=18,
    ax.text(0.49, 1.015, textstr, transform=ax.transAxes, fontsize=18,
            verticalalignment='top', bbox=props)

    textstr = '\n'.join((
        #r"3$\sigma$",
        r"Training:    73.47",
        r"Validation: 68.20"))

    props = dict(boxstyle='round', facecolor='white', alpha=alpha)
    #ax.text(0.345, 0.625, textstr, transform=ax.transAxes, fontsize=18,
    ax.text(0.17, 0.8, textstr, transform=ax.transAxes, fontsize=18,
            verticalalignment='top', bbox=props)

    sigma = 1.0
    #func = lambda x: 2.*100.*(0.5*(1 + special.erf(x/(sigma*np.sqrt(2)))) - 0.5)
    func = lambda x: 100.*special.erf(x/(sigma*np.sqrt(2)))
    #plt.plot(stds, func(stds), 'r--', linewidth=2, alpha=0.5, zorder=0, label=r"$\operatorname{erf}(x/\sqrt{2})$")
    plt.plot(stds, func(stds), 'r--', linewidth=2, alpha=0.5, zorder=0, label=r"$P(|\mathcal{N}(0,\sigma)| \, < \, x)$")

    # Add dots at 1/2/3 standard deviation points
    xvals = [1,2,3]
    coords = [73.471046, 96.861186, 99.774161]
    plt.scatter(xvals,coords, color='C0', s=60, zorder=5)

    coords = [68.202384, 94.649767, 99.434461]
    plt.scatter(xvals,coords, color='C1', s=60, zorder=9)

    # Add dashed lines
    alpha = 0.25
    plt.plot([1, 1], [0.0, 73.47], 'k--', lw=2, alpha=alpha)
    plt.plot([2, 2], [0.0, 96.86], 'k--', lw=2, alpha=alpha)
    plt.plot([3, 3], [0.0, 99.77], 'k--', lw=2, alpha=alpha)

    #ax.legend(legend_entries, fontsize=24)
    #ax.legend(fontsize=24)
    #ax.legend(fontsize=24, loc=(0.05,0.85), framealpha=1.0)
    ax.legend(fontsize=24, loc=(0.05,0.88), framealpha=1.0)
    plt.show()

# Run main() function when called directly
if __name__ == '__main__':
    main()

reason: import numpy,import scipy


text:
# my_cgs_worker.py
from pyrates.utility.grid_search import ClusterWorkerTemplate
import os
from pandas import DataFrame
from pyrates.utility import grid_search, welch
import numpy as np
from copy import deepcopy
from scipy.ndimage.filters import gaussian_filter1d

class MinimalWorker(ClusterWorkerTemplate):
    def worker_postprocessing(self, **kwargs):
        self.processed_results = DataFrame(data=None, columns=self.results.columns)
        for idx, data in self.results.iteritems():
            self.processed_results.loc[:, idx] = data * 1e3
        self.processed_results.index = self.results.index * 1e-3

class ExtendedWorker(MinimalWorker):
    def worker_gs(self, *args, **kwargs):

        # get parameters
        kwargs_tmp = deepcopy(kwargs)
        conditions = kwargs_tmp.pop('conditions')
        param_grid = kwargs_tmp.pop('param_grid')
        param_scalings = kwargs_tmp.pop('param_scalings')

        # simulate autonomous behavior in different conditions
        results, gene_ids = [], param_grid.index
        for c_dict in deepcopy(conditions):
            for key in param_grid:
                if key in c_dict:
                    c_dict[key] = deepcopy(param_grid[key]) * c_dict[key]
                elif key in param_grid:
                    c_dict[key] = deepcopy(param_grid[key])
            for key, key_tmp, power in param_scalings:
                c_dict[key] = c_dict[key] * c_dict[key_tmp]**power
            param_grid_tmp = DataFrame.from_dict(c_dict)
            r, self.result_map, sim_time = grid_search(*args, param_grid=param_grid_tmp, **deepcopy(kwargs_tmp))
            r = r.droplevel(2, axis=1)
            results.append(r)

        self.results = results
        return sim_time

    def worker_postprocessing(self, **kwargs):
        kwargs_tmp = kwargs.copy()
        param_grid = kwargs_tmp.pop('param_grid')
        freq_targets = kwargs_tmp.pop('freq_targets')
        targets = kwargs_tmp.pop('y')
        self.processed_results = DataFrame(data=None, columns=['fitness', 'r_e', 'r_i', 'r_a'])

        # calculate fitness
        for gene_id in param_grid.index:
            outputs, vars = [], []
            for i, r in enumerate(self.results):
                r = r * 1e3
                r.index = r.index * 1e-3
                cutoff = r.index[-1]*0.5
                cutoff2 = r.index[-1]*0.25
                func = np.mean #if i < len(self.results)-1 else np.max
                mean_re = func(r['r_e'][f'circuit_{gene_id}'].loc[cutoff:])
                mean_ri = func(r['r_i'][f'circuit_{gene_id}'].loc[cutoff:])
                mean_ra = func(r['r_a'][f'circuit_{gene_id}'].loc[cutoff:])
                outputs.append([mean_re, mean_ri, mean_ra])
                vars.append(np.var(r['r_i'][f'circuit_{gene_id}'].loc[cutoff:]) if freq_targets[i] == 0.0 else
                            1/np.var(r['r_i'][f'circuit_{gene_id}'].loc[cutoff2:]))
                freq_targets[i] = 0.0

            dist1 = fitness(outputs, targets)
            dist2 = fitness(vars, freq_targets)

            self.processed_results.loc[gene_id, 'fitness'] = 1/(dist1+dist2)
            self.processed_results.loc[gene_id, 'r_e'] = [rates[0] for rates in outputs]
            self.processed_results.loc[gene_id, 'r_i'] = [rates[1] for rates in outputs]
            self.processed_results.loc[gene_id, 'r_a'] = [rates[2] for rates in outputs]

def fitness(y, t, squared=True):
    y = np.asarray(y).flatten()
    t = np.asarray(t).flatten()
    diff = np.asarray([0.0 if np.isnan(t_tmp) else y_tmp - t_tmp for y_tmp, t_tmp in zip(y, t)]).flatten()
    diff = diff**2 if squared else np.abs(diff)
    t[np.isnan(t)] = 1.0
    t[t == 0] = 1.0
    weights = 1/np.abs(t)
    return weights @ np.abs(diff)

if __name__ == "__main__":
    cgs_worker = ExtendedWorker()
    cgs_worker.worker_init()
    #cgs_worker.worker_init(
    #    config_file="/data/u_rgast_software/PycharmProjects/BrainNetworks/BasalGanglia/stn_gpe_healthy_opt/Config/DefaultConfig_0.yaml",
    #    subgrid="/data/u_rgast_software/PycharmProjects/BrainNetworks/BasalGanglia/stn_gpe_healthy_opt/Grids/Subgrids/DefaultGrid_38/spanien/spanien_Subgrid_0.h5",
    #    result_file="~/my_result.h5",
    #    build_dir=os.getcwd()
    #

max_stars_repo_name: Richert/BrainNetworks

max_issues_repo_name: Richert/BrainNetworks

max_forks_repo_name: Richert/BrainNetworks

reason: import numpy,from scipy


text:
from keras.models import Sequential
from keras.layers import Conv2D
from keras.layers import MaxPooling2D
from keras.layers import Flatten
from keras.layers import Dense

# Initialising the CNN
classifier = Sequential()

# add conv
classifier.add(Conv2D(32, (3, 3), input_shape = (64, 64, 3), activation = 'relu'))

'''
number of Filter   32
size 3 x 3

64x64 3 color (RGB)
rectify
'''
# pooling
classifier.add(MaxPooling2D(pool_size = (2, 2)))
# second conv
classifier.add(Conv2D(32, (3, 3), activation = 'relu'))
classifier.add(MaxPooling2D(pool_size = (2, 2)))
# flatten
classifier.add(Flatten())

# add 128 node in hidden layer
classifier.add(Dense(units = 128, activation = 'relu'))

# add softmax  true or false
classifier.add(Dense(units = 1, activation = 'sigmoid'))

# compile CNN
classifier.compile(optimizer = 'adam', loss = 'binary_crossentropy', metrics = ['accuracy'])

from keras.preprocessing.image import ImageDataGenerator
train_datagen = ImageDataGenerator(rescale = 1./255,
    shear_range = 0.2,
    zoom_range = 0.2,
    horizontal_flip = True)
test_datagen = ImageDataGenerator(rescale = 1./255)
training_set = train_datagen.flow_from_directory('dataset/training_set',
    target_size = (64, 64),
    batch_size = 32,
    class_mode = 'binary')
test_set = test_datagen.flow_from_directory('dataset/test_set',
    target_size = (64, 64),
    batch_size = 32,
    class_mode = 'binary')
classifier.fit_generator(training_set,
    steps_per_epoch = 8000,
    epochs = 25,
    validation_data = test_set,
    validation_steps = 2000)

#save
classifier.save("catordog.h5");

# Part 3 - Making new predictions

from keras.models import load_model
import numpy as np
from keras.preprocessing import image

loaded_with = load_model('catordog.h5')
assert loaded_with

test_image = image.load_img('dataset/single_prediction/cat_or_dog_1.jpg', target_size = (64, 64))
test_image = image.img_to_array(test_image)
test_image = np.expand_dims(test_image, axis = 0)
result = classifier.predict(test_image)
training_set.class_indices
if result[0][0] == 1:
    prediction = 'dog'
else:
    prediction = 'cats'

max_stars_repo_name: Telexine/ImageRestoration

max_issues_repo_name: Telexine/ImageRestoration

max_forks_repo_name: Telexine/ImageRestoration


text:
"""
General image database
An image database creates a list of relative image path called image_set_index and
transform index to absolute image path. As to training, it is necessary that ground
truth and proposals are mixed together for training.
"""

import numpy as np
import scipy.sparse
from helper.processing.bbox_regression import bbox_overlaps

class IMDB(object):
    def __init__(self, name):
        self.name = name
        self.classes = []
        self.num_classes = 0
        self.image_set_index = []
        self.num_images = 0

        self.config = {}

    def image_path_from_index(self, index):
        raise NotImplementedError

    def gt_roidb(self):
        raise NotImplementedError

    def roidb(self, gt_roidb):
        raise NotImplementedError

    def create_roidb_from_box_list(self, box_list, gt_roidb):
        """
        given ground truth, prepare roidb
        :param box_list: [image_index][box_index][x1, x2, y1, y2]
        :param gt_roidb: [image_index]['boxes', 'gt_classes', 'gt_overlaps', 'flipped']
        :return: roidb: [image_index]['boxes', 'gt_classes', 'gt_overlaps', 'flipped']
        """
        assert len(box_list) == self.num_images, 'number of boxes matrix must match number of images'
        roidb = []
        for i in range(self.num_images):
            boxes = box_list[i]
            num_boxes = boxes.shape[0]
            overlaps = np.zeros((num_boxes, self.num_classes), dtype=np.float32)

            if gt_roidb is not None:
                gt_boxes = gt_roidb[i]['boxes']
                gt_classes = gt_roidb[i]['gt_classes']
                # n boxes and k gt_boxes => n * k overlap
                gt_overlaps = bbox_overlaps(boxes.astype(np.float), gt_boxes.astype(np.float))

                # for each box in n boxes, select only maximum overlap (must be greater than zero)
                argmaxes = gt_overlaps.argmax(axis=1)
                maxes = gt_overlaps.max(axis=1)
                I = np.where(maxes > 0)[0]
                overlaps[I, gt_classes[argmaxes[I]]] = maxes[I]

            overlaps = scipy.sparse.csr_matrix(overlaps)
            roidb.append({'boxes': boxes,
                          'gt_classes': np.zeros((num_boxes,), dtype=np.int32),
                          'gt_overlaps': overlaps,
                          'flipped': False})

        return roidb

    @staticmethod
    def merge_roidbs(a, b):
        """
        merge roidbs into one
        :param a: roidb to be merged into
        :param b: roidb to be merged
        :return: merged imdb
        """
        assert len(a) == len(b)
        for i in range(len(a)):
            a[i]['boxes'] = np.vstack((a[i]['boxes'], b[i]['boxes']))
            a[i]['gt_classes'] = np.hstack((a[i]['gt_classes'], b[i]['gt_classes']))
            a[i]['gt_overlaps'] = scipy.sparse.vstack([a[i]['gt_overlaps'], b[i]['gt_overlaps']])
        return a

    def append_flipped_images(self, roidb):
        """
        append flipped images to an roidb
        flip boxes coordinates, images will be actually flipped when loading into network
        :param roidb: [image_index]['boxes', 'gt_classes', 'gt_overlaps', 'flipped']
        :return: roidb: [image_index]['boxes', 'gt_classes', 'gt_overlaps', 'flipped']
        """
        print 'append flipped images to roidb'
        import cv2
        assert self.num_images == len(roidb)
        widths = [cv2.imread(self.image_path_from_index(self.image_set_index[i])).shape[1]
                  for i in range(self.num_images)]
        for i in range(self.num_images):
            boxes = roidb[i]['boxes'].copy()
            oldx1 = boxes[:, 0].copy()
            oldx2 = boxes[:, 2].copy()
            boxes[:, 0] = widths[i] - oldx2 - 1
            boxes[:, 2] = widths[i] - oldx1 - 1
            assert (boxes[:, 2] >= boxes[:, 0]).all()
            entry = {'boxes': boxes,
                     'gt_classes': roidb[i]['gt_classes'],
                     'gt_overlaps': roidb[i]['gt_overlaps'],
                     'flipped': True}
            roidb.append(entry)

        self.image_set_index *= 2
        return roidb

    def evaluate_detections(self, detections):
        raise NotImplementedError


text:
import numpy as np
import open3d as o3d

from pydrake.multibody.parsing import (Parser, ProcessModelDirectives,
                                       LoadModelDirectives)
from pydrake.multibody.plant import AddMultibodyPlantSceneGraph
from pydrake.systems.framework import DiagramBuilder

from manipulation.open3d_utils import (create_open3d_point_cloud,
                                       open3d_cloud_to_drake)
from manipulation.scenarios import AddRgbdSensors
from manipulation.utils import FindResource, AddPackagePaths

def MustardExampleSystem():
    builder = DiagramBuilder()

    # Create the physics engine + scene graph.
    plant, scene_graph = AddMultibodyPlantSceneGraph(builder, time_step=0.0)
    parser = Parser(plant)
    AddPackagePaths(parser)
    ProcessModelDirectives(
        LoadModelDirectives(FindResource("models/mustard_w_cameras.yaml")),
        plant, parser)

    plant.Finalize()

    # Add a visualizer just to help us see the object.
    use_meshcat = False
    if use_meshcat:
        meshcat = builder.AddSystem(MeshcatVisualizer(scene_graph))
        builder.Connect(scene_graph.get_query_output_port(),
                        meshcat.get_geometry_query_input_port())

    AddRgbdSensors(builder, plant, scene_graph)

    diagram = builder.Build()
    diagram.set_name("depth_camera_demo_system")
    return diagram

def MustardPointCloud():
    system = MustardExampleSystem()
    context = system.CreateDefaultContext()
    pcd = []
    for i in range(3):
        point_cloud = system.GetOutputPort(f"camera{i}_point_cloud").Eval(
            context)

        cloud = create_open3d_point_cloud(point_cloud)
        # Crop to region of interest.
        pcd.append(
            cloud.crop(
                o3d.geometry.AxisAlignedBoundingBox(min_bound=[-.3, -.3, -.3],
                                                    max_bound=[.3, .3, .3])))
    # Merge point clouds.
    merged_pcd = pcd[0] + pcd[1] + pcd[2]
    # Voxelize down-sample.
    down_sampled_pcd = merged_pcd.voxel_down_sample(voxel_size=0.005)

    return open3d_cloud_to_drake(down_sampled_pcd)


text:
import numpy as np
from scipy.spatial.distance import euclidean as dist

class Node:
  def __init__(self,point,value):
    self.value = value
    self.point = point
    self.parent = None
    self.H = 0
    self.G = 0

def children(point,grid):
  y,x = point
  h,w = grid.shape
  return [grid[r,c] for r in range(y-1,y+2)
                    for c in range(x-1,x+2)
                    if  r in range(h)  and
                        c in range(w)  and
                        (r!=y or c!=x) and
                        grid[r,c].value==1]

def create_grid(arr):
  return np.array([[Node((r,c),int(arr[r,c])) for c in range(arr.shape[1])] for r in range(arr.shape[0])])

def astar(start, goal, grid):
  openset = set()
  closedset = set()
  current = grid[start]
  openset.add(current)

  #While the open set is not empty
  while openset:
    current = min(openset,key=lambda o:o.G+o.H) # current is item with lowest F score in openset

    #If it is the item we want, retrace the path and return it
    if current.point == goal:
      path = []
      while current.parent:
        path.append(current)
        current = current.parent
      path.append(current)
      return path[::-1]

    openset.remove(current)
    closedset.add(current)

    for node in children(current.point,grid):
      if node in closedset:
        continue

      if node in openset:
        new_g = current.G + dist(current.point,node.point)
        if node.G > new_g:
          node.G = new_g
          node.parent = current
      else:
        node.G = current.G + dist(current.point,node.point)
        node.H = dist(node.point,goal)
        node.parent = current
        openset.add(node)

  #Throw an exception if there is no path
  raise ValueError('No Path Found')

'''
import random
mat = np.array([[1,0,0,0],
                [1,1,1,0],
                [1,1,1,1],
                [0,0,0,0]])
grid = np.array([[Node((r,c),mat[r,c]) for c in range(mat.shape[1])] for r in range(mat.shape[0])])
start = random.sample([(r,c) for c in range(grid.shape[1]) for r in range(grid.shape[0]) if grid[r,c].value==1],1)[0]
goal = random.sample([(r,c) for c in range(grid.shape[1]) for r in range(grid.shape[0]) if grid[r,c].value==1 and (r,c)!=start],1)[0]
print(start, goal)
print(np.array([[grid[r,c].value for c in range(grid.shape[1])] for r in range(grid.shape[0])]))
print(np.array([[1 if (r,c)==start else 2 if (r,c)==goal else 0 for c in range(grid.shape[1])] for r in range(grid.shape[0])]))
print([c.point for c in children(grid[start].point,grid)])
print([node.point for node in astar(grid[start],grid[goal],grid)])
'''

max_stars_repo_name: DramaCow/SRDetection

max_issues_repo_name: DramaCow/SRDetection

max_forks_repo_name: DramaCow/SRDetection


text:
import os
import os.path as osp

from Sample_points import SamplePoints_N
import scipy.io as io
import numpy as np

def txt_PointsCloud_parser(path_to_off_file):
    # Read the OFF file
    with open(path_to_off_file, 'r') as f:
        contents = f.readlines()
    num_vertices = len(contents)
    # print(num_vertices)
    # Convert all the vertex lines to a list of lists
    vertex_list = [list(map(float, contents[i].strip().split(' '))) for i in list(range(0, num_vertices))]
    # Return the vertices as a 3 x N numpy array

    return np.array(vertex_list)

if __name__== '__main__':

    input_pairs = []
    dataset_root_path = 'ModelNet40'
    processed_path = 'ModelNet40_'
    test = 1
    print('Processing')
    #  List of tuples grouping a label with a class
    gt_key = os.listdir(dataset_root_path)
    #gt_key_list = ['bathtub', 'bed', 'chair', 'desk', 'dresser', 'monitor', 'night_stand', 'sofa', 'table', 'toilet']
    # print('gt_key {}'.format(gt_key))
    for idx, obj in enumerate(gt_key):

        if test:
            path_to_files = osp.join(dataset_root_path, obj, 'test')
        else:
            path_to_files = osp.join(dataset_root_path, obj, 'train')
        files = os.listdir(path_to_files)
        filepaths = [(osp.join(path_to_files, file), idx)
                     for file in files]
        input_pairs = input_pairs + filepaths
        #print(input_pairs)

    for i, (path, idx ) in enumerate(input_pairs):
        floader_name = gt_key[idx]
        #floader_name = 'bathtub' ....
        if test:
            save_path = os.path.join(processed_path,floader_name, 'test')

        else:
            save_path = os.path.join(processed_path,floader_name, 'train')

        isExists = os.path.exists(save_path)
        if not isExists:						#判断如果文件不存在,则创建
            os.makedirs(save_path)
            print("%s 目录创建成功"%floader_name)

       # file_path_to_save =save_path + '/'+path.split('/')[-1].split('.')[0]+ '.points'

        file_path_to_save = os.path.join(save_path,floader_name+str(i)+ '.points')

        vertices = SamplePoints_N(path, 2000)
        # # import scipy.io as io
        np.savetxt(file_path_to_save,np.array(vertices))
        print(file_path_to_save)

        # else:
            # print("%s 目录已经存在"%i)
            # continue			#如果文件不存在,则继续上述操作,直到循环结束

        # print(path,idx, i)

max_stars_repo_name: packyan/DGCNN-Pytorch

max_issues_repo_name: packyan/DGCNN-Pytorch

max_forks_repo_name: packyan/DGCNN-Pytorch


text:
import os
import itertools as it
from ..tune.hyperparametertuning import tuning_wraper
from imblearn.under_sampling import RandomUnderSampler
import time
import multiprocessing as mp
from functools import partial
import argparse
import numpy as np
import torch
import pickle as pkl
import ast

parser = argparse.ArgumentParser()
parser.add_argument('--lr', type=float, nargs = "+",
                    help='learning rate')

parser.add_argument('--l2', type=float, nargs= "+",
                    help='weight decay')

parser.add_argument('--ks',  type=int, nargs='+', action="append",
                    help='kernel size')

parser.add_argument('--out-channels', type=int, nargs='+', action="append",
                    help='Channels')

parser.add_argument('--mini-batch-percent', type=float, nargs = "+",
                    help='Percent for mini batch')

parser.add_argument('--data-container', type=str,
                    help='data container path (after MIL)')

parser.add_argument('--epochs', type=int, default=30,
                    help='Number of epochs')

parser.add_argument('--out-dir', type=str,
                    help='output directory')

parser.add_argument('--adopt-dir', type=str, nargs="+",
                    help='ADOPT directory')

parser.add_argument('--run-methods', default =[True, True, True], nargs='+',
                    help='Methods for hyper param')

parser.add_argument('--load-models', type=bool, default=False,
                    help='Load already trained trial models')

parser.add_argument('--use-stratisfy', type=bool, default=False,
                    help='True use stratisfied mini-batch strategy')

parser.add_argument('--aggregation', type=str, default="maxpool",
                    help='aggregation type for logistic regression model')

parser.add_argument('--dropout', type=float, nargs="+", default=0.0,
                    help='weight decay')

parser.add_argument('--gru-hidden', type=int, default=5,
                    help='Hidden layers for GRU')

parser.add_argument('--model-type', type=str, choices=["lr_binary","imdope_binary", "imdope_mc"],
                    help='Model type')

parser.add_argument('--use-cuda', type=bool, default=False,
                    help='True uses cuda')

parser.add_argument('--n-classes', type=int, default=2,
                    help='Number of classes')

parser.add_argument('--optimizer', type=str, choices=["SGD", "adam"], default="adam",
                    help='Optimizer to use')

args = parser.parse_args()

if args.load_models :
    load_models = True
else :
    load_models = False

if args.use_stratisfy:
    use_stratisfy = True
else :
    use_stratisfy = False

if args.use_cuda:
    device = "cuda:0"
else:
    device = "cpu"

run_methods = [ast.literal_eval(i) for i in args.run_methods]

###### Data Processing
#%%
if __name__ == "__main__":
    myseed = 45237552
    np.random.seed(120)
    torch.manual_seed(myseed)

    with open(args.data_container, "rb") as f:
        dm = pkl.load(f)

    # Shaping data for MIL
    # dm.MIL_processing(test_ratio=args.test_ratio, val_ratio=args.val_ratio, anomaly=[args.anomaly])

    # Normalizing data & Saving unormalized values for anomalies and flight ids
    # dc.normalize_resample_data(sampling_method=None)

    ####### Parameters definition
    #%%
    lr = args.lr #[.01]#, .01]
    # print(lr)

    l2 = args.l2 #[0.01, 0.005] #[0.01, 0.005, 0.001]
    ks = args.ks #[[8, 5, 3]] #[[8, 5, 3],
                  # [10, 5 ,3],
                  # [8, 6, 4],
                  # [6, 3, 2]]
    # print(ks)
    out_channels = args.out_channels #[[16, 32, 64]] #[[16, 32, 64],
                     # [64, 128, 256],
                     # [16, 48, 144]]
    mini_batch = args.mini_batch_percent#[0.5]# [1, 0.5] #[1, 0.5, 0.25]
    last_layer = ["cnn"]#["dense", "cnn"]
    batch_norm = [True] #[True, False]
    epochs = [args.epochs]
    mini_b_strat = [use_stratisfy]

    aggregation = [args.aggregation]
    gru_hidden_size = [args.gru_hidden]
    dropout = [args.dropout]
    model_type = [args.model_type]
    n_classes = [args.n_classes]
    if model_type[0]=="imdope_binary":
        n_classes = [1]
    optimizer = [args.optimizer]

    if mini_b_strat[0] == False:
        params = {"learning_rate":lr,
                  "l2":l2,
                  "kernel_size":ks,
                  "n_filters":out_channels,
                  "batch_size_percent":mini_batch,
                  "n_classes":n_classes,
                  "use_batch_norm":batch_norm,
                  "optimizer":optimizer,
                  "epochs":epochs,
                  "gru_hidden_size":gru_hidden_size,
                  "dropout":dropout,
                  "model_type":model_type,
                  "aggregation":aggregation}
    else:
        params = {"learning_rate":lr,
                  "l2":l2,
                  "kernel_size":ks,
                  "n_filters":out_channels,
                  "batch_size_percent":mini_batch,
                  "n_classes":n_classes,
                  "use_batch_norm":batch_norm,
                  "optimizer":optimizer,
                  "epochs":epochs,
                  "use_stratified_batch_size":mini_b_strat,
                  "gru_hidden_size": gru_hidden_size,
                  "dropout": dropout,
                  "model_type": model_type,
                  "aggregation": aggregation
                  }

    n_processes = 2
    list_params = []
    procs = []
    load_trained_model = load_models
    combs = []
    dm.trainX, dm.valX = dm.trainX.astype(np.float64), dm.valX.astype(np.float64)
    data = {"X_train": dm.trainX.astype(np.float64), "y_train": dm.trainY,
            "data_model": dm,
            "X_test": dm.valX, "y_test": dm.valY}
#     print(dm.trainX.shape)
#     print(np.unique(dm.trainY))
    for name, _ in params.items():
        list_params.append(name)

    all_combinations = list(it.product(*(params[name] for name in list_params)))
    all_combinations = [list(tuple) for tuple in all_combinations]
    [i.insert(0, counter) for counter, i in enumerate(all_combinations)] # insert index of file

    print(f"Number of Combinations: {len(all_combinations)}")
    # %% Run trials
    if not os.path.exists("{}".format(args.out_dir)):
        os.makedirs("{}".format(args.out_dir))

    with open('{}/all_combinations.txt'.format(args.out_dir), 'a') as filehandle:
        filehandle.write(('%s\n' % str(list_params)))
        for listitem in all_combinations:
            filehandle.write('%s\n' % str(listitem))
    start = time.time()

    for run, combination in enumerate(all_combinations):
        path = os.path.join("{}".format(args.out_dir), f"Combination_{combination[0]}")
        if not load_trained_model:
            if os.path.exists(path):
                if os.path.exists(os.path.join(path,"completed.txt")):
                    continue
            else:
                os.makedirs(path)
        else:
            if not os.path.exists(os.path.join(path,"trial_model.pt")):
                continue

        myinput = {"combination": combination[1:],
                   "dm": dm, "list_params": list_params,
                   "path": path, "device": device}
        combs.append(myinput)
    # if ("cuda" in device) or ("cpu" in device):
    for iteration, comb in enumerate(combs):
        print(f"Starting combination_{iteration}")
        tuning_wraper(data, comb, load_trained_model,
        run_methods=run_methods, path_to_ADOPT_run=args.adopt_dir)
    # else:
    #     pool = mp.Pool(processes=n_processes)#processes=n_processes)
    #     func = partial(tuning_wraper, data)
    #     pool.map(func, combs)
    #     pool.close()
    #     # pool.join()
    #     # proc = Process(target=fun, args=(myinput,))
    #     # procs.append(proc)
    #     # proc.start()

    #     # temp_dict = dict(zip(list_params, combination))
    #     #
    #     # mt = model_trial(path, dm, temp_dict)
    #     # mt.init_my_model(X_train.shape[2]-2, X_train.shape[1], device)
    #     # mt.fit(X_train=X_res[:-100], y_train=y_res[:-100], l2=temp_dict["l2"], learning_rate=temp_dict["learning_rate"],
    #     #        num_epochs=temp_dict["epochs"])
    #     # mt.evaluate_basic_metrics(X_test[:-100], y_test[:-100])
    #     # mt.evaluate_ADOPT_overlap()
    #     # mt.evaluate_best_feature_overlap()
    #     # mt.save_all()
    # # for proc in procs:
    # #     proc.join()
    end_time = time.time()-start
    with open(f'./{args.out_dir}/all_combinations.txt', 'a+') as filehandle:
        filehandle.write(f"Total Time (min): {end_time/60}\n")