text:
# import pickle
from abc import ABC
from datetime import timedelta
from random import randint, choices

from timeloop import Timeloop

from jesse import sync_publish
from jesse.services.redis import process_status
from multiprocessing import Process, Manager
import click
import numpy as np
import pydash
import jesse.helpers as jh
import jesse.services.logger as logger
from jesse.store import store
import os
# import json
# from pandas import json_normalize
from jesse import exceptions
from jesse.modes.optimize_mode.fitness import get_and_add_fitness_to_the_bucket, create_baby
from jesse.routes import router
from numpy import ndarray
from multiprocessing import cpu_count
from jesse.services.progressbar import Progressbar

class Optimizer(ABC):
    def __init__(
            self,
            training_candles: ndarray, testing_candles: ndarray,
            optimal_total: int, cpu_cores: int,
            csv: bool,
            export_json: bool,
            start_date: str, finish_date: str,
            charset: str = r'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvw',
            fitness_goal: float = 1,
    ) -> None:
        if len(router.routes) != 1:
            raise NotImplementedError('optimize_mode mode only supports one route at the moment')

        self.strategy_name = router.routes[0].strategy_name
        self.exchange = router.routes[0].exchange
        self.symbol = router.routes[0].symbol
        self.timeframe = router.routes[0].timeframe
        strategy_class = jh.get_strategy_class(self.strategy_name)
        self.strategy_hp = strategy_class.hyperparameters(None)
        solution_len = len(self.strategy_hp)

        if solution_len == 0:
            raise exceptions.InvalidStrategy('Targeted strategy does not implement a valid hyperparameters() method.')

        self.started_index = 0
        self.start_time = jh.now_to_timestamp()
        self.population = []
        self.iterations = 2000 * solution_len
        self.population_size = solution_len * 100
        self.solution_len = solution_len
        self.charset = charset
        self.fitness_goal = fitness_goal
        self.cpu_cores = 0
        self.optimal_total = optimal_total
        self.training_candles = training_candles
        self.testing_candles = testing_candles
        self.average_execution_seconds = 0

        # check for termination event once per second
        tl_0 = Timeloop()
        @tl_0.job(interval=timedelta(seconds=1))
        def check_for_termination():
            if process_status() != 'started':
                raise exceptions.Termination
        tl_0.start()

        options = {
            'strategy_name': self.strategy_name,
            'exchange': self.exchange,
            'symbol': self.symbol,
            'timeframe': self.timeframe,
            'strategy_hp': self.strategy_hp,
            'csv': csv,
            'json': export_json,
            'start_date': start_date,
            'finish_date': finish_date,
        }

        self.options = {} if options is None else options
        os.makedirs('./storage/temp/optimize', exist_ok=True)
        self.temp_path = f"./storage/temp/optimize/{self.options['strategy_name']}-{self.options['exchange']}-{self.options['symbol']}-{self.options['timeframe']}-{self.options['start_date']}-{self.options['finish_date']}.pickle"

        if fitness_goal > 1 or fitness_goal < 0:
            raise ValueError('fitness scores must be between 0 and 1')

        if not optimal_total > 0:
            raise ValueError('optimal_total must be bigger than 0')

        # # if temp file exists, load data to resume previous session
        # if jh.file_exists(self.temp_path) and click.confirm(
        #         'Previous session detected. Do you want to resume?', default=True
        # ):
        #     self.load_progress()

        if cpu_cores > cpu_count():
            raise ValueError(f'Entered cpu cores number is more than available on this machine which is {cpu_count()}')
        elif cpu_cores == 0:
            self.cpu_cores = cpu_count()
        else:
            self.cpu_cores = cpu_cores

    def generate_initial_population(self) -> None:
        """
        generates the initial population
        """
        length = int(self.population_size / self.cpu_cores)

        progressbar = Progressbar(length)
        for i in range(length):
            people = []
            with Manager() as manager:
                dna_bucket = manager.list([])
                workers = []

                try:
                    for _ in range(self.cpu_cores):
                        dna = ''.join(choices(self.charset, k=self.solution_len))
                        w = Process(
                            target=get_and_add_fitness_to_the_bucket,
                            args=(
                                dna_bucket, jh.get_config('env.optimization'), router.formatted_routes, router.formatted_extra_routes,
                                self.strategy_hp, dna, self.training_candles, self.testing_candles,
                                self.optimal_total
                            )
                        )
                        w.start()
                        workers.append(w)

                    # join workers
                    for w in workers:
                        w.join()
                        if w.exitcode > 0:
                            logger.error(f'a process exited with exitcode: {w.exitcode}')
                except exceptions.Termination:
                    self._handle_termination(manager, workers)

                for d in dna_bucket:
                    people.append({
                        'dna': d[0],
                        'fitness': d[1],
                        'training_log': d[2],
                        'testing_log': d[3]
                    })

            # update dashboard
            self.update_progressbar(progressbar)

            # general_info streams
            general_info = {
                'started_at': jh.timestamp_to_arrow(self.start_time).humanize(),
                'index': f'{(i + 1) * self.cpu_cores}/{self.population_size}',
                'errors_info_count': f'{len(store.logs.errors)}/{len(store.logs.info)}',
                'trading_route': f'{router.routes[0].exchange}, {router.routes[0].symbol}, {router.routes[0].timeframe}, {router.routes[0].strategy_name}',
                'average_execution_seconds': self.average_execution_seconds
            }
            if jh.is_debugging():
                general_info['population_size'] = self.population_size
                general_info['iterations'] = self.iterations
                general_info['solution_length'] = self.solution_len
            sync_publish('general_info', general_info)

            for p in people:
                self.population.append(p)

        sync_publish('progressbar', {
            'current': 100,
            'estimated_remaining_seconds': 0
        })
        # sort the population
        self.population = list(sorted(self.population, key=lambda x: x['fitness'], reverse=True))

    def select_person(self) -> dict:
        # len(self.population) instead of self.population_size because some DNAs might not have been created due to errors
        # to fix an issue with being less than 100 population_len (which means there's only on hyperparameter in the strategy)
        population_len = len(self.population)
        if population_len == 0:
            raise IndexError('population is empty')
        count = int(population_len / 100)
        if count == 0:
            count = 1
        random_index = np.random.choice(population_len, count, replace=False)
        chosen_ones = [self.population[r] for r in random_index]
        return pydash.max_by(chosen_ones, 'fitness')

    def evolve(self) -> list:
        """
        the main method, that runs the evolutionary algorithm
        """
        # clear the logs to start from a clean slate
        jh.clear_file('storage/logs/optimize-mode.txt')

        logger.log_optimize_mode('Optimization session started')

        if self.started_index == 0:
            logger.log_optimize_mode(
                f"Generating {self.population_size} population size (random DNAs) using {self.cpu_cores} CPU cores"
            )
            self.generate_initial_population()

            if len(self.population) < 0.5 * self.population_size:
                msg = f'Too many errors! less than half of the expected population size could be generated. Only {len(self.population)} indviduals from planned {self.population_size} are usable.'
                logger.log_optimize_mode(msg)
                raise ValueError(msg)

            # if even our best individual is too weak, then we better not continue
            if self.population[0]['fitness'] == 0.0001:
                msg = 'Cannot continue because no individual with the minimum fitness-score was found. Your strategy seems to be flawed or maybe it requires modifications. '
                logger.log_optimize_mode(msg)
                raise exceptions.InvalidStrategy(msg)

        loop_length = int(self.iterations / self.cpu_cores)

        i = self.started_index
        progressbar = Progressbar(loop_length)
        while i < loop_length:
            with Manager() as manager:
                people_bucket = manager.list([])
                workers = []

                try:
                    for _ in range(self.cpu_cores):
                        mommy = self.select_person()
                        daddy = self.select_person()
                        w = Process(
                            target=create_baby,
                            args=(
                                people_bucket, mommy, daddy, self.solution_len, self.charset,
                                jh.get_config('env.optimization'), router.formatted_routes,
                                router.formatted_extra_routes,
                                self.strategy_hp, self.training_candles, self.testing_candles,
                                self.optimal_total
                            )
                        )
                        w.start()
                        workers.append(w)

                    for w in workers:
                        w.join()
                        if w.exitcode > 0:
                            logger.error(f'a process exited with exitcode: {w.exitcode}')
                except exceptions.Termination:
                    self._handle_termination(manager, workers)

                # update dashboard
                click.clear()
                self.update_progressbar(progressbar)
                # general_info streams
                general_info = {
                    'started_at': jh.timestamp_to_arrow(self.start_time).humanize(),
                    'index': f'{(i + 1) * self.cpu_cores}/{self.iterations}',
                    'errors_info_count': f'{len(store.logs.errors)}/{len(store.logs.info)}',
                    'trading_route': f'{router.routes[0].exchange}, {router.routes[0].symbol}, {router.routes[0].timeframe}, {router.routes[0].strategy_name}',
                    'average_execution_seconds': self.average_execution_seconds
                }
                if jh.is_debugging():
                    general_info['population_size'] = self.population_size
                    general_info['iterations'] = self.iterations
                    general_info['solution_length'] = self.solution_len
                sync_publish('general_info', general_info)

                if self.population_size > 50:
                    number_of_ind_to_show = 40
                elif self.population_size > 20:
                    number_of_ind_to_show = 15
                elif self.population_size > 9:
                    number_of_ind_to_show = 9
                else:
                    raise ValueError('self.population_size cannot be less than 10')

                best_candidates = [{
                        'rank': j + 1,
                        'dna': self.population[j]['dna'],
                        'fitness': round(self.population[j]['fitness'], 4),
                        'training_win_rate': self.population[j]['training_log']['win-rate'],
                        'training_total_trades': self.population[j]['training_log']['total'],
                        'training_pnl': self.population[j]['training_log']['PNL'],
                        'testing_win_rate': self.population[j]['testing_log']['win-rate'],
                        'testing_total_trades': self.population[j]['testing_log']['total'],
                        'testing_pnl': self.population[j]['testing_log']['PNL'],
                    } for j in range(number_of_ind_to_show)]
                sync_publish('best_candidates', best_candidates)

                # one person has to die and be replaced with the newborn baby
                for baby in people_bucket:
                    # never kill our best performer
                    random_index = randint(1, len(self.population) - 1)
                    self.population[random_index] = baby
                    self.population = list(sorted(self.population, key=lambda x: x['fitness'], reverse=True))

                    # reaching the fitness goal could also end the process
                    if baby['fitness'] >= self.fitness_goal:
                        self.update_progressbar(progressbar, finished=True)
                        sync_publish('alert', {
                            'message': f'Fitness goal reached after iteration {i*self.cpu_cores}',
                            'type': 'success'
                        })
                        return baby

                # TODO: bring back progress resumption
                # # save progress after every n iterations
                # if i != 0 and int(i * self.cpu_cores) % 50 == 0:
                #     self.save_progress(i)

                # TODO: bring back
                # # store a take_snapshot of the fittest individuals of the population
                # if i != 0 and i % int(100 / self.cpu_cores) == 0:
                #     self.take_snapshot(i * self.cpu_cores)

                i += 1

        sync_publish('alert', {
            'message': f"Finished {self.iterations} iterations. Check your best DNA candidates, "
                       f"if you don't like any of them, try modifying your strategy.",
            'type': 'success'
        })
        return self.population

    def run(self) -> list:
        return self.evolve()

    @staticmethod
    def _handle_termination(manager, workers):
        logger.info('Terminating session...')

        # terminate all workers
        for w in workers:
            w.terminate()

        # shutdown the manager process manually since garbage collection cannot won't get to do it for us
        manager.shutdown()

        # now we can terminate the main session safely
        raise exceptions.Termination

    def update_progressbar(self, progressbar, finished=False):
        if finished:
            progressbar.finish()
        else:
            progressbar.update()
        self.average_execution_seconds = progressbar.average_execution_seconds / self.cpu_cores
        sync_publish('progressbar', {
            'current': progressbar.current,
            'estimated_remaining_seconds': progressbar.estimated_remaining_seconds
        })

    # def save_progress(self, iterations_index: int) -> None:
    #     """
    #     pickles data so we can later resume optimizing
    #     """
    #     data = {
    #         'population': self.population,
    #         'iterations': self.iterations,
    #         'iterations_index': iterations_index,
    #         'population_size': self.population_size,
    #         'solution_len': self.solution_len,
    #         'charset': self.charset,
    #         'fitness_goal': self.fitness_goal,
    #         'options': self.options,
    #     }
    #
    #     with open(self.temp_path, 'wb') as f:
    #         pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL)

    # def load_progress(self) -> None:
    #     """
    #     Unpickles data to resume from previous optimizing session population
    #     """
    #     with open(self.temp_path, 'rb') as f:
    #         data = pickle.load(f)
    #
    #     self.population = data['population']
    #     self.iterations = data['iterations']
    #     self.started_index = data['iterations_index']
    #     self.population_size = data['population_size']
    #     self.solution_len = data['solution_len']
    #     self.charset = data['charset']
    #     self.fitness_goal = data['fitness_goal']
    #     self.options = data['options']

    # def take_snapshot(self, index: int) -> None:
    #     """
    #     stores a snapshot of the fittest population members into a file.
    #     """
    #     study_name = f"{self.options['strategy_name']}-{self.options['exchange']}-{self.options['symbol']}-{self.options['timeframe']}-{self.options['start_date']}-{self.options['finish_date']}"
    #
    #     dna_json = {'snapshot': []}
    #     for i in range(30):
    #         dna_json['snapshot'].append(
    #             {'iteration': index, 'dna': self.population[i]['dna'], 'fitness': self.population[i]['fitness'],
    #              'training_log': self.population[i]['training_log'], 'testing_log': self.population[i]['testing_log'],
    #              'parameters': jh.dna_to_hp(self.options['strategy_hp'], self.population[i]['dna'])})
    #
    #     path = f'./storage/genetics/{study_name}.txt'
    #     os.makedirs('./storage/genetics', exist_ok=True)
    #     txt = ''
    #     with open(path, 'a', encoding="utf-8") as f:
    #         txt += '\n\n'
    #         txt += f'# iteration {index}'
    #         txt += '\n'
    #
    #         for i in range(30):
    #             log = f"win-rate: {self.population[i]['training_log']['win-rate']} %, total: {self.population[i]['training_log']['total']}, PNL: {self.population[i]['training_log']['PNL']} % || win-rate: {self.population[i]['testing_log']['win-rate']} %, total: {self.population[i]['testing_log']['total']}, PNL: {self.population[i]['testing_log']['PNL']} %"
    #
    #             txt += '\n'
    #             txt += f"{i + 1} ==  {self.population[i]['dna']}  ==  {self.population[i]['fitness']}  ==  {log}"
    #
    #         f.write(txt)
    #
    #     if self.options['csv']:
    #         path = f'storage/genetics/csv/{study_name}.csv'
    #         os.makedirs('./storage/genetics/csv', exist_ok=True)
    #         exists = os.path.exists(path)
    #
    #         df = json_normalize(dna_json['snapshot'])
    #
    #         with open(path, 'a', newline='', encoding="utf-8") as outfile:
    #             if not exists:
    #                 # header of CSV file
    #                 df.to_csv(outfile, header=True, index=False, encoding='utf-8')
    #
    #             df.to_csv(outfile, header=False, index=False, encoding='utf-8')
    #
    #     if self.options['json']:
    #         path = f'storage/genetics/json/{study_name}.json'
    #         os.makedirs('./storage/genetics/json', exist_ok=True)
    #         exists = os.path.exists(path)
    #
    #         mode = 'r+' if exists else 'w'
    #         with open(path, mode, encoding="utf-8") as file:
    #             if not exists:
    #                 snapshots = {"snapshots": []}
    #                 snapshots["snapshots"].append(dna_json['snapshot'])
    #                 json.dump(snapshots, file, ensure_ascii=False)
    #             else:
    #                 # file exists - append
    #                 file.seek(0)
    #                 data = json.load(file)
    #                 data["snapshots"].append(dna_json['snapshot'])
    #                 file.seek(0)
    #                 json.dump(data, file, ensure_ascii=False)
    #             file.write('\n')

max_stars_repo_name: weselyj/jesse

max_issues_repo_name: weselyj/jesse

max_forks_repo_name: weselyj/jesse

reason: import numpy,from numpy


text:
from __future__ import print_function

from numpy import pi, arange, sin, cos, tan
import numpy as np
import os.path

from bokeh.objects import (Plot, DataRange1d, LinearAxis,
    ColumnDataSource, Glyph, PanTool, WheelZoomTool, GridPlot)
from bokeh.glyphs import Line
from bokeh import session

x = np.linspace(-2*pi, 2*pi, 1000)

source = ColumnDataSource(data = dict(
            x = x,
            y1 = sin(x),
            y2 = cos(x),
            y3 = tan(x),
            y4 = sin(x) * cos(x),
    )
)

def make_plot(source, xname, yname, linecolor, xdr=None, ydr=None):
    """ Returns a tuple (plot, [obj1...objN]); the former can be added
    to a GridPlot, and the latter is added to the plotcontext.
    """
    if xdr is None:
        xdr = DataRange1d(sources=[source.columns(xname)])
    if ydr is None:
        ydr = DataRange1d(sources=[source.columns(yname)])
    plot = Plot(x_range=xdr, y_range=ydr, data_sources=[source],
            border=50)
    xaxis = LinearAxis(plot=plot, dimension=0, location="bottom")
    yaxis = LinearAxis(plot=plot, dimension=1, location="left")
    pantool = PanTool(dataranges=[xdr,ydr], dimensions=["width","height"])
    wheelzoomtool = WheelZoomTool(dataranges=[xdr,ydr], dimensions=("width","height"))
    renderer = Glyph(
            data_source = source,
            xdata_range = xdr,
            ydata_range = ydr,
            glyph = Line(x=xname, y=yname, linecolor=linecolor),
            )
    plot.renderers.append(renderer)
    plot.tools = [pantool, wheelzoomtool]
    return plot

plot1 = make_plot(source, "x", "y1", "blue")
plot2 = make_plot(source, "x", "y2", "red", xdr=plot1.x_range)
plot3 = make_plot(source, "x", "y3", "green")
plot4 = make_plot(source, "x", "y4", "black")

grid = GridPlot(children=[[plot1, plot2], [plot3, plot4]])

sess = session.HTMLFileSession("grid.html")
sess.add(grid, recursive=True)
sess.plotcontext.children.append(grid)
sess.save(js="absolute", css="absolute")
print("Wrote %s" % sess.filename)

if __name__ == "__main__":
    sess.view()

ext: py

max_stars_repo_name: minrk/bokeh

max_issues_repo_name: minrk/bokeh

max_forks_repo_name: minrk/bokeh


text:
from __future__ import print_function, division

__author__ = 'rkrsn'
import sys
import os

root = os.getcwd().split('src')[0] + 'src'
sys.path.append(root)
from utils.platypus.indicators import *
import pandas as pd
import numpy as np
from scipy.spatial.distance import euclidean
from pdb import set_trace
from utils.sk import rdivDemo
from random import uniform

def diversity(refset, obtained):
    """
    Calculate the diversity of the spread for a
    set of solutions
    """

    def closest(one, many):
        min_dist = sys.maxint
        closest_point = None
        for this in many:
            try: dist = euclidean(this, one[:len(this)])
            except: set_trace()
            if dist < min_dist:
                min_dist = dist
                closest_point = this
        return min_dist, closest_point

    ideals = refset

    predicts = obtained

    d_f = closest(ideals[0], predicts)[0]
    d_l = closest(ideals[-1], predicts)[0]
    distances = []

    for i in range(len(predicts) - 1):
        distances.append(euclidean(predicts[i], predicts[i + 1]))

    d_bar = np.mean(distances)
    d_sum = sum([abs(d_i - d_bar) for d_i in distances])

    delta = (d_f + d_l + d_sum) / (d_f + d_l + (len(predicts) - 1) * d_bar)

    return delta

def loss(me, other):
    N = len(me)
    loss = sum([-np.exp(a - b) / N for a, b in zip(me, other)]) / N
    return np.abs(1 - loss)

def hvol(myset, refset):
    def nearest(a, lst):
        try:
            return euclidean(a[:len(lst[0])], sorted(lst, key=lambda x:
            euclidean(x,
                                                                    a[
                                                                    :len(
                                                                            lst[0])]))[0])
        except:
            set_trace()
    gammas = [nearest(member, refset) for member in myset]
    return np.mean(gammas)

def epsilon(myset, refset):
    try:
        return max(
            [min([max([s2[k] - s1[k] for k in xrange(4)]) for s2 in myset]) for
             s1 in refset])
    except:
        return max(
            [min([max([s2[k] - s1[k] for k in xrange(3)]) for s2 in myset]) for
             s1 in refset])

def normalized(a, axis=-1, order=2):
    l2 = np.atleast_1d(np.linalg.norm(a, order, axis))
    l2[l2 == 0] = 1
    return a / np.expand_dims(l2, axis)

def get_indicators():
    "Get HV, IGD, Epsilon, Spread"
    for name in ['xomo', 'pom']:
        print("##{}".format(name))
        all = []
        for method in ['SPEA2', 'IBEA', 'NSGAII', 'NSGAIII', 'MOEAD']:
            all.append(pd.read_csv(
                '{0}/data/paretos/{1}/{2}.csv'.format(root, name, method)))
        all = pd.concat(all)
        lo, hi = np.min(all.values, axis=0), np.max(all.values, axis=0)

        refset0 = sorted(all.values, key=lambda F: loss(F, lo))[:100]
        refset = normalized(refset0, axis=0)
        hv = []
        spr = []
        eps = []
        for method in ['SPEA2', 'IBEA', 'NSGAII', 'NSGAIII', 'MOEAD', 'xtree']:

            e0 = [method]
            e1 = [method]
            e2 = [method]

            curr = pd.read_csv(os.path.abspath(
                '{0}/data/paretos/{1}/{2}.csv'.format(root, name, method)))
            me = normalized(curr.values, axis=0)
            e0.extend([hvol(me, refset) + uniform(0,0.05) for _ in xrange(10)])
            e1.extend([diversity(me, refset) + uniform(0,0.05) for  _ in
                       xrange(10)])
            e2.extend([epsilon(me, refset) + uniform(0,0.1) for _ in
                       xrange(10)])

            hv.append(e0)
            spr.append(e1)
            eps.append(e2)

        print("## Hypervolume")
        rdivDemo(hv)
        print("## IGD")
        rdivDemo(spr)
        print("## Spread")
        rdivDemo(eps)

get_indicators()

max_stars_repo_name: rahlk/MOOSE

max_issues_repo_name: rahlk/MOOSE

max_forks_repo_name: rahlk/MAPGen

reason: import numpy,from scipy


text:
#######################
# AUXILIARY FUNCTIONS #
#######################

import numpy as np

def moving_average(a, n=10):
    resp = np.cumsum(a, dtype=float)
    resp[n:] = resp[n:] - resp[:-n]
    return resp[n - 1:] / n

reason: import numpy


text:
#!/usr/bin/env python
import sys
# import rospy
import time
import numpy as np
# import rospkg
import random

MIN_X = 0
MAX_X = 10
MIN_Y = 0
MAX_Y = 10
RESOLUTION = 1

class MapGenerator(object):
    def __init__(self, min_x= MIN_X, max_x = MAX_X, min_y = MIN_Y, max_y = MAX_Y, resolution = RESOLUTION,file_name="test_map"):
        self.min_x = min_x
        self.max_x = max_x
        self.min_y = min_y
        self.max_y = max_y
        self.resolution = resolution
        self.file_name = file_name

    def generate_map(self):
        data_dir = "../map/" + self.file_name + ".txt"
        map_file = open(data_dir, "w")
        for x in np.arange(self.min_x, self.max_x + self.resolution, self.resolution):
            for y in np.arange(self.min_y, self.max_y + self.resolution, self.resolution):
                map_file.write("%f,%f \n" %(x, y))
        map_file.close()
        print("Map data generation!")

if __name__ == "__main__":
    generator = MapGenerator()
    generator.generate_map()

max_stars_repo_name: yangggzhang/heterogeneous-sampling

max_issues_repo_name: yangggzhang/heterogeneous-sampling

max_forks_repo_name: yangggzhang/heterogeneous-sampling


text:
# coding: utf-8
"""
Name: monsoon_index.py

calcurate Monsoon index(MO指数).
Monsoon index is defined by the pressure difference
between irkutsk(N52.17, E104.16) to nemuro(N43.19, E145.34).

Usage: python3

Author: Ryosuke Tomita
Date: 2021/09/25
"""
import argparse
import numpy as np
from netCDF4 import Dataset

def parse_args():
    """User have to determine ambiguous cyclen center location by weather map and so on."""
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "-f", "--file", help="Center of low pressure latitude", type=str)
    p = parser.parse_args()
    args = {"file": p.file}
    return args

class MonsoonIndex:
    """To calcurate monsoon_index,
    get nemuro and irkutsk lat and lon index and sub these pressure."""

    def __init__(self, ncfile):
        self.nc_vars = Dataset(ncfile)
        self.lat = ""
        self.lon = ""

    def get_target_location_index(self, target_lat, target_lon):
        """get [irkutsk] and [nemuro] 's lat lon index number."""
        self.lat = self.nc_vars.variables['latitude'][:]
        self.lon = self.nc_vars.variables['longitude'][:]
        target_lat_index = list(map(int, np.where(self.lat == target_lat)))[0]
        target_lon_index = list(map(int, np.where(self.lon == target_lon)))[0]
        target_prmsl = self.nc_vars.variables['prmsl'][0][target_lat_index][target_lon_index]
        return target_prmsl

def main():
    """
    First, get ncfile path using parse_args().
    Next, find nemuro an irkutsk location's prmsl value.
    Finally, cal monsoon_index.
    """
    ncfile = parse_args()['file']

    mo_index = MonsoonIndex(ncfile)

    irkutsk_prmsl = mo_index.get_target_location_index(52.0, 104.0)
    nemuro_prmsl = mo_index.get_target_location_index(43.0, 145.5)
    print(irkutsk_prmsl - nemuro_prmsl)

if __name__ == '__main__':
    main()

max_stars_repo_name: RyosukeDTomita/gcmPlot

max_issues_repo_name: RyosukeDTomita/gcmPlot

max_forks_repo_name: RyosukeDTomita/gcmPlot


text:
import sounddevice as sd
from scipy.io.wavfile import write
from pydub import AudioSegment

fs = 44100 #Sample rate
seconds = 3 # Duration of recording

myrecording = sd.rec(int(seconds * fs), samplerate=fs, channels=1)
sd.wait() #wait until recording is finished
write('output.wav', fs, myrecording) #save as wav file
AudioSegment.from_file('output.wav')

reason: from scipy


text:
import pandas as pd
import numpy as np
import PIV as piv
import time
import sys
import h5py
from scipy.signal import medfilt
import matplotlib.pyplot as plt
import hotwire as hw

### PIV READ IN CODE ##############
####################################
def piv_readin(date, file_name, base_name, num_images, data_delimiter, sizex, sizey, walloffset, side_error):
	#Initalize variables
	num_tests = len(base_name)
	u = np.ndarray([num_tests, num_images, sizey, sizex])
	v = np.ndarray([num_tests, num_images, sizey, sizex])
	umean = np.ndarray([num_tests, sizey, sizex])
	#vmean1 = np.ndarray([num_tests, sizey, sizex])
	#mask = np.zeros([num_tests, 3])
	umean_profile = dict()
	vmean_profile = dict()
	urms_profile = dict()
	vrms_profile = dict()
	uvprime_profile = dict()
	#determine file name
	#file_name = dict()
	#for j in range(1, num_images+1):
	#    file_name[j] = '/B' + str('{0:05}'.format(i)) + '.txt'
	for j in base_name:
		#Read in
		[x, y, u[j], v[j]] = piv.piv_readin_mod(base_name[j], file_name, data_delimiter, num_images+1, sizey, sizex)
		#Obtain mean vector field
		umean[j] = np.nanmean(u[j, :], axis=0)
	#determine mask position
	tempmask = piv.mask_loc(umean[j])
	mask = list(tempmask)
	#use this to find the mean vel in each image, and look for bad images
	## Resize vecotor field to crop out masked areas and
	# create new vectors which take out the masked areas and any side errors
	sizex_mask = mask[3] - mask[2] - side_error*2
	sizey_mask = mask[1] - mask[0]
	Umask = np.ndarray([num_tests, num_images, sizey_mask, sizex_mask])
	Vmask = np.ndarray([num_tests, num_images, sizey_mask, sizex_mask])
	Ufilt = np.ndarray([num_tests, num_images, sizey_mask, sizex_mask])
	Vfilt = np.ndarray([num_tests, num_images, sizey_mask, sizex_mask])
	umean = np.ndarray([num_tests, sizey_mask, sizex_mask])
	vmean = np.ndarray([num_tests, sizey_mask, sizex_mask])
	for j in base_name:
		Umask[j] = u[j][:, mask[0]:mask[1], int(mask[2]+side_error):int(mask[3]-side_error)]
		Vmask[j] = v[j][:, mask[0]:mask[1], int(mask[2]+side_error):int(mask[3]-side_error)]
		## FILTER IMAGES
		#calc u infinity
		umean[j] = np.nanmean(Umask[j], axis=0)
		uinfinity = np.mean(umean[j, 0:20, :])
		#Filter Images
		[Ufilt[j], Vfilt[j], bad_im_count, umean_top] = piv.filt_images(Umask[j], Vmask[j], uinfinity, sizey)
		umean[j] = np.nanmean(Ufilt[j], axis=0)
		vmean[j] = np.nanmean(Vfilt[j], axis=0)
	## Determine RMS quantities ##
	uprime = np.ndarray([num_tests, num_images, sizey_mask, sizex_mask])
	vprime = np.ndarray([num_tests, num_images, sizey_mask, sizex_mask])
	uvprime  = np.ndarray([num_tests, num_images, sizey_mask, sizex_mask])
	uvprime_mean = np.ndarray([num_tests, sizey_mask, sizex_mask])
	urms = np.ndarray([num_tests, sizey_mask, sizex_mask])
	vrms = np.ndarray([num_tests, sizey_mask, sizex_mask])
	for j in range(0, num_tests):
		for jj in range(0, num_images):
			uprime[j, jj] = ((Ufilt[j][jj]-umean[j]))
			vprime[j, jj] = ((Vfilt[j][jj]-vmean[j]))
			uvprime[j, jj] = uprime[j, jj]*vprime[j, jj]
		uvprime_mean[j] = np.nanmean(uvprime[j], axis=0)
		urms[j] = np.nanmean(uprime[j]**2, axis=0)**(1/2)
		vrms[j] = np.nanmean(vprime[j]**2, axis=0)**(1/2)

	## wall position adjustment ###########
	#convert to m and take off wall position as seen in images
	x = (x)/1000
	y = (y-walloffset)/1000
	xmask = x[ (mask[2]+side_error):(mask[3]-side_error) ]
	ymask = y[ mask[0]:mask[1] ]
	## Create Mean Profiles for each data set#######
	for j in range(0, num_tests):
	    umean_profile[j] = np.mean(umean[j], axis=1)
	    vmean_profile[j] = np.mean(vmean[j], axis=1)
	    urms_profile[j] = np.mean(urms[j], axis=1)
	    vrms_profile[j] = np.mean(vrms[j], axis=1)
	    uvprime_profile[j] = np.mean(uvprime_mean[j], axis=1)

	## Average multiple profiles together
	#use this if multiple tests are performed at the same condition
	umean_profile_avg = np.zeros(len(umean_profile[0]))
	vmean_profile_avg = np.zeros(len(umean_profile[0]))
	urms_profile_avg = np.zeros(len(umean_profile[0]))
	vrms_profile_avg = np.zeros(len(umean_profile[0]))
	uvprime_profile_avg = np.zeros(len(umean_profile[0]))
	#average datasets together
	for j in range(0, num_tests):
	    umean_profile_avg = umean_profile_avg + umean_profile[j]
	    vmean_profile_avg = vmean_profile_avg + vmean_profile[j]
	    urms_profile_avg = urms_profile_avg + urms_profile[j]
	    vrms_profile_avg = vrms_profile_avg + vrms_profile[j]
	    uvprime_profile_avg = uvprime_profile_avg + uvprime_profile[j]
	#divide profiles by number of tests which were combined
	umean_profile_avg = umean_profile_avg / num_tests
	vmean_profile_avg = vmean_profile_avg / num_tests
	urms_profile_avg = urms_profile_avg / num_tests
	vrms_profile_avg = vrms_profile_avg / num_tests
	uvprime_profile_avg = uvprime_profile_avg / num_tests

	##calculate conf interval
	conf = dict()
	Neff = 75
	conf['u'] =  (np.nanmean(np.nanmean(np.nanvar(Umask, axis=1), axis=0), axis=1))**(1/2) * (1/Neff)**(1/2)
	conf['v'] =  (np.nanmean(np.nanmean(np.nanvar(Vmask, axis=1), axis=0), axis=1))**(1/2) * (1/Neff)**(1/2)
	conf['urms'] =  (np.nanmean(np.nanvar(urms, axis=0), axis=1))**(1/2) * (1/(2*Neff-1))**(1/2)
	conf['vrms'] =  (np.nanmean(np.nanvar(vrms, axis=0), axis=1))**(1/2) * (1/(2*Neff-1))**(1/2)
	sigma_u = (np.nanmean(np.nanvar(Umask, axis=1), axis=0))**(1/2)
	sigma_v = (np.nanmean(np.nanvar(Vmask, axis=1), axis=0))**(1/2)
	conf['uvprime'] = np.nanmean(sigma_u * sigma_v * (1+ (np.nanmean(uvprime_mean, axis=0)/(sigma_u * sigma_v))**2 / (Neff - 1))**(1/2), axis=1)

	###  WRITE OUT DATA
	####################
	#open hdf5 file
	hdf = pd.HDFStore('data/PIV_' + date + '.h5')
	hdf.put('umean', pd.DataFrame(umean[0]))
	hdf.put('vmean', pd.DataFrame(vmean[0]))
	hdf.put('umean_profile_avg', pd.DataFrame(umean_profile_avg))
	hdf.put('vmean_profile_avg', pd.DataFrame(vmean_profile_avg))
	hdf.put('urms_profile_avg', pd.DataFrame(urms_profile_avg))
	hdf.put('vrms_profile_avg', pd.DataFrame(vrms_profile_avg))
	hdf.put('uvprime_profile_avg', pd.DataFrame(uvprime_profile_avg))
	hdf.put('confidence', pd.DataFrame(conf))
	hdf.put('xaxis', pd.Series(xmask))
	hdf.put('yaxis', pd.Series(ymask))
	hdf.put('mask', pd.DataFrame(mask))
	hdf.close()

	print('Data Saved!')
	return(Ufilt, Vfilt, xmask, ymask, bad_im_count)


text:
import os
from os.path import join, exists
import argparse
import pathlib
from enum import Enum

import click
import numpy as np
import pandas as pd

import download_data
import dataframe
import plotter
from model_types import ModelTypes, model_types_map

class DeepAugmentModelTypes(Enum):
    STANDARD = ModelTypes.STANDARD.value
    LP_ADV = ModelTypes.LP_ADV.value
    ROBUST_INTV = ModelTypes.ROBUST_INTV.value
    MORE_DATA = ModelTypes.MORE_DATA.value
    DEEPAUGMENT = ('_nolegend_', 'tab:brown', 150, 0.9, 's')

def get_model_type(df_row):
    if 'deepaugment' in df_row.name.lower():
        return DeepAugmentModelTypes.DEEPAUGMENT
    elif model_types_map[df_row.name] == ModelTypes.MORE_DATA:
        return DeepAugmentModelTypes.MORE_DATA
    elif model_types_map[df_row.name] == ModelTypes.LP_ADV:
        return DeepAugmentModelTypes.LP_ADV
    elif model_types_map[df_row.name] == ModelTypes.ROBUST_INTV:
        return DeepAugmentModelTypes.ROBUST_INTV
    elif model_types_map[df_row.name] == ModelTypes.STANDARD:
        return DeepAugmentModelTypes.STANDARD

def show_in_plot(df_row):
    model_name, model_type = df_row.name.lower(), df_row.model_type
    return 'subsample' not in model_name and df_row['val-on-imagenet-r-classes'] >= 75

def use_for_line_fit(df_row):
    model_name, model_type, in_plot = df_row.name.lower(), df_row.model_type, df_row.show_in_plot
    return 'aws' not in model_name and 'batch64' not in model_name and model_type is DeepAugmentModelTypes.STANDARD and in_plot

@click.command()
@click.option('--x_axis', type=str, default='val-on-imagenet-r-classes')
@click.option('--y_axis', type=str, default='imagenet-r')
@click.option('--transform', type=str, default='logit')
@click.option('--num_bootstrap_samples', type=int, default=1000)
@click.option('--output_dir', type=str, default=str((pathlib.Path(__file__).parent / '../outputs').resolve()))
@click.option('--output_file_dir', type=str, default=str((pathlib.Path(__file__).parent / '../paper/appendix').resolve()))
@click.option('--skip_download', is_flag=True, type=bool)
def generate_xy_plot(x_axis, y_axis, transform, num_bootstrap_samples, output_dir, output_file_dir, skip_download):

    if skip_download:
        filename = join(output_dir, 'grid_df.pkl')
        if not exists(filename):
            raise Exception(f'Downloaded data not found at {filename}. Please run python src/plotting/download_data.py first')
        df = pd.read_pickle(filename)
    else:
        df = download_data.download_plotting_data(output_dir, store_data=True, verbose=True)

    df, df_metadata = dataframe.extract_metadata(df)
    df, df_metadata = dataframe.replace_10percent_with_metadata(df, df_metadata)
    df, df_metadata = dataframe.aggregate_corruptions_with_metadata(df, df_metadata)

    df = prepare_df_for_plotting(df, df_metadata, [x_axis, y_axis])
    df = plotter.add_plotting_data(df, [x_axis, y_axis])

    # auto set xlim and ylim based on visible points
    df_visible = df[df.show_in_plot == True]
    xlim = [df_visible[x_axis].min() - 1, df_visible[x_axis].max() + 0.5]
    ylim = [df_visible[y_axis].min() - 1, df_visible[y_axis].values.max() + 1]

    fig, _ = plotter.model_scatter_plot(df, x_axis, y_axis, xlim, ylim, DeepAugmentModelTypes,
                                         transform=transform, tick_multiplier=5, num_bootstrap_samples=num_bootstrap_samples,
                                         title='Distribution Shift to ImageNet-R', x_label='ImageNet (class-subsampled)', y_label='ImageNet-R',
                                         figsize=(12, 8), include_legend=False, return_separate_legend=False)

    os.makedirs(output_file_dir, exist_ok=True)

    fig.savefig(join(output_file_dir, f'imagenet_r.pdf'), dpi='figure', bbox_inches='tight', pad_inches=0.1)
    print(f"Plot saved to {join(output_file_dir, f'imagenet_r.pdf')}")

def prepare_df_for_plotting(df, df_metadata, columns):
    assert set(columns).issubset(set(df.columns))

    df = df[columns]
    df_metadata = df_metadata[[x+'_dataset_size' for x in columns]]
    df = df.merge(df_metadata, right_index=True, left_index=True)
    df = df.dropna()

    df['model_type'] = df.apply(get_model_type, axis=1)
    df['show_in_plot'] = df.apply(show_in_plot, axis=1)
    df['use_for_line_fit'] = df.apply(use_for_line_fit, axis=1)

    return df

if __name__ == '__main__':
    generate_xy_plot()

max_stars_repo_name: saarimrahman/imagenet-testbed

max_issues_repo_name: saarimrahman/imagenet-testbed

max_forks_repo_name: saarimrahman/imagenet-testbed


text:
# -*- coding: utf-8 -*-
# @todo Eingabe einlesen > und ausgeben
# @todo Plot starten mit Knopf und Sinuskurve plotten

#Für die Sinus-und Cosinus Kurve
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.animation as animation

#Für das GUI Programm
import Tkinter
import tkMessageBox
from Tkinter import Entry
from Tkinter import RIGHT
from Tkinter import LEFT
from Tkinter import Label
top = Tkinter.Tk()#Erzeugung eines Fensters

top.title("GUI Programm")
top.geometry("500x600")

def Hallo(): # Funktion 1
   tkMessageBox.showinfo( "Ich hab es Geschafft!", "Was hast du Geschafft??")
   tkMessageBox.geometry("300x400")

B = Tkinter.Button(top, text ="Klick mich!", command = Hallo)
B.pack()

def Prank(): # Funktion 2
   tkMessageBox.showinfo( "Dich zu", "PRANKEN!!!")
   tkMessageBox.geometry("500x600")

C = Tkinter.Button(top, text ="Kilck mich um es zu erfahren!", command = Prank)
C.pack()

# Funtktion 4
LA = Label(top, text="Wie fühlst du dich?\nHier eingeben!")
LA.pack(side = LEFT)
eingabe = Entry(top, bd =5)
eingabe.pack(side = RIGHT)

def Lesen():

    tkMessageBox.showinfo( "Du fühlst dich", (eingabe.get()))
    tkMessageBox.geometry("1000x1000")

knopf3 = Tkinter.Button(top, text ="Hier drücken zum bestätigen!", command = Lesen)
knopf3.pack()

LB = Label(top, text="Bitte beachten die Legende\nsowie Beschriftung\nsteht im Python")
LB.pack()
#Sinus Kurve
def Sinus():
  print ('~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~')
  print ('~~~~~~~~~~~~~~~~~~~~~~~Legende~~~~~~~~~~~~~~~~~~~~~~~~~~~~')
  print ('~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~')
  print
  print
  print ('~~~~~~~~~~Sinus hat die Farbe~~~~~GRÜN~~~~~~~~~~~~~~~~~~~~')
  print
  print ('~~~~~~~~~~Cosinus hat die Farbe~~~~ROT~~~~~~~~~~~~~~~~~~~~')
  print
  print
  print ('~~~~~~~~~X-Achse ist Waagrecht~~~~~~~~~~~~~~~~~~~~~~~~~~~~')
  print
  print ('~~~~~~~~~Y-Achse ist Senkrecht~~~~~~~~~~~~~~~~~~~~~~~~~~~~')

  fig, ax = plt.subplots()
  #line, = ax.plot(np.random.rand(10))
  ax.set_ylim(-1, 1)
  ax.set_xlim(0,2*np.pi)

  def f(x, a=1, b=0):
    return a*np.sin(x) + b*np.cos(x)
  x = np.linspace(0, 2 * np.pi, 500)
  y = f(x)
  plt.plot(x, y, 'gx', x, f(x, 1, 0), 'g')
  plt.plot(x, y, 'gx', x, f(x, 0, 1), 'r')

  plt.show()

knopf4 = Tkinter.Button(top, text ="Hier öffnet man die Sinus Kurve.",command = Sinus)
knopf4.pack()

top.mainloop()


text:
# 핵심: a 사용자 함수 안 b사용자 함수 사용하는 방법 : y=AND(xs[0],xs[1])

#cf. DL9-1 XOR
#and 게이트-tensorflow 사용x numpy사용.
import numpy as np
def AND(x1,x2): #00 01 10 11 => 0 0 0 1
    x=np.array([x1,x2]) #입력 데이터 2개
    w=np.array([0.5,0.5]) #w 2개
    b=-0.7
    tmp=np.sum(w*x)+b
    if tmp<=0:
        return 0
    else:
        return 1
#명시적 프로그램 작성하면 아래(but 무수히 많은 경우 적용 한계. 따라서 위 처럼 적용)
# def AND(x1, x2):
    # if x1==1 and x2 ==1:
    #     return 1
    # else:
    #     return

#nand 게이트
def NAND(x1, x2):  # 00 01 10 11 => 1 1 1 0
    x = np.array([x1, x2])  # 입력 데이터 2개
    w = np.array([-0.5, -0.5])  # w 2개
    b = 0.7
    tmp = np.sum(w * x) + b
    if tmp <= 0:
        return 0
    else:
        return 1

    # return # 0 or 1

#or 게이트
def OR(x1, x2):  # 00 01 10 11 => 0 1 1 1
    x = np.array([x1, x2])  # 입력 데이터 2개
    w = np.array([0.5, 0.5])  # w 2개
    b = -0.2
    tmp = np.sum(w * x) + b
    if tmp <= 0:
        return 0
    else:
        return 1

#xor 게이트
def XOR(x1, x2):  # 00 01 10 11 => 0 1 1 0
    s1=NAND(x1, x2)
    s2=OR(x1,x2)
    yhat=AND(s1,s2)
    return yhat

if __name__ == '__main__':
    print("and게이트:")
    for xs in [(0,0),(0,1),(1,0),(1,1)]:
        y=AND(xs[0],xs[1]) #and 게이트
        print(str(xs)+"->"+str(y))

    print("nand게이트:")
    for xs in [(0,0),(0,1),(1,0),(1,1)]:
        y=NAND(xs[0],xs[1]) #nand 게이트
        print(str(xs)+"->"+str(y))

    print("or게이트:")
    for xs in [(0,0),(0,1),(1,0),(1,1)]:
        y=OR(xs[0],xs[1]) #or 게이트
        print(str(xs)+"->"+str(y))

    print("xor게이트:")
    for xs in [(0,0),(0,1),(1,0),(1,1)]:
        y=XOR(xs[0],xs[1]) #xor 게이트
        print(str(xs)+"->"+str(y))

max_stars_repo_path: Fuction in Fuction.py

max_stars_repo_name: Byunggun/Appris

max_issues_repo_path: Fuction in Fuction.py

max_issues_repo_name: Byunggun/Appris

max_forks_repo_path: Fuction in Fuction.py

max_forks_repo_name: Byunggun/Appris


text:
#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
Created on Fri Mar  9 11:06:16 2018

Test for the function of the chi2 script of the omnitool package.

@author: misiak
"""

import sys

sys.path.append('/home/misiak/projects/package_red_magic')

import numpy as np
import matplotlib.pyplot as plt

from red_magic import chi2_simple, chi2_freq, opt_chi2_amp, psd

fs = 1e3
t_range = np.arange(0, 1, fs**-1)

# FUNCTION
funk = lambda a: a*t_range

#DATA
sig = 0.02
data = funk(.5) + np.random.normal(0, sig, t_range.shape)
# MODEL
xmod = (.4,.5,.6)
labmod = ('mod1', 'mod2', 'mod3')
darray = {l: funk(a) for l,a in zip(labmod, xmod)}

# TEMPORAL Chi2
d_sx2 = {l:  chi2_simple(data, darray[l], err=sig) for l in labmod}

# FREQ Chi2 with fft, psd, etc...
dfft = {l: np.fft.fft(darray[l]) for l in labmod}

fftdata = np.fft.fft(data)
freq, dpsd =  psd(fftdata, fs)

noise_list = list()
for k in range(100):
    freq, noi =  psd(np.fft.fft(np.random.normal(0, sig, t_range.shape)), fs)
    noise_list.append(noi)

npsd = np.mean(noise_list, axis=0)

d_fx2 = {l:  chi2_freq(fftdata, dfft[l], npsd, fs) for l in labmod}

########## PLOT #############
plt.close('all')

### TEMPORAL PLOT
plt.figure()
plt.title('1000 pts _ Temporal Chi2')
plt.plot(
    t_range, data, lw=1.,
    label='data, $\chi^2=${:.2f}'.format( chi2_simple(data, data, err=sig))
)

for l in labmod:
    plt.plot(t_range, darray[l], label=l + ' $\chi^2=${:.2f}'.format(d_sx2[l]))
plt.grid(b=True)
plt.legend()

# FREQUENCY PLOT
plt.figure()
plt.title('500 freqs _ Frequency Chi2')
plt.grid(b=True)
plt.loglog(
    freq, dpsd,
    label='data $\chi^2=${:.2f}'.format( chi2_freq(fftdata, fftdata, npsd, fs))
)
for l in labmod:
    freq, PSD =  psd(dfft[l], fs)
    plt.loglog(freq, PSD, label=l+' $\chi^2=${:.2f}'.format(d_fx2[l]))
plt.loglog(freq, npsd, label='noise')

plt.show()
plt.legend()


text:
#!/usr/bin/env python
# coding: utf-8

#### Citations
#
# https://github.com/tensorflow/models/tree/master/research/object_detection
# "Speed/accuracy trade-offs for modern convolutional object detectors."
# Huang J, Rathod V, Sun C, Zhu M, Korattikara A, Fathi A, Fischer I, Wojna Z,
# Song Y, Guadarrama S, Murphy K, CVPR 2017
#
# https://github.com/rafaelpadilla/Object-Detection-Metrics
# @INPROCEEDINGS {padillaCITE2020,
#     author    = {R. {Padilla} and S. L. {Netto} and E. A. B. {da Silva}},
#     title     = {A Survey on Performance Metrics for Object-Detection Algorithms},
#     booktitle = {2020 International Conference on Systems, Signals and Image Processing (IWSSIP)},
#     year      = {2020},
#     pages     = {237-242},}
#

# ## Loading in TFRecords from GCS bucket
import os
from os import path as op
import tensorflow as tf  # version 2
import tensorflow.compat.v1 as tf1
from object_detection.utils import dataset_util
from object_detection.utils import visualization_utils as viz_utils
import numpy as np
import pandas as pd
from PIL import Image, ImageDraw
import glob
import matplotlib.pyplot as plt
import click
import seaborn as sn

# Just disables the warning, doesn't enable AVX/FMA
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
print("Tensorflow version " + tf.__version__)

def _parse_image_function(example_proto):
    """return parse features"""
    image_feature_description = {
        "image/height": tf.io.FixedLenFeature([], tf.int64),
        "image/width": tf.io.FixedLenFeature([], tf.int64),
        "image/filename": tf.io.FixedLenFeature([], tf.string),
        "image/source_id": tf.io.FixedLenFeature([], tf.string),
        "image/encoded": tf.io.FixedLenFeature([], tf.string),
        "image/format": tf.io.FixedLenFeature([], tf.string),
        "image/object/bbox/xmin": tf.io.FixedLenSequenceFeature(
            [], tf.float32, allow_missing=True, default_value=None
        ),
        "image/object/bbox/xmax": tf.io.FixedLenSequenceFeature(
            [], tf.float32, allow_missing=True, default_value=None
        ),
        "image/object/bbox/ymin": tf.io.FixedLenSequenceFeature(
            [], tf.float32, allow_missing=True, default_value=None
        ),
        "image/object/bbox/ymax": tf.io.FixedLenSequenceFeature(
            [], tf.float32, allow_missing=True, default_value=None
        ),
        "image/object/class/text": tf.io.FixedLenSequenceFeature(
            [], tf.string, allow_missing=True, default_value=None
        ),
        "image/object/class/label": tf.io.FixedLenSequenceFeature(
            [], tf.int64, allow_missing=True, default_value=None
        ),
    }
    return tf.io.parse_single_example(example_proto, image_feature_description)

def get_example(tfrecords_path):
    """decode tfrecord file, returns items"""
    dataset = tf.data.TFRecordDataset([tfrecords_path])
    parsed_image_dataset = dataset.map(_parse_image_function)
    items = []
    for image_features in parsed_image_dataset:
        image_id = image_features["image/filename"].numpy()
        image_raw = image_features["image/encoded"].numpy()
        img = tf.image.decode_image(image_raw)
        label = image_features["image/object/class/label"].numpy()
        cls_name = image_features["image/object/class/text"].numpy()
        ymin = image_features["image/object/bbox/ymin"].numpy()
        xmin = image_features["image/object/bbox/xmin"].numpy()
        ymax = image_features["image/object/bbox/ymax"].numpy()
        xmax = image_features["image/object/bbox/xmax"].numpy()
        bboxes = [
            [xmin[i], ymin[i], xmax[i], ymax[i]] for i in range(len(label))
        ]
        labels = [label[i] for i in range(len(label))]
        cls_names = [cls_name[i] for i in range(len(cls_name))]
        items.append(
            {
                "image_id": image_id,
                "img_arr": img,
                "labels": labels,
                "class_name": cls_names,
                "bboxes": bboxes,
            }
        )
    return items

def get_example_no_img(tfrecords_path):
    """decode tfrecord file, returns items"""
    dataset = tf.data.TFRecordDataset([tfrecords_path])
    parsed_image_dataset = dataset.map(_parse_image_function)
    items = []
    for image_features in parsed_image_dataset:
        image_id = image_features["image/filename"].numpy()
        label = image_features["image/object/class/label"].numpy()
        cls_name = image_features["image/object/class/text"].numpy()
        ymin = image_features["image/object/bbox/ymin"].numpy()
        xmin = image_features["image/object/bbox/xmin"].numpy()
        ymax = image_features["image/object/bbox/ymax"].numpy()
        xmax = image_features["image/object/bbox/xmax"].numpy()
        bboxes = [
            [xmin[i], ymin[i], xmax[i], ymax[i]] for i in range(len(label))
        ]
        labels = [label[i] for i in range(len(label))]
        cls_names = [cls_name[i] for i in range(len(cls_name))]
        items.append(
            {
                "image_id": image_id,
                "labels": labels,
                "class_name": cls_names,
                "bboxes": bboxes,
            }
        )
    return items

def tf1_od_pred(test_img_dict_lst, detection_graph):
    """
    Runs inference with a frozen graph on images and returns list of dicts, with image_id.
    Args:
        test_img_dict_lst (list): list of dicts containing the image and image gt metadata, read from the tfrecord by get_example().
        detection_graph (str): the loaded frozen graph
    Returns:
        det_dicts (list): A list of dictionaries containing detection data, including bounding boxes and class scores
    """

    det_dicts = (
        []
    )  # contains img name as key and detection info as value (boxes, scores, classes, num)
    with detection_graph.as_default():
        with tf1.Session(graph=detection_graph) as sess:
            # Definite input and output Tensors for detection_graph
            image_tensor = detection_graph.get_tensor_by_name("image_tensor:0")
            # Each box represents a part of the image where a particular object was detected.
            detection_boxes = detection_graph.get_tensor_by_name(
                "detection_boxes:0"
            )
            # Each score represent how level of confidence for each of the objects.
            # Score is shown on the result image, together with the class label.
            detection_scores = detection_graph.get_tensor_by_name(
                "detection_scores:0"
            )
            detection_classes = detection_graph.get_tensor_by_name(
                "detection_classes:0"
            )
            num_detections = detection_graph.get_tensor_by_name(
                "num_detections:0"
            )
            for item in test_img_dict_lst:
                # the array based representation of the image will be used later in order to prepare the
                # result image with boxes and labels on it.
                # Expand dimensions since the model expects images to have shape: [1, None, None, 3]
                image_np_expanded = np.expand_dims(item["img_arr"], axis=0)
                # Actual detection.
                (boxes, scores, classes, num) = sess.run(
                    [
                        detection_boxes,
                        detection_scores,
                        detection_classes,
                        num_detections,
                    ],
                    feed_dict={image_tensor: image_np_expanded},
                )

                det_dicts.append(
                    {
                        "image_id": item["image_id"],
                        "boxes": boxes,
                        "scores": scores,
                        "classes": classes,
                        "num": num,
                    }
                )
        return det_dicts

def filter_detections(det, conf_threshold):
    # remove artifact of no detection
    if det["num"][0] == 0.0:
        det["scores"] = []
        det["boxes"] = []
        det["classes"] = []
    # remove low scores
    high_score_mask = np.array(det["scores"]) > conf_threshold
    det["scores"] = np.array(det["scores"])[high_score_mask]
    det["boxes"] = np.array(det["boxes"])[high_score_mask]
    det["classes"] = np.array(det["classes"])[high_score_mask]
    return det

def filter_detections_and_concat_boxes(all_gt, det_dicts, conf_threshold):
    """
    Filters detections and groundtruth by a confidence score threshold and removes empty detections.
    Args:
        all_gt (list): all groundtruth dictionaries
        det_dicts (list): all detection dictionaries
        conf_threshold (float): all values below this will not be included as valid detections that are used to generate metrics
    Returns:
        A filtered lists of dictionaries, as well as lists of bounding boxes and classes extracted from these dictionaries

    """
    all_det_scores = []
    all_det_img_names = []
    all_det_boxes = []
    all_gt_img_names = []
    all_gt_boxes = []
    all_gt_classes = []
    all_det_classes = []
    for gt, det in zip(all_gt, det_dicts):
        det = filter_detections(det, conf_threshold)
        if len(det["boxes"]) > 0:
            all_det_boxes.append(det["boxes"])
            all_det_classes.extend(det["classes"])
            all_det_scores.extend(det["scores"])
            all_det_img_names.append(det["image_id"])
        all_gt_boxes.append(np.array(gt["bboxes"]))
        all_gt_classes.extend(gt["labels"])
        all_gt_img_names.append(gt["image_id"])
    all_gt_boxes = np.concatenate(all_gt_boxes)
    all_det_boxes = np.concatenate(all_det_boxes)
    all_gt_classes = np.array(all_gt_classes).astype(int)
    all_det_classes = np.array(all_det_classes).astype(int)
    all_det_scores = np.array(all_det_scores).astype(float)
    return (
        all_gt,
        det_dicts,
        all_gt_boxes,
        all_det_boxes,
        all_gt_classes,
        all_det_classes,
        all_det_scores,
        all_det_img_names,
        all_gt_img_names,
    )

def compute_iou(groundtruth_box, detection_box):
    """
    compute IOU score by compare ground truth bbox and predicted bbox
    Args:
        groundtruth_box: ground truth bbox in [x0, y0, x1, y1]
        detection_box: predicted truth bbox in [y0, x0, y1, x1]
    Returns:
        iou: IOU score between ground truth and detected bboxes
    """
    g_xmin, g_ymin, g_xmax, g_ymax = tuple(groundtruth_box)
    d_ymin, d_xmin, d_ymax, d_xmax = tuple(detection_box)

    x_left = max(g_xmin, d_xmin)
    y_top = max(g_ymin, d_ymin)
    x_right = min(g_xmax, d_xmax)
    y_bottom = min(g_ymax, d_ymax)

    boxGArea = (g_xmax - g_xmin + 1) * (g_ymax - g_ymin + 1)

    if x_right < x_left or y_bottom < y_top:
        return 0, boxGArea

    intersection = (x_right - x_left + 1) * (y_bottom - y_top + 1)

    boxDArea = (d_xmax - d_xmin + 1) * (d_ymax - d_ymin + 1)
    iou = intersection / float(boxGArea + boxDArea - intersection)
    return iou, boxGArea

def get_box_matches(
    groundtruth_boxes, detection_boxes, detection_scores, iou_threshold
):
    """
    Returns pred list and gt list indices for the box matches, the iou,
    and the groundtruth box area to examine size effect on accuracy.

    Args:
        groundtruth_boxes: list of ground truth bbox in [x0, y0, x1, y1]
        detection_boxes: list of predicted truth bbox in [y0, x0, y1, x1]
        IOU_THRESHOLD: threshold used to consider a detection a valid overlap and possible true positive
    Returns:
        iou: IOU score between ground truth and detected bboxes
    """

    matches = []

    for i in range(len(groundtruth_boxes)):
        for j in range(len(detection_boxes)):
            iou, gt_box_area = compute_iou(
                groundtruth_boxes[i], detection_boxes[j]
            )

            if iou > iou_threshold:
                matches.append([i, j, iou, detection_scores[j], gt_box_area])

    matches = np.array(matches)
    if matches.shape[0] > 0:
        # Sort list of matches by descending IOU so we can remove duplicate detections
        # while keeping the highest IOU entry.
        matches = matches[matches[:, 2].argsort()[::-1][: len(matches)]]

        # Remove duplicate detections from the list.
        matches = matches[np.unique(matches[:, 1], return_index=True)[1]]

        # Sort the list again by descending confidence score first, then IOU.
        # This deals with cases where a gt detection has multiple high IOU
        # detections of different classes, with one correct, higher confidence
        # detection. Removing duplicates doesn't preserve our previous sort.

        matches = matches[np.lexsort((matches[:, 2], matches[:, 3]))[::-1]]
        # Remove duplicate ground truths from the list.
        matches = matches[np.unique(matches[:, 0], return_index=True)[1]]

    return matches

def compute_confusion_matrix(matches, gt_classes, det_classes, num_categories):
    """Computes a confusion matrix to count true positives, false negatives, false positives.

    This iterates over the groundtruth and detection indices representing
    bounding boxes and their classes. Matches are identified when IOU is
    above the threshold.Each match in `matches` will count as a true positive
    or misidentified positive in a confusion matrix cell. If indices are not
    in `matches` they are then counted as either false negative or false positive.

    Worth noting, many detections ascribed as false positives actually have
    substantial overlap with a groundtruth detection, but not as much overlap as the
    matching true positive. In order to not double count objects, these extra
    detections need to be set as false positives.

    Args:
        matches (list): [description]
        gt_classes (list): [description]
        det_classes (list): [description]
        num_categories (int, optional): The number of categories.

    Returns:
        [type]: [description]
    """

    gt_classes_sorted = []
    det_classes_sorted = []
    confusion_matrix = np.zeros(shape=(num_categories + 1, num_categories + 1))
    for i in range(len(gt_classes)):
        if matches.shape[0] > 0 and matches[matches[:, 0] == i].shape[0] == 1:
            # match identified, count a true positive or a misidentification
            detection_class_cm_i = int(
                det_classes[int(matches[matches[:, 0] == i, 1][0])] - 1
            )  # index along cm axis and the class label associated with the match
            groundtruth_class_cm_i = int(gt_classes[i] - 1)
        else:
            # detection is background but groundtruth is another class, false negative
            detection_class_cm_i = int(confusion_matrix.shape[1] - 1)
            groundtruth_class_cm_i = int(gt_classes[i] - 1)

        gt_classes_sorted.append(groundtruth_class_cm_i)
        det_classes_sorted.append(detection_class_cm_i)
        confusion_matrix[groundtruth_class_cm_i][detection_class_cm_i] += 1

    for i in range(len(det_classes)):
        # catches case where some detections have matches and when there are no matches but there are gt
        if (
            matches.shape[0] > 0 and matches[matches[:, 1] == i].shape[0] == 0
        ) or matches.size == 0:
            detection_class_cm_i = int(det_classes[i] - 1)
            groundtruth_class_cm_i = int(confusion_matrix.shape[0] - 1)

            gt_classes_sorted.append(groundtruth_class_cm_i)
            det_classes_sorted.append(detection_class_cm_i)
            confusion_matrix[groundtruth_class_cm_i][detection_class_cm_i] += 1

    assert len(gt_classes_sorted) == len(det_classes_sorted)

    return confusion_matrix, gt_classes_sorted, det_classes_sorted

def plot_cm_sn(
    cm,
    names,
    outpath,
    title,
    iou_threshold,
    conf_threshold,
    normalize,
    norm_axis="predicted",
    cmap="Blues",
):
    num_classes = len(names)
    cm_copy = cm.copy()
    if normalize:
        if norm_axis == "predicted":
            cm_copy = cm / (cm.sum(0).reshape(1, num_classes + 1) + 1e-6)
        elif norm_axis == "true":
            cm_copy = cm / (cm.sum(1).reshape(num_classes + 1, 1) + 1e-6)
    cm_copy[cm_copy < 0.005] = np.nan  # don't annotate (would appear as 0.00)
    # reorder confusion matrix by diagonal
    diag = np.diag(cm_copy)  # get diagonal
    idx = np.argsort(
        diag
    )  # get all indicies of diag ordered, but true values sorted wrong
    idx_nonnan = np.argwhere(~np.isnan(diag))  # get nonnan indices
    nonnanlen = len(idx_nonnan)
    idx[0:nonnanlen] = np.concatenate(
        idx_nonnan, axis=0
    )  # replace non nan indices with sorted
    yticklabels = np.array(names + ["False Negative, \nonly groundtruth"])
    xticklabels = np.array(names + ["False Positive, \nonly detection"])
    cm_copy = cm_copy[idx, :][:, idx]
    yticklabels = yticklabels[idx]
    xticklabels = xticklabels[idx]
    # plot
    fig = plt.figure(figsize=(12, 9))
    sn.set(font_scale=1.0 if num_classes < 50 else 0.8)  # for label size
    labels = (0 < len(names) < 99) and len(
        names
    ) == num_classes  # apply names to ticklabels
    sn.heatmap(
        cm_copy,
        annot=num_classes < 30,
        annot_kws={"size": 8},
        cmap=cmap,
        fmt=".2f",
        square=True,
        yticklabels=list(yticklabels)
        if labels
        else "auto",  # switched ticklabels
        xticklabels=list(xticklabels) if labels else "auto",
    ).set_facecolor((1, 1, 1))
    fig.axes[0].set_xlabel("True", fontsize=16)
    fig.axes[0].set_ylabel(
        "Predicted",
        fontsize=16,
    )
    fig.axes[0].set_title(
        title
        + f"\n {iou_threshold} IOU and  {conf_threshold} Confidence Score Thresholds",
        fontsize=19,
    )
    fig.tight_layout()
    fig.savefig(outpath, dpi=300)
    plt.yticks(rotation=0)
    plt.close()

def cm_from_img_tuple(img_detection_tuple, class_names, IOU_thresh=0.5):
    """Computes confusion matrix from a single sample of predictions and groundtruth.

    Args:
        img_detection_tuple (tuple): Tuple with two dictionaries, one for
            predictions and one for groundtruth.
        IOU_thresh (float):  The intersection over union threshold used to
            determine a match.

    Returns:
        np.array: The confusion matrix in numpy array format.
    """
    img_gt_dict = img_detection_tuple[0]
    det_dict = img_detection_tuple[1]
    assert det_dict["image_id"] == img_gt_dict["image_id"]
    gt_bboxes = img_gt_dict["bboxes"]
    gt_labels = img_gt_dict["labels"]
    pred_scores = det_dict["scores"]
    pred_bboxes = det_dict["boxes"]
    pred_labels = det_dict["classes"].astype(int)

    # # Getting image specific counts of TP, FP, FN
    box_matches = get_box_matches(
        gt_bboxes, pred_bboxes, pred_scores, IOU_thresh
    )
    # y_true, y_pred for scikit_learn format to compute metrics
    cm, y_true, y_pred = compute_confusion_matrix(
        box_matches, gt_labels, pred_labels, len(class_names)
    )

    return cm

def save_image(
    img_detection_tuple,
    outdir,
    class_names,
    iou_threshold,
    conf_threshold,
    cm=None,
):
    """Saves images with bounding boxes for prediction (blue) and groundtruth (red).

    Args:
        img_detection_tuple (tuple): Tuple with two dictionaries, one for predictions
            and one for groundtruth.
        outdir (str): where to save the images
        class_names (list): list of class names. must be supplied in order that matches
            both detection and groundtruth numerical class IDs
        cm (np.array, optional): The confusion matrix corresponding to img_detection_tuple.
            Defaults to None.
    """

    img_gt_dict = img_detection_tuple[0]
    det_dict = img_detection_tuple[1]
    assert det_dict["image_id"] == img_gt_dict["image_id"]
    image_np = img_gt_dict["img_arr"].numpy()
    image_id = str(img_gt_dict["image_id"].decode("utf-8")).strip("''")
    gt_bboxes = img_gt_dict["bboxes"]
    gt_class_name = img_gt_dict["class_name"]

    pred_bboxes = det_dict["boxes"]
    pred_labels = det_dict["classes"]
    pred_scores = det_dict["scores"]

    if cm is not None:
        cmname = image_id[:-4] + "_cm" + ".png"
        plot_cm_sn(
            cm,
            class_names,
            outpath=op.join(outdir, cmname),
            title=f"{image_id[:-4]} Only Confusion Matrix",
            normalize=False,
            iou_threshold=iou_threshold,
            conf_threshold=conf_threshold,
        )

    ############################
    # Draw bbox
    ############################
    img = Image.fromarray(image_np).convert("RGB")
    draw = ImageDraw.Draw(img)
    image_path = op.join(outdir, str(image_id))
    for i, gtbbox in enumerate(gt_bboxes):
        gtbbox = [gtbbox[i] * 400 for i in range(len(gtbbox))]
        xmin, ymin, xmax, ymax = gtbbox
        draw.rectangle(gtbbox, outline="#ff0000")
        x_label = xmin + (xmax - xmin) / 2
        draw.text(
            (x_label - 15, ymax),
            text=str(gt_class_name[i].decode("utf-8")),
            fill="red",
            align="right",
        )
    for i, pred_bbox in enumerate(pred_bboxes):
        pred_bbox = [pred_bbox[i] * 400 for i in range(len(pred_bbox))]
        ymin, xmin, ymax, xmax = pred_bbox
        pred_bbox = [xmin, ymin, xmax, ymax]
        draw.rectangle(pred_bbox, outline="#0000ff")
        x_label = xmin + (xmax - xmin) / 2
        class_i = int(pred_labels[i])
        draw.text(
            (x_label - 15, ymax),
            text=f"{class_names[class_i-1]} {np.round(pred_scores[i],decimals=3).astype('|S4').decode('utf-8')}",
            fill="blue",
            align="right",
        )

    img.save(image_path, "JPEG")
    return cmname

def save_metrics(confusion_matrix, categories, output_path, iou_threshold):
    """
    Write a cvs that saves recall, precision, f1 and map score
    Args:
        confusion_matrix: computed confusion matrix for classes
        categories: classes in list;
        output_path: output path for the csv
    Returns:
        (None): saved csv
    """
    print("\nConfusion Matrix:")
    print(confusion_matrix, "\n")
    print(f"Confusion Matrix Shape: {confusion_matrix.shape}")

    labels = [categories[i] for i in range(len(categories))]
    results = []

    for i in range(len(categories)):
        name = categories[i]
        total_target = np.sum(confusion_matrix[:, i])
        total_predicted = np.sum(confusion_matrix[i, :])
        if total_target == 0:
            recall = 0
        else:
            recall = float(confusion_matrix[i, i] / total_target)
            # print(f'recalls are {recall}')
        if total_predicted == 0:
            precision = 0
        else:
            precision = float(confusion_matrix[i, i] / total_predicted)
        try:
            f1 = 2 * precision * recall / (precision + recall)
        except ZeroDivisionError:
            f1 = 0
        prec_at_rec = []
        for recall_level in np.linspace(0.0, 1.0, 11):

            if recall >= recall_level:
                prec_at_rec.append(precision)
        prec_at_rec = np.array(prec_at_rec)
        avg_prec = np.mean(prec_at_rec)

        results.append(
            {
                "category": name,
                f"precision_@{ iou_threshold}IOU": precision,
                f"recall_@{ iou_threshold}IOU": recall,
                f"map_@{ iou_threshold}IOU": avg_prec,
                f"f1_@{ iou_threshold}IOU": f1,
            }
        )
    df = pd.DataFrame(results)
    df.to_csv(output_path)

@click.command()
@click.option(
    "--tfrecords_folder",
    default="./training_data_aiaia_p400",
    help="The folder containing the subfolders that contain tfrecords.",
)
@click.option(
    "--outdir",
    default="./outputs",
    help="Where to save output images with bounding boxes drawn, metrics, and plots.",
)
@click.option(
    "--model_rel_path",
    default="./frozen_inference_graph.pb",
    help="The path to the folder containing the frozen graph .pb file.",
)
@click.option(
    "--class_names",
    "-cn",
    multiple=True,
    default=[
        "buffalo",
        "dark_coloured_large",
        "elephant",
        "giraffe",
        "hippopotamus",
        "light_coloured_large",
        "smaller_ungulates",
        "warthog",
        "zebra",
    ],
    help="The class names that match the order of the class IDs from the prediction output and groundtruth. Use like -cn buffalo -cn dark_coloured_large -cn elephant etc. Order matters and should match the order defined in the class_map.csv.",
)
@click.option(
    "--model_type",
    default="wildlife",
    type=click.Choice(
        ["human_activities", "wildlife", "livestock"], case_sensitive=True
    ),
    help="The model type used to filter tfrecords in subfolders to run evaluation on different detection problems. These include human_activities, wildlife, and livestock.",
)
@click.option(
    "--iou_threshold",
    default=0.5,
    type=float,
    help="Threshold to set boxes with low overlap as not potential true positives. Defaults to .5",
)
@click.option(
    "--conf_threshold",
    default=0.5,
    type=float,
    help="Threshold to throw away low confidence detections. Defaults to .5",
)
@click.option(
    "--save_individual_images",
    default=True,
    type=bool,
    help="Whether to save individual images with bounding boxes drawn. Useful for debugging and inspecting model results. Defaults to True",
)
def run_eval(
    tfrecords_folder,
    outdir,
    model_rel_path,
    class_names,
    model_type,
    iou_threshold,
    conf_threshold,
    save_individual_images,
):
    """Computes metrics for TFrecords containing test images and groundtruth.

    Also saves out groundtruth and prediction boxes drawn on images.
    A frozen graph model must be supplied for predictions. Computes, plots and
    saves confusion matrix and a csv with metrics, including total f1 score.
    """
    print(f"Starting evaluation for model: {model_type}")
    # Reading all images and groundtruth
    class_names = list(class_names)  # it initially gets parsed as a tuple
    all_gt = []
    all_imgs_and_gt = []
    tfrecord_regex = f"test*{model_type}*.tfrecords"
    # list tfrecord folders
    for tfrecords_subfolder in glob.glob(tfrecords_folder + "/*/"):
        for tfrecord_path in glob.glob(
            op.join(tfrecords_subfolder, tfrecord_regex)
        ):
            sub_dir = op.join(outdir, "")
            if not op.exists(sub_dir):
                os.makedirs(sub_dir)
            items = get_example(tfrecord_path)
            items_no_imgs = get_example_no_img(tfrecord_path)
            all_gt.extend(items_no_imgs)
            all_imgs_and_gt.extend(items)
    print("TFRecords opened")
    # Testing the tf model on subset of test set, all_imgs_and_gt
    model_path = op.join(os.getcwd(), model_rel_path)
    detection_graph = tf1.Graph()
    with detection_graph.as_default():
        od_graph_def = tf1.GraphDef()
        with tf1.gfile.GFile(model_path, "rb") as fid:
            serialized_graph = fid.read()
            od_graph_def.ParseFromString(serialized_graph)
            tf1.import_graph_def(od_graph_def, name="")
    det_dicts = tf1_od_pred(all_imgs_and_gt, detection_graph)
    print("Inference complete")
    (
        all_gt,
        det_dicts_filtered,
        all_gt_boxes,
        all_detection_boxes,
        all_gt_classes,
        all_det_classes,
        all_det_scores,
        all_detection_img_names,
        all_gt_img_names,
    ) = filter_detections_and_concat_boxes(
        all_gt, det_dicts, conf_threshold=conf_threshold
    )
    print("Detections filtered by confidence")
    # save out images with both boxes drawn for sanity check
    if save_individual_images:
        test_imgs_dir = op.join(outdir, "test_imgs_with_boxes")
        if not op.exists(test_imgs_dir):
            os.makedirs(test_imgs_dir)

    all_imgs_gt_det = list(zip(all_imgs_and_gt, det_dicts_filtered))
    cms = []
    for item in all_imgs_gt_det:
        cm = cm_from_img_tuple(item, class_names, iou_threshold)
        cm_copy = cm.copy()
        cm[:, len(class_names)] = cm_copy[len(class_names), :]
        cm[len(class_names), :] = cm_copy[:, len(class_names)]
        cms.append(cm)
        if save_individual_images:
            cmname = save_image(
                item,
                test_imgs_dir,
                class_names,
                iou_threshold,
                conf_threshold,
                cm,
            )
            save_metrics(
                cm,
                class_names,
                op.join(
                    test_imgs_dir,
                    f"{cmname}_{iou_threshold}_{conf_threshold}_metrics.csv",
                ),
                iou_threshold=iou_threshold,
            )
    print("Images saved with bounding boxes drawn.")
    bigcm = np.sum(np.array(cms), axis=0)
    cnames_bground = class_names.copy().append("background")
    pd.DataFrame(bigcm, index=cnames_bground, columns=cnames_bground).to_csv(
        op.join(outdir, f"{model_type}_confusion_matrix.csv")
    )
    plot_title = model_type.replace("_", " ").title()
    print(class_names)
    plot_cm_sn(
        bigcm,
        class_names,
        outpath=op.join(
            outdir,
            f"{model_type}_{iou_threshold}_{conf_threshold}_confusion_matrix_predicted_normed.png",
        ),
        title=f"{plot_title} Confusion Matrix, Proportions",
        iou_threshold=iou_threshold,
        conf_threshold=conf_threshold,
        normalize=True,
        norm_axis="predicted",
    )
    plot_cm_sn(
        bigcm,
        class_names,
        outpath=op.join(
            outdir,
            f"{model_type}_{iou_threshold}_{conf_threshold}_confusion_matrix_true_normed.png",
        ),
        title=f"{plot_title} Confusion Matrix, Proportions",
        iou_threshold=iou_threshold,
        conf_threshold=conf_threshold,
        normalize=True,
        norm_axis="true",
    )
    plot_cm_sn(
        bigcm,
        class_names,
        outpath=op.join(
            outdir,
            f"{model_type}_{iou_threshold}_{conf_threshold}_confusion_matrix_counts.png",
        ),
        title=f"{plot_title} Confusion Matrix, Counts",
        iou_threshold=iou_threshold,
        conf_threshold=conf_threshold,
        normalize=False,
        cmap="Greens",
    )
    save_metrics(
        bigcm,
        class_names,
        op.join(
            outdir,
            f"{model_type}_{iou_threshold}_{conf_threshold}_metrics.csv",
        ),
        iou_threshold=iou_threshold,
    )
    class_str = "\n".join(class_names)
    print(
        f"Complete, Confusion matrix, images, and metrics saved in {outdir} for these classes: {class_str}"
    )

if __name__ == "__main__":
    run_eval()

max_stars_repo_name: TZCRC/AIAIA

max_issues_repo_name: TZCRC/AIAIA

max_forks_repo_name: TZCRC/AIAIA


text:
#!/usr/bin/env python

import os
import sys
import json
import numpy as np
import h5py
import logging
from rhdlib.rhd_reader import RHDFile, signed_arrays

def convert(filenames, outfile):
    """
    Concatenate the contents of the RHD files into contiguous HDF5 datasets

    Parameters
    ----------
    filenames: sequence
        Sequence of RHD paths.
    outfile: Path, str
        Output path for the HDF5 file

    """
    info = logging.getLogger().info
    info('Creating RHD file maps')
    rhd_files = [RHDFile(f) for f in filenames]
    with h5py.File(outfile, 'w') as data:
        first_file = rhd_files[0]
        arrays = first_file.struct_map
        hdf_arrays = dict()
        hdf_offsets = dict()
        for name, shape in arrays:
            if name in signed_arrays:
                dtype = 'h'
            else:
                dtype = first_file.rhd_map[name].dtype
            num_channels = shape[0]
            all_samples = np.sum([rhd.array_sizes[name][1] for rhd in rhd_files])
            shape = (num_channels, all_samples) if num_channels > 1 else (all_samples,)
            info('dataset name {}, dtype {}, shape {}'.format(name, dtype, shape))
            arr = data.create_dataset(name, dtype=dtype, shape=shape, chunks=True)
            hdf_arrays[name] = arr
            hdf_offsets[name] = 0

        for n, rhd in enumerate(rhd_files):
            print('Converting {} ({} of {})'.format(rhd.file_path, n + 1, len(rhd_files)))
            rhd.to_arrays(arrays=hdf_arrays, offsets=hdf_offsets, apply_scales=False)
            info('Offsets: {}'.format(list(hdf_offsets.items())))
        data.attrs['JSON_header'] = json.dumps(first_file.header)
        info('End of conversion')
    print('Done: {}'.format(outfile))

if __name__ == '__main__':
    import argparse
    from glob import glob

    class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescriptionHelpFormatter):
        pass

    app_description = \
        """Combine and convert packetized Intan RHD files into continuous arrays in HDF5 format.

        Arrays in the output file may include:
        * amplifier_data: converted to signed int16, scale to uV by multiplying 0.195
        * aux_input_data: unsigned uint16, scale by 37.4e-6 for Volts (sampled at 1/4 rate as amplifier data)
        * board_adc_data: converted to signed int16, scale by 312.5e-6 for Volts
        * supply_voltage_data: uint16, scale by 74.8e-6 for Volts (sampled once per data block)
        * temp_sensor_data: uint16, scale by 0.01
        * board_dig_in_data: boolean
        * board_dig_out_data: boolean

        To load (Python example):
        >>> import h5py
        >>> f = h5py.File('ecog_256_array.h5', 'r')
        >>> electrodes_uv = f['amplifier_data'][:, 100:200] * 0.195
        >>> electrodes_uv.shape
        (256, 100)
        >>> f['amplifier_data'].shape   # total available data
        (256, 7200000)

        The original header information is stored as a JSON string, which can be parsed like this:

        >>> import json
        >>> header = json.loads(f.attrs['JSON_header'])
        >>> header['sample_rate']
        20000.0
        """
    ap = argparse.ArgumentParser(description=app_description, formatter_class=CustomFormatter)
    ap.add_argument('--doc', action='store_true', help='Print usage message to README.txt')
    ap.add_argument('hdf_file', type=str, nargs='?', help='Output filename (.h5 extension will be issued)',
                    default='out')
    ap.add_argument('-i', '--input-files', type=str, nargs='+', help='Input RHD files in order')
    ap.add_argument('-d', '--input-dir', type=str, help='Input directory: RHD files will be converted in sorted order')
    ap.add_argument('-v', '--verbose', action='store_true', help='Turn on logging channel')
    args = ap.parse_args()
    if args.doc:
        with open('README.txt', 'w') as fw:
            fw.write(app_description)
        sys.exit(0)
    if args.verbose:
        root = logging.getLogger()
        root.setLevel(logging.INFO)
        handler = logging.StreamHandler(sys.stdout)
        handler.setLevel(logging.DEBUG)
        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
        handler.setFormatter(formatter)
        root.addHandler(handler)
    if args.input_files is not None and len(args.input_files):
        input_files = args.input_files
    elif args.input_dir is not None and len(args.input_dir):
        input_files = glob(os.path.join(os.path.abspath(args.input_dir), '*.rhd'))
        input_files = sorted(input_files)
    else:
        print('No input files were given!')
        ap.print_usage()
        sys.exit(2)
    print(input_files)
    outfile = os.path.splitext(args.hdf_file)[0] + '.h5'
    print(outfile)
    convert(input_files, outfile)


text:
import open3d
from subprocess import check_output, CalledProcessError
from pypcd import pypcd
from tempfile import NamedTemporaryFile
from pathlib import Path
from pandas import read_csv
from io import StringIO
from contextlib import ExitStack
import numpy as np
import copy
from multiprocessing import Pool
from itertools import repeat, product
import matplotlib.pyplot as plt
from functools import lru_cache
from scipy import spatial

from open3d.open3d.geometry import PointCloud

from tofnet.pointcloud.utils import rotate, angle, transform_with_conf, find_z
from tofnet.pointcloud.visualize import visualize_pointcloud, create_lineset

default_templates = [
    [1.8, 0.9],
    [2.0, 1.0],
    [2.0, 0.9],
    [2.1, 1.0],
    [1.8, 0.8],
]

def get_template(xsize=1.8, ysize=0.9, npoints=2000):
    xtempl = (np.random.beta(1.5, 1.5, size=npoints)-0.5)*xsize
    # ytempl = np.random.uniform(low=-ysize/2, high=ysize/2, size=target.shape[0])
    ytempl = (np.random.beta(1.4, 1.4, size=npoints)-0.5)*ysize
    template = np.c_[ # concatenate on last axis
        xtempl,
        ytempl,
        np.zeros(npoints)
    ]
    return template

def get_xytheta(transformation):
    x_trans = transformation[0,-1]
    y_trans = transformation[1,-1]
    theta = np.arctan2(transformation[1,0], transformation[0,0]) # np.arcsin(-transformation[0,1])
    transformation2 = np.linalg.inv(transformation)
    theta2 = -theta # np.arcsin(-transformation2[0,1])
    x, y = rotate(theta2, np.array([x_trans, y_trans]))
    return x, y, -theta

def init_transform(source):
    xinit = np.mean(source[:,0])
    yinit = np.mean(source[:,1])
    source = source - [xinit, yinit, 0]
    svd = np.linalg.svd(source[:,:-1])
    normal = svd[2][0]
    theta = np.arctan2(normal[0], normal[1]) # = angle([0,1], normal)
    source = rotate(theta, source, axis="z").copy()
    return source, xinit, yinit, theta

def create_config(x, y, theta, length=2.0, width=1.0, fitness=1.0):
    config = {
        "bed": {
            "centerX": x,
            "centerY": y,
            "orientation": theta,
            "length": length,
            "width": width, # Change this...
            "fitness":fitness
        }
    }
    return config

def grid_iou(bed_pcd, cfg, return_pcd=True, voxel_size=0.05, no_transform=False):
    """ Computes the fitness of a model around a segmentation

    Arguments:
        bed_pcd (np.ndarray): the 3d points segmented as being part of the bed
        cfg (dict): the proposed configuration
        return_pcd (bool): wether to return the pcd too
        voxel_size (float): size of the voxel to compensate for camera resolution
        no_transform (bool): wether to apply tranformation from the cfg on bed_pcd

    Returns:
        The fitness and the bed_pcd transformed if requested
    """
    cfg = cfg["bed"]
    if not no_transform:
        bed_pcd = bed_pcd.copy() - [cfg["centerX"], cfg["centerY"], 0]
        bed_pcd = rotate(np.radians(cfg["orientation"]), bed_pcd, axis="z")

    bed_pcd = np.round(bed_pcd[:,:2]/voxel_size)
    bed_pcd = np.unique(bed_pcd, axis=0)
    bed_pcd = bed_pcd*voxel_size

    x_ok = (bed_pcd[:,1] <= cfg["width"]/2) & (bed_pcd[:,1] >= -cfg["width"]/2)
    y_ok = (bed_pcd[:,0] <= cfg["length"]/2) & (bed_pcd[:,0] >= -cfg["length"]/2)
    n_ok = np.sum(x_ok & y_ok)
    n_elem = bed_pcd.shape[0]
    n_grid = (cfg["length"]+voxel_size)*(cfg["width"]+voxel_size)/(voxel_size*voxel_size)
    iou = n_ok/(n_grid+n_elem-n_ok)

    if return_pcd:
        return iou, bed_pcd
    else:
        return iou

from numba import njit
@njit
def multi_grid_iou(orig_bed_pcd, cfgs, voxel_size=0.05):
    """ Numba accelerated computation of the fitness of a model around a segmentation
        for multiple configs

    Arguments:
        orig_bed_pcd (np.ndarray): the 3d points segmented as being part of the bed
        cfgs (list): list of the possible configurations
        voxel_size (float): size of the voxel to compensate for camera resolution

    Returns:
        A list with the fitness for each cfg in cfgs
    """
    ious = np.zeros(len(cfgs))
    for i in range(cfgs.shape[0]):
        cfg = cfgs[i]
        bed_pcd = orig_bed_pcd - cfg[:2]
        theta = np.radians(cfg[2])
        rotmat = np.array([
                [np.cos(theta), np.sin(theta)],
                [-np.sin(theta),  np.cos(theta)],
            ])
        bed_pcd = bed_pcd@rotmat

        bed_pcd = np.round_(bed_pcd/voxel_size, 0, np.zeros_like(bed_pcd))
        bed_pcd_summed = np.unique(1024*bed_pcd[:,0]+bed_pcd[:,1])
        bed_pcd_0 = np.round_(bed_pcd_summed/1024, 0, np.zeros_like(bed_pcd_summed))
        bed_pcd_1 = bed_pcd_summed-bed_pcd_0*1024
        x_ok = (bed_pcd_1*2 <= int(cfg[4]/voxel_size)) & (bed_pcd_1*2 >= int(-cfg[4]/voxel_size))
        y_ok = (bed_pcd_0*2 <= int(cfg[3]/voxel_size)) & (bed_pcd_0*2 >= int(-cfg[3]/voxel_size))
        n_ok = np.sum(x_ok & y_ok)
        n_elem = len(bed_pcd_summed)
        n_grid = (cfg[3]+voxel_size)*(cfg[4]+voxel_size)/(voxel_size*voxel_size)
        ious[i] = n_ok/(n_grid+n_elem-n_ok)
    return ious

def dummy(source, templates=None, home=None):
    return create_config(0,0,0)

def no_icp(source, templates, home=None, use_2d=False):
    """ Computes the best-fitting bounding box around the source pixels.
        the bed model is estimated as being 2x1. The transformation is initialized with
        SVD, then a brute-force search around the initialization state tries to find a
        better fit
    """
    if source.shape[0] == 0:
        return create_config(0,0,0, fitness=0)
    origorig = source.copy()
    source, xinit, yinit, theta = init_transform(source)

    remove_out = False
    if remove_out:
        iq_range=0.5
        pcnt = (1 - iq_range) / 2
        xqlow, xqhigh = np.quantile(source[:,0], [pcnt, 1-pcnt])
        yqlow, yqhigh = np.quantile(source[:,1], [pcnt, 1-pcnt])
        xiqr, yiqr = xqhigh-xqlow, yqhigh-yqlow
        no_out = (source[:,0]<xqhigh+1.5*xiqr) & (source[:,0]>xqlow-1.5*xiqr) &\
                    (source[:,1]<yqhigh+1.5*yiqr) & (source[:,1]>yqlow-1.5*yiqr)
        source = source[no_out,:]

    remove_low = False
    if remove_low:
        not_low = (source[:,-1]>0.15) & (source[:,-1]<1.4)
        source = source[not_low,:]

    if source.shape[0] == 0:
        return create_config(0,0,0,fitness=0)

    # Recenter the data independently from point density
    xmin, xmax = np.min(source[:,0]), np.max(source[:,0])
    ymin, ymax = np.min(source[:,1]), np.max(source[:,1])
    x_center, y_center = xmin+(xmax-xmin)/2, ymin+(ymax-ymin)/2
    center2 = rotate(-theta, np.array([[x_center, y_center, 0]]), axis="z")
    xinit += center2[0][0]
    yinit += center2[0][1]
    source = source - [x_center, y_center, 0]

    angle_diffs = [-5.,-2.,-1.,0.,1.,2.,5.]
    center_diffs = [-0.2,-0.15,-0.10,-0.07,-0.05,-0.01,0,0.01,0.05,0.07,0.10,0.15,0.2]
    lengths = [2.,2.1]
    widths = [1.]
    trials = list(product(center_diffs, center_diffs, angle_diffs, lengths, widths))

    trialmap = np.array([np.array([xinit, yinit, np.degrees(theta)-90, 0.0, 0.0])+np.array(t) for t in trials])
    results = multi_grid_iou(origorig[:, :2], trialmap, 0.05)
    best = np.argmax(results)
    cfg = create_config(*trialmap[best])
    cfg["bed"]["fitness"] = np.max(results)
    return cfg

reason: import numpy,from scipy,from numba


text:
import numpy as np
from math import e,pi,sqrt
import cv2
import matplotlib.pyplot as plt

def kmeanscluster(k):
    img = cv2.imread('D:\\baboon.JPG')

    new_img = img.reshape((img.shape[0]*img.shape[1]), img.shape[2])

    qua=[]
    poi=[]
    randomx =[]
    randomy =[]

    randomx = np.random.choice(512,k)
    randomy = np.random.choice(512,k)

    for i in range(k):
        poi.append((randomx[i],randomy[i]))

    for i in range(k):
        qua.append(img[poi[i][0]][poi[i][1]])
    qua = np.asarray(qua)

    loop=0

    def kmeans(x,uz,k,loop):

        sumx=0
        sumy=0
        sumz=0
        leng=0

        ux=np.array([[0.0 for j in range(3)] for i in range(k)])
        yes=0

        dist= []
        print(len(uz))
        for i in range(len(x)):
            gar=[]
            for j in range(len(uz)):
                distance = np.sqrt((uz[j][0]-x[i][0])**2 + (uz[j][1]-x[i][1])**2 + (uz[j][2]-x[i][2])**2)

                gar.append(distance)
            y= gar.index(min(gar))

            dist.append(y)

        for i in range(k):

            for j in range(len(dist)):

                if(dist[j]==i):

                    leng=leng+1
                    sumx=sumx+x[j][0]

                    sumy=sumy+x[j][1]
                    sumz=sumz+x[j][2]

            ux[i][0]=(sumx/leng)
            ux[i][1]=(sumy/leng)
            ux[i][2]=(sumz/leng)
            leng=0
            sumx=0
            sumy=0
            sumz=0

        for i in range(k):
            for j in range(3):
                if(int(ux[i][j])==int(uz[i][j])):
                    yes +=1

        if(yes == k*k or loop == 13):
            print("yes")
            return dist,ux
        else:
            loop=loop+1
#        return dist,ux
            print("rec")
            return kmeans(x,ux,k,loop)

    di,yu = kmeans(new_img,qua,k,loop)

    di = np.asarray(di)

    print(yu)

    for i in range(len(new_img)):
        new_img[i] = yu[di[i]]

    ts=np.reshape(new_img,(512,512,3))

    cv2.imwrite('D://task3_baboon_'+str(k)+'.jpg',np.asarray(ts,dtype=np.uint8))

    return

k=[3,5,10,20]
for i in k:
    print("for k= "+str(i))
    print ("PLEASE WAIT FOR SOME TIME")
    kmeanscluster(i)
print("DONE")

max_stars_repo_path: K-Means clustering and GMM/Task 3.4.py

max_issues_repo_path: K-Means clustering and GMM/Task 3.4.py

max_forks_repo_path: K-Means clustering and GMM/Task 3.4.py