text:
from __future__ import division
import numpy as np
from numba import jit
import itertools
import time
import matplotlib.pyplot as plt
import sys
import abc
import json
import pickle
import urllib2
import urllib
from urllib import urlopen
import re
import pandas as pd
from sklearn.decomposition import PCA
from text_alignment import *
from text_cleaning import clean_document
from compiler.ast import flatten
from elasticsearch import Elasticsearch
import re
import csv
from sklearn.metrics import roc_curve, auc
import seaborn as sns
import random
from heapq import *
from sklearn.metrics import jaccard_similarity_score

class Experiment():

    def __init__(self, bills, algorithm, match_score = 3, mismatch_score = -1,
                gap_score = -2, gap_start = -5, gap_extend = -0.5, total_num = 15):

        '''
        total_num : total_num of bills to consider
        '''
        groups = set([value['match'] for key, value in bills.items()])
        num_groups = len(groups)

        bills_to_keep = []
        for i in range(num_groups):
            match_group = [key for key, value in bills.items() if value['match'] == i]

            take = total_num / num_groups

            while match_group != [] and take > 0:
                choice = random.choice(range(len(match_group)))
                bills_to_keep.append(match_group.pop(choice))

                take -= 1

        #get rest of bills to get total_num of bills
        left = set([key for key, value in bills.items()]) - set(bills_to_keep)
        left = list(left)

        for i in range(total_num - len(left)):
            choice = random.choice(range(len(match_group)))
            bills_to_keep.append(match_group.pop(choice))

    	self.bills = {key : bills[key] for key in bills_to_keep}

        self.algorithm = algorithm
        if bills == {}:
            self.scores = None
        else:
    	   self.scores = np.zeros((max(self.bills.keys())+1, max(self.bills.keys())+1))
    	self.results = {}
        self.match_score = match_score
        self.mismatch_score = mismatch_score
        self.gap_score = gap_score
        self.gap_start = gap_start
        self.gap_extend = gap_extend

    def evaluate(self):
        self.evaluate_algorithm(self.bills)

        self.plot_scores(self.scores, self.bills)

        return self.scores, self.results

    def evaluate_algorithm(self):
        '''
        args:
            matches: dictionary with field corresponding to text and match groups

        returns:
            matrix with scores between all pairs and a dictionary with information
        '''
        param_combs = self._gen_param_combs()
        for i,j in param_combs:

            if self.bills[i] == {} or self.bills[j] == {}:
                continue

            if bills[i]['text'] == '' or bills[j]['text'] == '':
                continue

            text1 = self._prepare_text_left(bills[i]['text'], bills[i]['state'])
            text2 = self._prepare_text_right(bills[j]['text'], bills[j]['state'])

            self.results[(i,j)] = {}
            self.results[(i,j)]['left_length'] = len(flatten(text1))
            self.results[(i,j)]['right_length'] = len(text2)
            self.results[(i,j)]['left_state'] = bills[i]['state']
            self.results[(i,j)]['right_state'] = bills[j]['state']

            #instantiate aligner with appropriate parameters
            if self.algorithm == LocalAligner:
                f = self.algorithm(self.match_score, self.mismatch_score, self.gap_score)
            elif self.algorithm == AffineLocalAligner:
                f = self.algorithm(self.match_score, self.mismatch_score, self.gap_start, self.gap_extend)

            alignment = f.align(text1, text2)

            self.scores[i,j] = self._get_score(alignment, i, j)

            self.results[(i,j)]['alignments'] = alignment.alignments
            self.results[(i,j)]['score'] = self._get_score(alignment, i, j)
            self.results[(i,j)]['match'] = (bills[i]['match'] == bills[j]['match'])
            self.results[(i,j)]['diff'] = [self._diff(a) for a in alignment.alignments]
            self.results[(i,j)]['features'] = [self._alignment_features(a[1],a[2]) for a in alignment.alignments]

            print 'i: ' + str(i) + ', j: ' + str(j) + ' score: ' + str(alignment.alignments[0][0])

        return self.scores, self.results

    def plot_roc_score(self):
        truth = [value['match'] for key, value in self.results.items()]
        score = [value['score'] for key, value in self.results.items()]

        roc = roc_curve(truth, score)
        fpr = roc[0]
        tpr = roc[1]
        roc_auc = auc(fpr, tpr)

        plt.figure()
        plt.plot(fpr, tpr, label='ROC curve (area = %0.2f)' % roc_auc)
        plt.plot([0, 1], [0, 1], 'k--')
        plt.xlim([0.0, 1.0])
        plt.ylim([0.0, 1.05])
        plt.xlabel('False Positive Rate')
        plt.ylabel('True Positive Rate')
        plt.title('Receiver operating characteristic example')
        plt.legend(loc="lower right")
        plt.show()

    def plot_roc_matches(self):
        truth = [value['match'] for key, value in self.results.items()]
        score = [value['features'][0]['num_matches'] for key, value in self.results.items()]

        roc = roc_curve(truth, score)
        fpr = roc[0]
        tpr = roc[1]
        roc_auc = auc(fpr, tpr)

        plt.figure()
        plt.plot(fpr, tpr, label='ROC curve (area = %0.2f)' % roc_auc)
        plt.plot([0, 1], [0, 1], 'k--')
        plt.xlim([0.0, 1.0])
        plt.ylim([0.0, 1.05])
        plt.xlabel('False Positive Rate')
        plt.ylabel('True Positive Rate')
        plt.title('Receiver operating characteristic example')
        plt.legend(loc="lower right")
        plt.show()

    def save(self):
        if self.algorithm == LocalAligner:
            with open('../../data/experiment_results/experiment_{0}_m_{1}_mm_{2}_g_{3}.p'.format(self.algorithm._algorithm_name,
                                self.match_score, self.mismatch_score, self.gap_score), 'wb') as fp:
                pickle.dump(self, fp)

    @abc.abstractmethod
    def _gen_param_combs(self):
        pass

    @abc.abstractmethod
    def _prepare_text_left(self):
    	pass

    def _prepare_text_right(self, text, state):
        if state == 'model_legislation':
            text = clean_document(text, doc_type = state)
        else:
            text = clean_document(text, doc_type = 'state_bill' , state_id = state)

        return map(lambda x: x.split(), text)

    @abc.abstractmethod
    def _get_score(self, alignment, i, j):
        pass

    #alignment feature methods
    def _alignment_features(self,left, right):
        '''
        This function takes as input two alignments and produce features of these
        '''
        return alignment_features(left, right)

    def inspect_alignments(self, match_type = 0, start_score = 'max'):
        '''
            match_type is 0 if you want to inspect non-matches
            and 1 if you want to inspect matches
        '''
        alignments = [(value['score'], value['alignments'], key)  for key, value in self.results.items() if value['match'] == match_type]
        sorted_alignments = sorted(alignments, key=lambda tup: tup[0], reverse = True)

        if start_score == 'max':
            for total_score, alignments, key in sorted_alignments:
                for score, left, right in alignments:
                    for i in range(len(left)):
                        print left[i], right[i]

                    print 'alignment_score: ', score

                print 'total_alignments_score: ', total_score
                print 'key: ', key
                print '\n'

                raw_input("Press Enter to continue...")
        else:
            for total_score, alignments, key in sorted_alignments:
                if total_score > start_score:
                    pass
                else:
                    for score, left, right in alignments:
                        for i in range(len(left)):

                            print left[i], right[i]

                        print 'alignment_score: ', score

                    print 'total_alignments_score: ', total_score
                    print 'key: ', key
                    print '\n'

                raw_input("Press Enter to continue...")

    #plotting functions
    def plot_scores(self):

        matchScores = []
        nonMatchScores = []

        for i in self.bills.keys():
            for j in self.bills.keys():

                if (i,j) not in self.results or self.results[(i,j)]['score'] == 0:
                    #ignore if score zero because url is broken
                    pass
                elif i < j and self.results[(i,j)]['match']:
                    matchScores.append(min(self.results[(i,j)]['score'],200))
                else:
                    nonMatchScores.append(min(self.results[(i,j)]['score'],200))

        bins = np.linspace(min(nonMatchScores + matchScores), max(nonMatchScores + matchScores), 100)
        plt.hist(nonMatchScores, bins, alpha=0.5, label='Non-Matches')
        plt.hist(matchScores, bins, alpha=0.5, label='Matches')
        plt.legend(loc='upper right')
        plt.xlabel('Alignment Score')
        plt.ylabel('Number of Bill Pairs')
        plt.title('Distribution of Alignment Scores')
        plt.show()

        #make boxplot
        data_to_plot = [matchScores, nonMatchScores]
        fig = plt.figure(1, figsize=(9, 6))
        ax = fig.add_subplot(111)
        bp = ax.boxplot(data_to_plot)
        ax.set_xticklabels(['Match Scores', 'Non-Match Scores'])
        fig.show()

    def plot_num_matches(self):

        matchScores = []
        nonMatchScores = []

        for i in self.bills.keys():
            for j in self.bills.keys():

                if self.scores[i,j] == 0:
                    #ignore if score zero because url is broken
                    pass
                elif i < j and self.results[(i,j)]['match']:
                    matchScores.append(min(self.results[(i,j)]['features'][0]['num_matches'],200))
                else:
                    nonMatchScores.append(min(self.results[(i,j)]['features'][0]['num_matches'],200))

        bins = np.linspace(min(nonMatchScores + matchScores), max(nonMatchScores + matchScores), 100)
        plt.hist(nonMatchScores, bins, alpha=0.5, label='Non-Matches')
        plt.hist(matchScores, bins, alpha=0.5, label='Matches')
        plt.legend(loc='upper right')
        plt.xlabel('Alignment Score')
        plt.ylabel('Number of Bill Pairs')
        plt.title('Distribution of Alignment Scores')
        plt.show()

        #make boxplot
        data_to_plot = [matchScores, nonMatchScores]
        fig = plt.figure(1, figsize=(9, 6))
        ax = fig.add_subplot(111)
        bp = ax.boxplot(data_to_plot)
        ax.set_xticklabels(['Match Scores', 'Non-Match Scores'])
        fig.show()

    def low_rank_plot(self):

        #convert dictionary to matrix
        matches = [[value for key, value in values['features'].items()] \
            for keys, values in self.results.items() if values['match'] == 1]

        non_matches = [[value for key, value in values['features'].items()] \
            for keys, values in self.results.items() if values['match'] == 0]

        #matches from 0 to match_index
        match_index = len(matches)

        data = np.array(matches + non_matches)

        sklearn_pca = PCA(n_components=2)
        sklearn_transf = sklearn_pca.fit_transform(data)

        #plot data
        fig = plt.figure()
        ax1 = fig.add_subplot(111)
        plt.plot(sklearn_transf[:match_index,0],sklearn_transf[:match_index,1],
                 'o', markersize=7, color='blue', alpha=0.5, label='matches')
        plt.plot(sklearn_transf[match_index:,0], sklearn_transf[match_index:,1],
                 '^', markersize=7, color='red', alpha=0.5, label='non-matches')
        plt.xlim([-50,1000])
        plt.ylim([-500,500])
        plt.legend(loc='upper left');
        plt.show()

    ############################################################
    ##alignments utils
    def _diff(self, alignment):
        a = alignment[1]
        b = alignment[2]
        length = min(len(alignment[1]), len(alignment[2]))

        diff = []
        for i in range(length):
            if a[i] == b[i] or a[i] == '-' or b[i] == '-':
                continue
            else:
                diff.append((a[i], b[i]))

        return diff

########################################################################################################################
class DocExperiment(Experiment):

    def _gen_param_combs(self):
        keys = self.bills.keys()

        param_combs = []
        for i in keys:
            for j in keys:
                if i < j:
                    param_combs.append((i,j))

        return param_combs

    def _prepare_text_left(self, text, state):
        if state == 'model_legislation':
            text = clean_document(text, doc_type = state)
        else:
            text = clean_document(text, doc_type = 'state_bill' , state_id = state)

        return map(lambda x: x.split(), text)

    def _get_score(self, alignment, i, j):
        return alignment.alignments[0][0]

########################################################################################################################
class DocLengthExperiment(DocExperiment):
    def _get_score(self, alignment, i, j):

        return weight_length(alignment, self.results[(i,j)]['left_length'],
            self.results[(i,j)]['right_length'])

class DocTfidfExperiment(DocExperiment):
    def __init__(self, bills, algorithm, state_tfidf, match_score = 3, mismatch_score = -1,
            gap_score = -2, gap_start = -5, gap_extend = -0.5, total_num = 20):
        '''
        total_num : total_num of bills to consider
        '''
        self.bills = {key : value for key, value in bills.items() if key <= total_num}
        self.algorithm = algorithm
        if bills == {}:
            self.scores = None
        else:
           self.scores = np.zeros((max(self.bills.keys())+1, max(self.bills.keys())+1))
        self.results = {}
        self.match_score = match_score
        self.mismatch_score = mismatch_score
        self.gap_score = gap_score
        self.gap_start = gap_start
        self.gap_extend = gap_extend
        self.state_tfidf = state_tfidf

    def _get_score(self, alignment, i, j):
        return weight_tfidf(alignment, self.state_tfidf, self.results[(i,j)]['left_state'],
            self.results[(i,j)]['right_state'])

########################################################################################################################
class SectionExperiment(Experiment):

    def _gen_param_combs(self):
        keys = self.bills.keys()

        param_combs = []
        for i in keys:
            for j in keys:
                if i != j:
                    param_combs.append((i,j))

        return param_combs

    def _prepare_text_left(self, text, state):
        if state == 'model_legislation':
            text = clean_document(text, doc_type = state, split_to_section = True)
        else:
            text = clean_document(text, doc_type = 'state_bill' , state_id = state, split_to_section = True)

        return map(lambda x: x.split(), text)

    def _get_score(self, alignment, i, j):

        scores = [score for score, left, right in alignment.alignments]

        return np.mean(scores)

########################################################################################################################

class SectionLengthExperiment(SectionExperiment):
    def _get_score(self, alignment, i, j):

        return weight_length(alignment, self.results[(i,j)]['left_length'],
            self.results[(i,j)]['right_length'])

class SectionTfidfExperiment(SectionExperiment):
    def __init__(self, bills, algorithm, state_tfidf, match_score = 3, mismatch_score = -1,
            gap_score = -2, gap_start = -5, gap_extend = -0.5, total_num = 20):
        '''
        total_num : total_num of bills to consider
        '''
        self.bills = {key : value for key, value in bills.items() if key <= total_num}
        self.algorithm = algorithm
        if bills == {}:
            self.scores = None
        else:
           self.scores = np.zeros((max(self.bills.keys())+1, max(self.bills.keys())+1))
        self.results = {}
        self.match_score = match_score
        self.mismatch_score = mismatch_score
        self.gap_score = gap_score
        self.gap_start = gap_start
        self.gap_extend = gap_extend
        self.state_tfidf = state_tfidf

    def _get_score(self, alignment, i, j):
        return weight_tfidf(alignment, self.state_tfidf, self.results[(i,j)]['left_state'],
            self.results[(i,j)]['right_state'])

########################################################################################################################
class GridSearch():

    def __init__(self, bills, algorithm, match_scores = [2,3,4], mismatch_scores = [-1,-2,-3],
                gap_scores = [-1,-2,-3], gap_starts = [-2,-3,-4], gap_extends = [-0.5,-1,-1.5]):
        self.bills = bills
        self.algorithm = algorithm
        if bills == {}:
            self.scores = None
        else:
           self.scores = np.zeros((max(self.bills.keys())+1, max(self.bills.keys())+1))
        self.grid = {}
        self.grid_df = None
        self.match_scores = match_scores
        self.mismatch_scores = mismatch_scores
        self.gap_scores = gap_scores
        self.gap_starts = gap_starts
        self.gap_extends = gap_extends

    def evaluate_algorithm(self):
        #only works for doc experiemnt currently
        #determine parameters to do grid search on for given algorithm
        if self.algorithm == LocalAligner:
            for match_score in self.match_scores:
                for mismatch_score in self.mismatch_scores:
                    for gap_score in self.gap_scores:

                        print 'running LocalAligner model: match_score {0} mismatch_score {1} gap_score {2}'.format(match_score, mismatch_score, gap_score)

                        e = DocExperiment(self.bills, LocalAligner, match_score = match_score,
                                            mismatch_score = mismatch_score, gap_score = gap_score)

                        e.evaluate_algorithm()

                        self.grid[(match_score, mismatch_score, gap_score)] = e

        elif self.algorithm == AffineLocalAligner:
            for match_score in self.match_scores:
                for mismatch_score in self.mismatch_scores:
                    for gap_start in self.gap_starts:
                        for gap_extend in self.gap_extends:

                            print 'running AffineLocalAligner model: match_score {0} mismatch_score {1} \
                                    gap_start {2} gap_extend'.format(match_score, mismatch_score,
                                                                     gap_start, gap_extend)

                            e = DocExperiment(self.bills, AffineLocalAligner, match_score = match_score,
                                                mismatch_score = mismatch_score, gap_start = gap_start,
                                                gap_extend = gap_extend)

                            e.evaluate_algorithm()

                            self.grid[(match_score, mismatch_score, gap_score)] = e

            return self.grid

    def _create_grid_df(self):
        t = []
        for key1, value1 in self.grid.items():
            for key2, value2 in value1.results.items():
                t.append(list(key1) + [key2, value2['score'], value2['match']])

        self.grid_df = pd.DataFrame(t)

        if self.algorithm == LocalAligner:
            self.grid_df.columns = ['match_score', 'mismatch_score', 'gap_score', 'pair', 'score', 'match']
        elif self.algorithm == AffineLocalAligner:
            self.grid_df.columns = ['match_score', 'mismatch_score', 'gap_start', 'gap_extend', 'pair', 'score', 'match']

    def plot_grid(self):

        self._create_grid_df()

        df = self.grid_df
        #make maximum possible 500
        df.loc[df['score']>500,'score'] = 500

        #match plot
        df_match = df[(df['mismatch_score'] == -2) & (df['gap_score'] == -1)]

        g = sns.FacetGrid(df_match, col="match_score")
        g = g.map(sns.boxplot, "match", "score")
        sns.plt.ylim(0,400)
        sns.plt.show()

        #mismatch plot
        df_mismatch = df[(df['match_score'] == 3) & (df['gap_score'] == -1)]

        g = sns.FacetGrid(df_mismatch, col="mismatch_score")
        g = g.map(sns.boxplot, "match", "score")
        sns.plt.ylim(0,400)
        sns.plt.show()

        #gap plot
        df_gap = df[(df['match_score'] == 3) & (df['mismatch_score'] == -2)]

        g = sns.FacetGrid(df_gap, col="gap_score")
        g = g.map(sns.boxplot, "match", "score")
        sns.plt.ylim(0,400)
        sns.plt.show()

    def plot_roc(self):
        experiments = [(str(key), value) for key,value in self.grid.items()]
        roc_experiments(experiments)

    def save(self, name):
        with open('../../data/grid_results/{0}.p'.format(name), 'wb') as fp:
            pickle.dump(self, fp)

############################################################
##helper function
def alignment_features(left, right):
    '''
    This function takes as input two alignments and produce features of these
    '''
    #alignment features
    features = {}
    features['length'] = len(left)
    features['num_gaps_l'] = 0
    features['num_gaps_r'] = 0
    features['num_mismatches'] = 0
    features['num_matches'] = 0
    features['avg_gap_length_l'] = []
    features['avg_gap_length_r'] = []
    features['avg_consec_match_length'] = []
    features['jaccard_score'] = jaccard_similarity_score(left,right)

    #helper variables
    prev_gap_l = False
    prev_gap_r = False
    prev_match = False
    for i in range(len(left)):
        # print 'i: ', i
        # print 'features: ', features
        if left[i] == '-':
            features['num_gaps_l'] += 1
            if not prev_gap_l:
                features['avg_gap_length_l'].append(1)
                prev_gap_l = True
            else:
                features['avg_gap_length_l'][-1] += 1
        else:
            prev_gap_l = False
        if right[i] == '-':
            features['num_gaps_r'] += 1
            if not prev_gap_r:
                features['avg_gap_length_r'].append(1)
                prev_gap_r = True
            else:
                features['avg_gap_length_r'][-1] += 1
        else:
            prev_gap_r = False
        if left[i] != '-' and right[i] != '-':
            if left[i] != right[i]:
                features['num_mismatches'] += 1
            elif left[i] == right[i]:
                features['num_matches'] += 1
                if not prev_match:
                    features['avg_consec_match_length'].append(1)
                    prev_match = True
                else:
                    features['avg_consec_match_length'][-1] += 1
        if left[i] != right[i]:
            prev_match = False

    if features['avg_gap_length_l'] != []:
        features['avg_gap_length_l'] = np.mean(features['avg_gap_length_l'])
    else:
        features['avg_gap_length_l'] = 0
    if features['avg_gap_length_r'] != []:
        features['avg_gap_length_r'] = np.mean(features['avg_gap_length_r'])
    else:
        features['avg_gap_length_r'] = 0
    if features['avg_consec_match_length'] != []:
        features['avg_consec_match_length'] = np.mean(features['avg_consec_match_length'])
    else:
        features['avg_consec_match_length'] = 0

    return features

#good test case: results[(21,22)]
def test_alignment_features():

    def true_features(length, num_gaps_l, num_gaps_r, num_mismatches,
                        num_matches, avg_gap_length_l, avg_gap_length_r,
                        avg_consec_match_length):
        truth = {}
        truth['length'] = length
        truth['num_gaps_l'] = num_gaps_l
        truth['num_gaps_r'] = num_gaps_r
        truth['num_mismatches'] = num_mismatches
        truth['num_matches'] = num_matches
        truth['avg_gap_length_l'] = avg_gap_length_l
        truth['avg_gap_length_r'] = avg_gap_length_r
        truth['avg_consec_match_length'] = avg_consec_match_length

        return truth

    def check_features_truth(truth, features):
        for field in features.keys():
            if features[field] != truth[field]:
                print field + ' is inconsistent'

    a1 = range(1,10) + ['-', '-'] + range(12,15)
    a2 = range(1,4) + ['-', '-', 7] + range(7,15)

    features = alignment_features(a1, a2)
    truth = true_features(14,2,2,1,9,2,2,3)

    print 'first test: '
    check_features_truth(truth, features)

    b1 = range(1,10)
    b2 = ['-','-',3,5,6,7, '-', '-', 9]
    features = alignment_features(b1, b2)
    truth = true_features(9,0,4,3,2,0,2,1)

    print 'second test: '
    check_features_truth(truth, features)

############################################################
##analysis functions
def roc_experiments(experiments):
    '''
    args:
        experiments : list of tuples where first entry is name of experiment and second entry is experiment object
    returns:
        roc plot of all the experiments
    '''
    fpr = dict()
    tpr = dict()
    roc_auc = dict()
    for i in range(len(experiments)):
        truth = [value['match'] for key, value in experiments[i][1].results.items()]
        score = [value['score'] for key, value in experiments[i][1].results.items()]

        roc = roc_curve(truth, score)
        fpr[i] = roc[0]
        tpr[i] = roc[1]
        roc_auc[i] = auc(fpr[i], tpr[i])

    # best_experiments = range(len(experiments))
    #find 5 models with largest auc
    t = [(key,value)  for key,value in  roc_auc.items()]
    best = nlargest(5, t, key=lambda x: x[1])
    best_experiments = [b[0] for b in best]

    # Plot ROC curve
    plt.figure()
    for i in best_experiments:
        plt.plot(fpr[i], tpr[i], label='ROC curve of algorithm {0} (area = {1:0.2f})'
                                       ''.format(experiments[i][0], roc_auc[i]))

    plt.plot([0, 1], [0, 1], 'k--')
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('Comparison of ROC curves of multiple experiments')
    plt.legend(loc="lower right")
    plt.show()

############################################################
##data creating, saving, and loading

def load_bills():
    with open('../../data/bills.p','rb') as fp:
        bills =pickle.load(fp)

    return bills

def load_pickle(name):
    with open('{0}.p'.format(name),'rb') as fp:
        f =pickle.load(fp)

    return f

def load_scores():
    scores = np.load('../../data/scores.npy')

    return scores

def save_results(results):
    with open('../../data/results.json','wb') as fp:
        pickle.dump(results, fp)

def load_results():
    with open('../../data/results.json','rb') as fp:
        data =pickle.load(fp)
    return data

def test_experiment(experiment, bills, algorithm, file_name):
    print 'testing {0}...'.format(file_name)

    e = experiment(bills, algorithm)
    e.evaluate_algorithm()

    with open('{0}.p'.format(file_name), 'wb') as fp:
        pickle.dump(e, fp)

    return e

if __name__ == '__main__':

    bills = load_bills()

    e = test_experiment(DocExperiment, bills, LocalAligner, 'local_experiment')

    test_experiment(SectionExperiment, bills, LocalAligner, 'section_local_experiment')

    #test_experiment(GridSearch, bills, LocalAligner, 'grid_local_experiment')

    #test_experiment(GridSearch, bills, AffineLocalAligner, 'grid_affine_experiment')

    # test_experiment(DocExperiment, bills, AffineLocalAligner, 'affine_local_experiment')

    # test_experiment(SectionExperiment, bills, AffineLocalAligner, 'affine_section_experiment')

    # test_experiment(DocLengthExperiment, bills, LocalAligner, 'length_doc_local_experiment')

    # test_experiment(SectionLengthExperiment, bills, LocalAligner, 'section_doc_local_experiment')

    # print 'testing DocTfidfExperiment local alignment...'
    # print 'loading state tfidfs....'
    # with open('state_tfidfs.p', 'rb') as fp:
    #     tfidf = pickle.load(fp)

    # print 'evaluating algorithm....'
    # e = DocTfidfExperiment(bills, LocalAligner, tfidf)
    # e.evaluate_algorithm()

    # with open('doc_tfidf_experiment.p', 'wb') as fp:
    #     pickle.dump(e, fp)

    # print 'testing sectionTfidfExperiment local alignment...'
    # print 'loading state tfidfs....'
    # with open('state_tfidfs.p', 'rb') as fp:
    #     tfidf = pickle.load(fp)

    # print 'evaluating algorithm....'
    # e = SectionTfidfExperiment(bills, LocalAligner, tfidf)
    # e.evaluate_algorithm()

    # with open('section_tfidf_experiment.p', 'wb') as fp:
    #     pickle.dump(e, fp)

########################################################################################################################
########################################################################################################################
#code to plot roc curves for bad version of affine experiment

# experiments = [(str(key), value['results']) for key,value in self.grid.items()]

# fpr = dict()
# tpr = dict()
# roc_auc = dict()
# for i in range(len(experiments)):
#     truth = [value['match'] for key, value in experiments[i][1].items()]
#     score = [value['score'] for key, value in experiments[i][1].items()]

#     roc = roc_curve(truth, score)
#     fpr[i] = roc[0]
#     tpr[i] = roc[1]
#     roc_auc[i] = auc(fpr[i], tpr[i])

# # best_experiments = range(len(experiments))
# #find 5 models with largest auc
# t = [(key,value)  for key,value in  roc_auc.items()]
# best = nlargest(5, t, key=lambda x: x[1])
# best_experiments = [b[0] for b in best]

# # Plot ROC curve
# plt.figure()
# for i in best_experiments:
#     plt.plot(fpr[i], tpr[i], label='ROC curve of algorithm {0} (area = {1:0.2f})'
#                                    ''.format(experiments[i][0], roc_auc[i]))

# plt.plot([0, 1], [0, 1], 'k--')
# plt.xlim([0.0, 1.0])
# plt.ylim([0.0, 1.05])
# plt.xlabel('False Positive Rate')
# plt.ylabel('True Positive Rate')
# plt.title('Comparison of ROC curves of multiple experiments')
# plt.legend(loc="lower right")
# plt.show()

reason: import numpy,from numba


text:
# Reference: Chiu 1991
# https://www.r-bloggers.com/cross-validation-for-kernel-density-estimation/
import numpy as np
from scipy import stats
from scipy import integrate
from scipy import optimize

#def Silverman1986_h(X):
#    std = np.std(X)
#    X_shape = X.shape
#    N=X_shape[0]
#    return 1.06*std*N**(-.2)

def Silverman1986_h(X):
    kde = stats.gaussian_kde(X,'silverman')
    return kde.factor

def Scott1986_h(X):
    kde = stats.gaussian_kde(X,'scott')

def Chiu1999_crossvalidation(X):

    def fhati(X,h,i):
        if h is float: _h=h
        else: _h=h[0]

        Xi = np.delete(X,i)
        kde = stats.gaussian_kde(Xi,_h)
        return kde(X[i])

    def J(X,h):
        if h is float: _h=h
        else: _h=h[0]

        fhat = stats.gaussian_kde(X,_h)
        #F1 = integrate.quad(lambda x: fhat(x)**2,-np.inf,np.inf)[0]
        F1 = fhat.integrate_kde(fhat)
        F2 = np.array([fhati(X,h,i) for i in range(X.shape[0])])
        return F1-2*np.mean(F2)

    #h0 = Silverman1986_h(X)
    h0 = .5
    results = optimize.minimize(
        lambda h: J(X,h),
        h0,
        method='Nelder-Mead')

    return results.x[0]

if __name__ == "__main__":
    #mu1, sigma1, N1 = 0, 1, 50
    #mu2, sigma2, N2 = 1, 1, 50

    #X1 = np.random.normal(mu1,sigma1,50)
    #X2 = np.random.normal(mu2,sigma2,50)
    #X = np.concatenate((X1,X2))

    from pypospack.pyposmat.data import PyposmatDataFile

    for i in range(9):
        fn = '../../../data_test/Ni__eam__born_exp_fs_00/data__Ni__eam__born_exp_fs_02/pyposmat.kde.{}.out'.format(i)
        data=PyposmatDataFile()
        data.read(filename=fn)
        X=data.df[data.parameter_names].values

        print("Silverman1986_h:{}".format(Silverman1986_h(X.T)))
        print("Scott1986_h:{}".format(Scott1986_h(X.T)))
        h = Chiu1999_crossvalidation(X.T)
        print('Chiu1999_h:{}'.format(h))
        print(80*'-')

ext: py

max_stars_repo_name: eragasa/pypospack

max_issues_repo_name: eragasa/pypospack

max_forks_repo_name: eragasa/pypospack

reason: import numpy,from scipy


text:
import os
import cv2
import sys
import time
import torch
import argparse
import torch.optim
import numpy as np
from formal_utils import *
from skimage.util import view_as_windows
from skimage.transform import resize
from torch.utils.data import Dataset

use_cuda = torch.cuda.is_available()

# Fixing for deterministic results
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False

class occlusion_analysis:
    def __init__(self, image, net, num_classes=256, img_size=227, batch_size=64,
                 org_shape=(224, 224)):
        self.image = image
        self.model = net
        self.num_classes = num_classes
        self.img_size = img_size
        self.org_shape = org_shape
        self.batch_size = batch_size

    def explain(self, neuron, loader, l_map, heatmap_type='SP', path='./'):

        # Compute original output
        org_softmax = torch.nn.Softmax(dim=1)(self.model(self.image))

        eval0 = org_softmax.data[0, neuron]

        batch_heatmap = torch.Tensor().to('cuda')

        # Create save_path for storing intermediate steps
        path = os.path.join(path, 'intermediate_steps')
        mkdir_p(path)

        for i, data in enumerate(loader):
            data = data.to('cuda')
            if heatmap_type == 'SP':
                softmax_out = torch.nn.Softmax(dim=1)(self.model(data * self.image))
                delta = eval0 - softmax_out.data[:, neuron]
                amax, aind = softmax_out.max(dim=1)
                gt_val = softmax_out.data[:, neuron]

                # For saving intermediate steps
                for j in range(data.shape[0])[:1]:
                    temp_img = np.uint8(255 * unnormalize(
                        np.moveaxis((data[j, :] * self.image[0, :]).cpu().detach().numpy().transpose(), 0, 1)))
                    cv2.imwrite(
                        os.path.abspath(os.path.join(path, 'intermediate_{:05d}_{}_{:.3f}_{}_{:.3f}.jpg'
                                     .format(i * self.batch_size + j, l_map[aind[j].item()].split(',')[0].split(' ')[0].split('-')[0],
                                             amax[j].item(), l_map[neuron].split(',')[0].split(' ')[0].split('-')[0],
                                             gt_val[j].item()))),
                        cv2.cvtColor(temp_img, cv2.COLOR_BGR2RGB))

            elif heatmap_type == 'SPG':
                inpaint_img, _ = impant_model.generate_background(self.image, data, batch_process=True)
                inpaint_img = self.image * data + inpaint_img * (1 - data)
                softmax_out = torch.nn.Softmax(dim=1)(self.model(inpaint_img))
                delta = eval0 - softmax_out.data[:, neuron]
                amax, aind = softmax_out.max(dim=1)
                gt_val = softmax_out.data[:, neuron]

                # For saving intermediate steps
                for j in range(inpaint_img.shape[0])[:1]:
                    temp_img = np.uint8(255 * unnormalize(
                        np.moveaxis(inpaint_img[j, :].cpu().detach().numpy().transpose(), 0, 1)))
                    cv2.imwrite(
                        os.path.abspath(os.path.join(path, 'intermediate_{:05d}_{}_{:.3f}_{}_{:.3f}.jpg'
                                     .format(i * self.batch_size + j, l_map[aind[j].item()].split(',')[0].split(' ')[0].split('-')[0],
                                             amax[j].item(), l_map[neuron].split(',')[0].split(' ')[0].split('-')[0],
                                             gt_val[j].item()))),
                        cv2.cvtColor(temp_img, cv2.COLOR_BGR2RGB))

            batch_heatmap = torch.cat((batch_heatmap, delta))

        sqrt_shape = len(loader)
        attribution = np.reshape(batch_heatmap.cpu().numpy(), (sqrt_shape, sqrt_shape))

        return attribution

if __name__ == '__main__':

    # Hyper parameters
    parser = argparse.ArgumentParser(description='Processing Meaningful Perturbation data')
    parser.add_argument('--img_path', type=str,
                        default='/home/chirag/ILSVRC2012_img_val_bb/ILSVRC2012_img_val/',
                        help='filepath for the example image')

    parser.add_argument('--algo', type=str,
                        default='SPG', help='SP|SPG')

    parser.add_argument('--size', type=int,
                        default=224, help='mask size to be optimized')

    parser.add_argument('--patch_size', type=int,
                        default=77, help='patch size for occlusion')

    parser.add_argument('--stride', type=int,
                        default=3, help='stride size for occlusion')

    parser.add_argument('--batch_size', type=int,
                        default=32, help='batch size')

    parser.add_argument('--true_class', type=int,
                        default=565,
                        help='target class of the image you want to explain')

    parser.add_argument('--dataset', type=str,
                        default='imagenet', help='dataset to run on imagenet | places365')

    parser.add_argument('--save_path', type=str,
                        default='./',
                        help='path for saving results')

    parser.add_argument('--weight_file', type=str,
                        default='/home/chirag/gpu3_codes/generative_inpainting_FIDO/model_logs/release_imagenet_256/',
                        help='path for the weight files of the inpainter model for imagenet | places365')
    args = parser.parse_args()

    if args.dataset == 'imagenet':

        model = load_model(arch_name='resnet50')

        # load the class label
        label_map = load_imagenet_label_map()

    elif args.dataset == 'places365':

        model = load_model_places365(arch_name='resnet50')

        # load the class label
        label_map = load_class_label()
    else:
        print('Invalid datasest!!')
        exit(0)

    model = torch.nn.DataParallel(model).to('cuda')
    model.eval()

    for p in model.parameters():
        p.requires_grad = False

    batch_size = int((224 - args.patch_size) / args.stride) + 1

    # Create all occlusion masks initially to save time
    # Create mask
    input_shape = (3, args.size, args.size)
    total_dim = np.prod(input_shape)
    index_matrix = np.arange(total_dim).reshape(input_shape)
    idx_patches = view_as_windows(index_matrix, (3, args.patch_size, args.patch_size), args.stride).reshape((-1,) +
                                                                                (3,
                                                                                args.patch_size,
                                                                                args.patch_size))

    # Start perturbation loop
    batch_size = int((args.size - args.patch_size) / args.stride) + 1
    batch_mask = torch.zeros(((idx_patches.shape[0],) + input_shape), device='cuda')
    total_dim = np.prod(input_shape)
    for i, p in enumerate(idx_patches):
        mask = torch.ones(total_dim, device='cuda')
        mask[p.reshape(-1)] = 0  # occ_val
        batch_mask[i] = mask.reshape(input_shape)

    trainloader = torch.utils.data.DataLoader(batch_mask.cpu(), batch_size=batch_size, shuffle=False,
                                              num_workers=0)
    del mask
    del batch_mask

    if args.algo == 'SPG':

        # Tensorflow CA-inpainter from FIDO
        sys.path.insert(0, './generative_inpainting/')
        from CAInpainter import CAInpainter

        impant_model = CAInpainter(batch_size, checkpoint_dir=args.weight_file)

    init_time = time.time()

    original_img = cv2.imread(args.img_path, 1)

    shape = original_img.shape
    img = np.float32(original_img) / 255

    gt_category = args.true_class

    # Convert to torch variables
    img = preprocess_image(img, args.size)

    if use_cuda:
        img = img.to('cuda')

    # Path to the output folder
    save_path = os.path.join(args.save_path, '{}'.format(args.algo), '{}'.format(args.dataset))
    mkdir_p(save_path)

    # save original image
    # Compute original output
    org_softmax = torch.nn.Softmax(dim=1)(model(img))
    eval0 = org_softmax.data[0, gt_category]
    pill_transf = get_pil_transform()
    cv2.imwrite(os.path.abspath(os.path.join(save_path, 'real_{}_{:.3f}_image.jpg'
                             .format(label_map[gt_category].split(',')[0].split(' ')[0].split('-')[0], eval0))),
                cv2.cvtColor(np.array(pill_transf(get_image(args.img_path))), cv2.COLOR_BGR2RGB))

    t1 = time.time()
    with torch.no_grad():
        # Occlusion class
        heatmap_occ = occlusion_analysis(img, net=model, num_classes=1000, img_size=args.size,
                                         batch_size=batch_size, org_shape=shape)
        for stride in [args.stride]:
            for p_size in [args.patch_size]:
                heatmap = heatmap_occ.explain(neuron=gt_category, loader=trainloader,
                                              heatmap_type=args.algo, path=save_path, l_map=label_map)
                np.save(
                    os.path.abspath(os.path.join(save_path, 'mask_{}.npy'.format(args.algo))),
                    heatmap)

    # print('Time taken: {:.3f}'.format(time.time() - init_time))

reason: import numpy


text:
import numpy as np
import pandas as pd

def transform_returns(df, alpha=1.0):
    '''Extracts weighted returns data from
    CRSP raw data.

    Parameters
    ----------
    df : array-like

    alpha : float, optional default=1.0
        Real number to multiply weighted returns by.

    Returns
    -------
    Pandas dataframe containing weighted returns.
    '''
    # Pivot the raw data since assets are stacked
    df = pd.pivot(df,
                  values = ['BIDLO', 'ASKHI', 'VOL', 'RETX'],
                  index='date', columns = 'PERMNO'
                 )

    # Infers objects; those that can't be inferred are
    # converted to `NaN`
    df = df.convert_objects(convert_numeric=True)

    # Replace all missing entries with '0'
    df.replace('NaN',0)

    # Functions for `vectorize` to apply elementwise to
    # pairs of elements from columns
    average = lambda x, y : ((x + y) / 2.0)

    multiply = lambda x, y : (x * y)

    divide = lambda x, y : (x / y)

    permnos = df['RETX'].columns.tolist()

    for permno in permnos:
        # Step 1 - Calculate midprice
        df[('MIDPRCE', permno)] = np.vectorize(average)(df[('ASKHI', permno)],
                                                        df[('BIDLO', permno)])

        # Step 2 - Calculate midprice dollar-value traded
        df[('DLRTRDED', permno)] = np.vectorize(multiply)(df[('MIDPRCE', permno)],
                                                          df[('VOL', permno)])

    # Sum total dollar-value traded by day
    df['TTLDLRTRDED'] = df['DLRTRDED'].sum(axis=1)

    # Drop indexes containing '0' entries in 'TTLDLRTRDED'
    # These are days with no market activity
    df.drop(df.index[df['TTLDLRTRDED'] == 0], inplace=True)

    for permno in permnos:
        # Step 3 - Calculate the percentage of total dollar-value traded
        df[('PRCNTDLRTRDED', permno)] = np.vectorize(divide)(df[('DLRTRDED', permno)],
                                                             df['TTLDLRTRDED'])

        # Step 4 - Calculate weighted return
        df[('WGHTEDRETX', permno)] = alpha * np.vectorize(multiply)(
            df[('PRCNTDLRTRDED', permno)], df[('RETX', permno)])

    # Sort dataframe by PERMNO
    df = df.sort_index(axis=1, level=1)

    # Only return weighted returns
    X = df['WGHTEDRETX']

    return X


text:
import boto3
import os
import numpy as np
import os
import subprocess
from botocore.exceptions import ClientError

def getFramerate(video):
    con = 'ffprobe -v error -select_streams v:0 -show_entries stream=avg_frame_rate -of default=noprint_wrappers=1:nokey=1 "' + video + '"'
    proc = subprocess.Popen(con, stdout=subprocess.PIPE, shell=True)
    framerateString = str(proc.stdout.read())[2:-3]
    a = int(framerateString.split('/')[int(0)])
    b = int(framerateString.split('/')[int(1)])
    return int(np.round(np.divide(a,b)))

def processVideo(args):
    con = 'python video_to_slomo.py --ffmpeg ' + args["ffmpeg"] + ' --video ' + args["video"] + ' --sf ' + args["sf"] + \
          ' --checkpoint ' + args["checkpoint"] + ' --fps ' + args["fps"]  + ' --output ' + args["output"]
    print(con)
    proc = subprocess.Popen(con, stdout=subprocess.PIPE, shell=True)
    print(proc.stdout.read())

def create_presigned_url(bucket_name, object_name, expiration=36000):
    # Generate a presigned URL for the S3 object
    s3_client = boto3.client('s3', config=boto3.session.Config(signature_version='s3v4'), region_name='eu-west-2')
    try:
        response = s3_client.generate_presigned_url('get_object',
                                                    Params={'Bucket': bucket_name,
                                                            'Key': object_name},
                                                    ExpiresIn=expiration)
    except ClientError as e:
        logging.error(e)
        return None

    # The response contains the presigned URL
    return response

def sendEmail(email, downloadbucket, video):
    # Replace sender@example.com with your "From" address.
    # This address must be verified with Amazon SES.
    SENDER = "Slomotatron <slomo@jcws.co.uk>"

    download_link = create_presigned_url(downloadbucket, video)

    # Replace recipient@example.com with a "To" address. If your account
    # is still in the sandbox, this address must be verified.
    RECIPIENT = email

    # If necessary, replace us-west-2 with the AWS Region you're using for Amazon SES.
    AWS_REGION = "eu-west-1"

    # The subject line for the email.
    SUBJECT = "Your super slomo video"

    # The email body for recipients with non-HTML email clients.
    BODY_TEXT = ("Slomotatron has processed your video\r\n" +
                "donwload your video from the link below\r\n" + download_link
                )

    # The HTML body of the email.
    BODY_HTML = """

Slomotatron

Slomotatron has processed your video, download it using the following link
        """+ video[video.rfind("/")+1:] + """

    """
    # The character encoding for the email.
    CHARSET = "UTF-8"

    # Create a new SES resource and specify a region.
    client = boto3.client('ses',region_name=AWS_REGION)

    # Try to send the email.
    try:
        #Provide the contents of the email.
        response = client.send_email(
            Destination={
                'ToAddresses': [
                    RECIPIENT,
                ],
            },
            Message={
                'Body': {
                    'Html': {
                        'Charset': CHARSET,
                        'Data': BODY_HTML,
                    },
                    'Text': {
                        'Charset': CHARSET,
                        'Data': BODY_TEXT,
                    },
                },
                'Subject': {
                    'Charset': CHARSET,
                    'Data': SUBJECT,
                },
            },
            Source=SENDER,
            # If you are not using a configuration set, comment or delete the
            # following line
            # ConfigurationSetName=CONFIGURATION_SET,
        )
    # Display an error if something goes wrong.
    except ClientError as e:
        print(e.response['Error']['Message'])
    else:
        print("Email sent! Message ID:"),
        print(response['MessageId'])

# Create SQS client
sqs = boto3.client('sqs', region_name='eu-west-2')
s3 = boto3.client('s3', region_name='eu-west-2')

queue_url = 'https://sqs.eu-west-2.amazonaws.com/839229338431/dev-slomo.fifo'
s3attachmentsBucket = 'slomo-app-api-dev-uploadsbucket-ko2jzrics82r'
s3downloadBucket = 'slomo-app-api-dev-downloadsbucket-t4wypnf2t36r'
videoInputFolder = os.path.join('..', 'input')

while 1:
    # Receive message from SQS queue
    response = sqs.receive_message(
        QueueUrl=queue_url,
        AttributeNames=[
            'SentTimestamp'
        ],
        MaxNumberOfMessages=1,
        MessageAttributeNames=[
            'All'
        ],
        VisibilityTimeout=3600,
        WaitTimeSeconds=10
    )

    if 'Messages' in response:
        message = response['Messages'][0]
        messageAttrbibutes = message['MessageAttributes']
        uploadAttachment = messageAttrbibutes['uploadAttachment']['StringValue']
        savePath = os.path.join(videoInputFolder, uploadAttachment)
        userId = messageAttrbibutes['userId']['StringValue']
        email = messageAttrbibutes['email']['StringValue']
        model = messageAttrbibutes['aiModel']['StringValue']
        slomoFactor = messageAttrbibutes['slomoFactor']['StringValue']
        receipt_handle = message['ReceiptHandle']
        #download attachment
        attachmentPath = "private/" + userId + "/" + uploadAttachment
        s3.download_file(s3attachmentsBucket, attachmentPath, savePath)
        #get fps of file using ffmpeg

        framerate = getFramerate(savePath)
        #process file to slow motion
        outputFile = os.path.join("..", "output", uploadAttachment)
        args = {
            "ffmpeg": "/usr/bin",
            "video": '"' + savePath + '"',
            "sf": str(slomoFactor),
            "checkpoint": "checkpoints/" + model + ".ckpt",
            "fps": str(framerate),
            "output": '"' + outputFile + '"',
        }

        result = processVideo(args)

        #copy file to download folder
        s3.upload_file(outputFile, s3downloadBucket, attachmentPath)
        #generate email to client
        sendEmail(email, s3downloadBucket, attachmentPath)

        # Delete received message from queue
        sqs.delete_message(
            QueueUrl=queue_url,
            ReceiptHandle=receipt_handle
        )
        print('Received and deleted message: %s' % message)
    else:
        print('no messages')

max_stars_repo_name: Jimcumming/Super-SloMo

max_issues_repo_name: Jimcumming/Super-SloMo

max_forks_repo_name: Jimcumming/Super-SloMo


text:
import numpy as np
from ._ReadMSM import _ReadMSM
import KT17 as kt17
from ..Pos.GetAberrationAngle import GetAberrationAngle
from ._MSM2Dipolar import _MSM2Dipolar
from ..Pos.GetMercuryPos import GetMercuryPos
from . import MagGlobals

def _Dip(Date,Minute=False):
	'''
	This procedure should return magnetometer data rotated into
	a coordinate system useful for studying waves, with components in
	the poloidal, toroidal and parallel directions.

	Inputs:
		Date: 32-bit integer containing the date with the format
			yyyymmdd.
		Minute: Set to True to use minute resolution data, or False for
			full time resolution data.

	Returns:
		numpy.recarray
	'''
	tmp = GetMercuryPos(Date)
	Rau = tmp.r
	ModelParams = [Rau,50.0]
	dtype = MagGlobals.dtypes['Dip']

	data = _ReadMSM(Date,Minute=Minute,res=None,Ab=None,DetectGaps=None)

	if data.size == 0:
		print('No MSM data found for {:d}'.format(Date))
		return np.recarray(0,dtype=dtype)

	t = data.ut
	#data already rotated into aberrated corrdinates
	b1,b2,b3 = data.Bx,data.By,data.Bz
	p1,p2,p3 = data.Xmsm,data.Ymsm,data.Zmsm

	#now to get model field vectors in aberrated coordinates
	#(no need to aberrate them again) including MP check
	m1,m2,m3 = kt17.ModelField(p1,p2,p3,Params=ModelParams)

	#rotate data vectors, then model vectors, then subtract parallel component
	tx,ty,tz = _MSM2Dipolar(b1,b2,b3,m1,m2,m3,p1,p2,p3)
	ttx,tty,ttz = _MSM2Dipolar(m1,m2,m3,m1,m2,m3,p1,p2,p3)
	tz-=ttz

	b1 = tx
	b2 = ty
	b3 = tz

	#returned coordinates will be in aberrated coords
	# Bx,By,Bz == Bpol, Btor, Bpar

	out= np.recarray(t.size,dtype=dtype)

	out.Date = data.Date
	out.ut = data.ut
	out.utc = data.utc
	out.Bpol = b1
	out.Btor = b2
	out.Bpar = b3
	out.Xmsm = p1
	out.Ymsm = p2
	out.Zmsm = p3

	return out


text:
from typing import Callable, Dict, Optional, Type, TypeVar

import numpy as np
import torch
import torch.nn.functional as F
from torch import nn, optim
from torch.utils.data import Dataset, TensorDataset

T = TypeVar("T", Type[nn.Module], Callable[..., torch.Tensor])

activation_map: Dict[str, Dict[str, T]] = {
    "nn": {
        "ReLU": nn.ReLU,
        "Tanh": nn.Tanh,
        "Sigmoid": nn.Sigmoid,
    },
    "f": {
        "ReLU": F.relu,
        "Tanh": F.tanh,
        "Sigmoid": F.sigmoid,
    },
}

optimizer_mapping: Dict[str, Type[optim.Optimizer]] = {
    "Adam": optim.Adam,
    "Adadelta": optim.Adadelta,
    "rmsprop": optim.RMSprop,
}

convolution_mapping: Dict[str, Type[nn.modules.conv._ConvNd]] = {
    "Conv1D": nn.Conv1d,
    "Conv2D": nn.Conv2d,
    "Conv3D": nn.Conv3d,
}

pooling_map: Dict[str, Dict[str, T]] = {
    "nn": {
        "Max1D": nn.MaxPool1d,
        "Max2D": nn.MaxPool2d,
        "Max3D": nn.MaxPool3d,
        "Average1D": nn.AvgPool1d,
        "Average2D": nn.AvgPool2d,
        "Average3D": nn.AvgPool3d,
    },
    "f": {
        "Max1D": F.max_pool1d,
        "Max2D": F.max_pool2d,
        "Max3D": F.max_pool3d,
        "Average1D": F.avg_pool1d,
        "Average2D": F.avg_pool2d,
        "Average3D": F.avg_pool3d,
    },
}

upsampling_fn_map = {
    # TODO: PyTorch에는 Upsample이라는 class로 1~4D까지 처리한다.
    "UpSampling1D": "tf.keras.layers.UpSampling1D",
    "UpSampling2D": "tf.keras.layers.UpSampling2D",
    "UpSampling3D": "tf.keras.layers.UpSampling3D",
}

device = "cuda" if torch.cuda.is_available() else "cpu"

def createNumpyDataset(x: np.array, y: Optional[np.array] = None) -> Dataset:
    x_tensor = torch.Tensor(x).float()
    if y is not None:
        y_tensor = torch.Tensor(y).float()
        return TensorDataset(x_tensor, y_tensor)
    else:
        return TensorDataset(x_tensor)

max_stars_repo_name: seculayer/automl-mlps

max_issues_repo_name: seculayer/automl-mlps

max_forks_repo_name: seculayer/AutoAPE-mlps


text:
#! /usr/bin/env python
import argparse, os, sys, errno
import logging
logging.basicConfig(level=logging.DEBUG, format='[%(asctime)s] [%(levelname)s] %(name)s: %(message)s')
logger = logging.getLogger('create_cv_folds')

def prepare_output_file(filename):
    try:
        os.makedirs(os.path.dirname(filename))
    except OSError as e:
        if e.errno != errno.EEXIST:
            raise e

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-n', '--n-samples', type=int, required=True,
        help='number of samples')
    parser.add_argument('-o', '--output-file', type=str, required=True,
        help='prefix for output file names')
    parser.add_argument('-k', type=int, default=5)
    args = parser.parse_args()

    from sklearn.model_selection import KFold
    import numpy as np
    import h5py

    prepare_output_file(args.output_file)
    fout = h5py.File(args.output_file, 'w')
    i = 0
    kfold = KFold(args.k, shuffle=True)
    for train_index, test_index in kfold.split(np.arange(args.n_samples)):
        g = fout.create_group(str(i))
        g.create_dataset('train', data=train_index)
        g.create_dataset('test', data=test_index)
        i += 1
    fout.close()

max_stars_repo_name: ltbyshi/cardiacai

max_issues_repo_name: ltbyshi/cardiacai

max_forks_repo_name: ltbyshi/cardiacai


text:
# test.py ---
#
# Description:
# Author: Goncalo Pais
# Date: 28 Jun 2019
# https://arxiv.org/abs/1904.01701
#
# Instituto Superior Técnico (IST)

# Code:

import datetime
import numpy as np
import time
from transformations import rotation_from_matrix
import sklearn.metrics as metrics
import pickle
from registration.registration import initializePointCoud, globalRegistration, selectFunction, refine_registration, makeMatchesSet
# from registration import preProcessData, globalRegistration, selectFunction, refine_registration

def saveToFile(dict):

    f = open('results/test.pickle', 'wb')
    pickle.dump(dict, f)
    f.close()

def RotationError(R1, R2):

    R1 = np.real(R1)
    R2 = np.real(R2)

    R_ = np.matmul(R1, np.linalg.inv(R2))
    ae = np.arccos((np.trace(R_) - 1) / 2)

    ae = np.rad2deg(ae)

    frob_norm = np.linalg.norm(R_ - np.eye(3), ord='fro')

    return ae, frob_norm

def TranslationError(t1, t2):

    return np.linalg.norm(t1-t2)

def test_process(mode, sess, data, cur_global_step, summary_writer, config, xin1, xin2, Rin, tin, fin, is_training,
                 Rhat, that, logits, loss, weights, reg_flag, reg_function):

    import tensorflow as tf

    if mode == 'test':
        print("[{}] {}: Start testing\n".format(config.data_te, time.asctime()))

    if mode == 'valid':
        print("[{}] {}: Start validating\n".format(config.data_te, time.asctime()))

    # ----------------------------------------
    # Unpack some data for simple coding
    x1 = data['x1']
    x2 = data['x2']
    Rs = data['R']
    ts = data['t']
    fs = data['flag']

    num_samples = len(x1)

    R_hats = []
    t_hats = []
    flags = []
    delta = []
    cls_logs = []
    cls_weights = []
    losses = []
    skews = []
    precisions = []
    accuracies = []

    per = 1
    refine = False

    for idx_cur in range(num_samples):

        # Actual construction of the batch
        x1_b = np.array(x1[idx_cur]).reshape((1, 1, -1, 3))
        x1_b = x1_b[:, :, :int(x1_b.shape[2] * per), :]
        x2_b = np.array(x2[idx_cur]).reshape((1, 1, -1, 3))
        x2_b = x2_b[:, :, :int(x2_b.shape[2] * per), :]

        Rs_b = np.array(Rs[idx_cur]).reshape((1, 9))
        ts_b = np.array(ts[idx_cur]).reshape((1, 3))
        fs_b = np.array(fs[idx_cur]).reshape((1,-1))
        fs_b = fs_b[:, :int(fs_b.shape[1] * per)]

        # Feed Dict
        feed_dict = {
            xin1: x1_b,
            xin2: x2_b,
            Rin: Rs_b,
            tin: ts_b,
            fin: fs_b,
            is_training: False,
        }
        fetch = {
            "R_hat": Rhat,
            "t_hat": that,
            "logits": logits,
            "weights": weights,
            "loss": loss,
        }

        time_start = time.time()
        res = sess.run(fetch, feed_dict=feed_dict)
        time_end = time.time() - time_start

        delta.append(time_end)
        if mode == 'test':
            print("[{}] Detection time - {}".format(idx_cur, delta[idx_cur]))

        R_hats.append(res['R_hat'])
        t_hats.append(res['t_hat'])
        cls_logs.append(res['logits'])
        cls_weights.append(res['weights'])
        losses.append(res['loss'])

        log_pos = np.array(res['logits'] > 0, dtype=np.uint8)
        flags.append(log_pos.astype(bool))

        accuracies.append(metrics.accuracy_score(fs_b[0], log_pos[0]))

    t_errors = []
    r_errors = []
    frob_errors = []
    unused_idx = []

    els = []

    for idx_cur in range(num_samples):

        if reg_flag:

            pts1 = initializePointCoud(x1[idx_cur])
            pts2 = initializePointCoud(x2[idx_cur])

            reg_fun = selectFunction(reg_function)

            if np.asarray(pts1.points).size > 0:

                result, elapsed, pts1_down, pts2_down = globalRegistration(pts1, pts2, reg_fun)
                print('[{}] {} registration took: {}'.format(idx_cur, reg_function, elapsed))
                delta[idx_cur] = elapsed
                e = elapsed

                if reg_function == 'global' and refine:
                    print()
                    transformation, elapsed = refine_registration(pts1_down, pts2_down, result.correspondence_set)
                    print('[{}] {} refinement took: {}'.format(idx_cur, 'Umeyama', elapsed))
                    delta[idx_cur] = delta[idx_cur] + elapsed
                    elapsed = e + elapsed

                    els.append(elapsed)
                    R_hat = transformation[:3,:3]
                    t_hat = transformation[:3, 3]

                else:

                    els.append(elapsed)
                    R_hat = result.transformation[:3, :3]
                    t_hat = result.transformation[:3, 3]

            else:
                print('[{}] IDX registered'.format(idx_cur))
                unused_idx.append(idx_cur)
                R_hat = R_hats[idx_cur].reshape(3, 3)
                t_hat = t_hats[idx_cur]

        else:

            if refine:

                pts1 = initializePointCoud(x1[idx_cur])
                pts2 = initializePointCoud(x2[idx_cur])

                transformation = np.eye(4)
                transformation[:3, :3] = R_hats[idx_cur].reshape(3, 3)
                transformation[:3,  3] = t_hats[idx_cur]

                set = makeMatchesSet(flags[idx_cur])

                transformation, elapsed = refine_registration(pts1, pts2, set)
                print('[{}] {} refinement took: {}'.format(idx_cur, 'Umeyama', elapsed))
                delta[idx_cur] = delta[idx_cur] + elapsed
                els.append(elapsed)

                R_hat = transformation[:3, :3]
                t_hat = transformation[:3, 3]

            else:

                R_hat = R_hats[idx_cur].reshape(3, 3)
                t_hat = t_hats[idx_cur]

                if config.representation == 'linear':
                    u, s, vh = np.linalg.svd(R_hat, compute_uv=True)
                    R_hat = np.matmul(u, vh)
                    R_hat = np.linalg.det(R_hat) * R_hat

        R_gt = Rs[idx_cur].reshape(3,3)
        t_gt = ts[idx_cur]

        rot_error, frob_error = RotationError(R_gt, R_hat)
        r_errors.append(rot_error)
        frob_errors.append(frob_error)
        t_errors.append(TranslationError(t_hat, t_gt))

    mean_rotation_error = np.mean(np.abs(r_errors))
    mean_translation_error = np.mean(np.abs(t_errors))
    median_rotation_error = np.median(np.abs(r_errors))
    median_translation_error = np.median(np.abs(t_errors))
    mean_frob_error = np.median(np.abs(frob_errors))
    mean_losses = np.mean(np.abs(losses))
    mean_accuracies = np.mean(accuracies)
    mean_deltas = np.mean(delta)

    indx_min = np.argmin(np.abs(r_errors))

    summaries = []

    summaries += [
        tf.Summary.Value(tag="ErrorComputation/mean_rotation_error_{}".format(mode),
            simple_value=mean_rotation_error)]
    summaries += [
        tf.Summary.Value(tag="ErrorComputation/mean_translation_error_{}".format(mode),
            simple_value=mean_translation_error)]
    summaries += [
        tf.Summary.Value(tag="ErrorComputation/mean_frob_error_{}".format(mode),
            simple_value=mean_frob_error)]
    summaries += [
        tf.Summary.Value(tag="ErrorComputation/mean_losses_{}".format(mode),
            simple_value=mean_losses)]
    summaries += [
        tf.Summary.Value(tag="ErrorComputation/mean_accuracies_{}".format(mode),
                         simple_value=mean_accuracies)]

    summary_writer.add_summary(
        tf.Summary(value=summaries), global_step=cur_global_step)

    if mode == 'test':
    # r_errors = np.unwrap(r_errors)
        print('Rotation mean error (deg) - {}'.format(mean_rotation_error))
        print('Translation mean error (m) - {}'.format(mean_translation_error))
        print('Rotation median error (deg) - {}'.format(median_rotation_error))
        print('Translation median error (m) - {}'.format(median_translation_error))
        print('Frob norm mean error (deg) - {}'.format(mean_frob_error))
        print('Total Loss error - {}'.format(mean_losses))
        print('Accuracies - {}'.format(mean_accuracies))
        print('Mean delta - {}'.format(mean_deltas))

        if els:
            print('Mean elapsed - {}'.format(np.mean(els)))

        print('Idx min: {} \tRot_min = {}'.format(indx_min, np.abs(r_errors[indx_min])))

        dict = {'RotationError':np.abs(r_errors),'TranslationError': np.abs(t_errors), 'FrobError': frob_errors,
                'TotalLoss': losses, 'Accuracies': accuracies, 'Delta': delta, 'R_gt': Rs, 't_gt': ts, 'x1': x1,
                'x2': x2, 'flag':flags, 'Rhat': R_hats, 'that': t_hats}
        # saveToFile(dict)

    return mean_frob_error


text:
import struct
import numpy

import GLWindow
import ModernGL

# This example is not working with NPOT Textures

pixels = numpy.round(numpy.random.rand(512, 512)).astype('float32')
grid = numpy.dstack(numpy.mgrid[0:512, 0:512][::-1] / 512).astype('float32')

wnd = GLWindow.create_window(512, 512)
ctx = ModernGL.create_context()

prog = ctx.program(
    ctx.vertex_shader('''
        #version 330

        in vec2 vert;
        out vec2 text;

        void main() {
            text = vert;
            gl_Position = vec4((vert * 2.0 - 1.0) * 1, 0.0, 1.0);
        }
    '''),
    ctx.fragment_shader('''
        #version 330

        uniform sampler2D Texture;

        in vec2 text;
        out vec4 color;

        void main() {
            color = texture(Texture, text);
        }
    '''),
])

trans = ctx.program(
    ctx.vertex_shader('''
        #version 330

        uniform sampler2D Texture;

        in vec2 text;
        out float vert;

        #define LIVING 0.0
        #define DEAD 1.0

        #define DX 1.0 / 512.0
        #define DY 1.0 / 512.0

        void main() {
            bool living = texture(Texture, text).r < 0.5;

            int neighbours = 0;
            if (texture(Texture, text + vec2(-DX, -DY)).r < 0.5) neighbours++;
            if (texture(Texture, text + vec2(-DX, 0.0)).r < 0.5) neighbours++;
            if (texture(Texture, text + vec2(-DX, DY)).r < 0.5) neighbours++;
            if (texture(Texture, text + vec2(DX, -DY)).r < 0.5) neighbours++;
            if (texture(Texture, text + vec2(DX, 0.0)).r < 0.5) neighbours++;
            if (texture(Texture, text + vec2(DX, DY)).r < 0.5) neighbours++;
            if (texture(Texture, text + vec2(0.0, DY)).r < 0.5) neighbours++;
            if (texture(Texture, text + vec2(0.0, -DY)).r < 0.5) neighbours++;

            if (living) {
                vert = (neighbours == 2 || neighbours == 3) ? LIVING : DEAD;
            } else {
                vert = (neighbours == 3) ? LIVING : DEAD;
            }
        }
    '''),
    varyings=['vert']
)

texture = ctx.texture((512, 512), 1, pixels.tobytes(), floats=True)
texture.filter = ModernGL.NEAREST
texture.swizzle = 'RRR1'
texture.use()

vbo = ctx.buffer(struct.pack('8f', 0, 0, 0, 1, 1, 0, 1, 1))
vao = ctx.simple_vertex_array(prog, vbo, ['vert'])

text = ctx.buffer(grid.tobytes())
tao = ctx.simple_vertex_array(trans, text, ['text'])
pbo = ctx.buffer(reserve=pixels.nbytes)

while wnd.update():
    ctx.viewport = wnd.viewport

    tao.transform(pbo)
    texture.write(pbo)

    vao.render(ModernGL.TRIANGLE_STRIP)

max_stars_repo_name: asnt/moderngl

max_issues_repo_name: asnt/moderngl

max_forks_repo_name: asnt/moderngl


text:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Apr 22 15:48:30 2020

@author: David Chong Tian Wei
"""

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import random
from ventiliser.BreathVariables import BreathVariables

class Evaluation:
    """
    Class to help visualise and evaluate breaths extracted from a record.

    Attributes
    ----------
    pressures : Array like of real
        Pressure data points for a record
    flows : Array like of real
        Flow data points for a record
    breaths : Array like of BreathVariables
        Breaths as calculating using mapper and phaselabeller
    freq : real
        Frequency in Hz of the recording sampling rate

    """
    @staticmethod
    def load_breaths_from_csv(path):
        """
        Utility method to load a csv file output from PhaseLabeller.get_breaths_raw to a list of BreathVariables

        Parameters
        ----------
        path : string
            Path to the raw breaths file

        Returns
        -------
        array like of BreathVariables objects
        """
        csv = pd.read_csv(path)
        breaths = csv.apply(Evaluation.__breathvariables_from_row, axis=1)
        return breaths
    @staticmethod
    def __breathvariables_from_row(x):
        """
        Helper method to apply on pandas dataframe

        Parameters
        ----------
        x : Pandas Series
            A row from a pandas dataframe containing breaths

        Returns
        -------
        BreathVariables object
        """
        output = BreathVariables()
        for attr in x.index:
            setattr(output, attr, int(x[attr]))
        return output

    def __init__(self, pressures, flows, breaths, freq):
        """
        Initialises the evaluation object with the data, predicted breaths, and frequency of the record

        Parameters
        ----------
        pressures : Array like of real
            Pressure data points for a record
        flows : Array like of real
            Flow data points for a record
        breaths : Array like of BreathVariables
            Breaths as calculating using mapper and phaselabeller
        freq : real
            Frequency in Hz of the recording sampling rate

        Returns
        -------
        None
        """

        self.pressures = np.array(pressures)
        self.flows = np.array(flows)
        self.breaths = breaths
        self.freq = freq

    def compare(self, labels, breath_attr):
        """
        Compares an attribute from the currently loaded breaths with a list of values. Identifies the label which is the closest match to each breath.

        Parameters
        ----------
        labels : array like of int
            A list of indices that you wish to compare with the currently loaded breaths
        breath_attr : string
            A BreathVariables attribute you wish to perform the comparison on

        Returns
        -------
        Pandas Dataframe
            A dataframe containing the closest matching breath to the given labels based on the attribute along with the difference
        """
        if self.breaths is None:
            print("No breaths to compare to")
            return
        labels = np.array(labels)
        output = []
        for breath in self.breaths:
            delta = abs(labels - getattr(breath, breath_attr))
            best = np.argmin(np.array(delta))
            output += [{"breath_index" : self.breaths.index(breath), "label_index" : best, "delta" : delta[best]}]
        return pd.DataFrame(output)

    def get_comparison_stats(self, labels, breath_attr):
        """
        Compares labels against an attribute of the loaded breaths. If multiple breaths map to the same label, the one with the smallest delta is kept and the remaining considered false positives.

        Parameters
        ----------
        labels : array like of int
            A list of indices that you wish to compare with the currently loaded breaths
        breath_attr : string
            A BreathVariables attribute you wish to perform the comparison on

        Returns
        -------
        dict
            A dictionary containing total number of breaths, number of matched breaths, total number of labels, number of matched labels, mean delta of all matches, std delta of all matches
        """
        comparison = self.compare(labels, breath_attr)
        mapped_labels = comparison["label_index"].unique()
        matches = []
        for label in mapped_labels:
            mapped_breaths = comparison.loc[comparison["label_index"] == label].sort_values(by=["delta"], ascending=True)
            matches += [mapped_breaths["delta"].iat[0]]
        return {"n_predicted" : len(self.breaths),
                  "n_predicted_matched" : len(matches),
                  "n_labels" : len(labels),
                  "n_labels_matched" : len(mapped_labels),
                  "delta_mean" : np.mean(matches),
                  "delta_std" : np.std(matches)}

    def plot_breath(self, index, attrs=[], p_y="Pressure", f_y="Flow", v_y="Volume", show=True):
        """
        Plots the pressure, volume and flow of a breath and allows labelling of key points

        Parameters
        ----------
        index : int
            Index of the breath you wish to plot
        attrs : array like of string, optional
            List of attributes you wish to annotate onto the breath (See BreathVariables for available attributes). Defaults to empty list
        p_y : string, optional
            Label for pressure plot y axis. Defaults to 'Pressure'.
        f_y : string, optional
            Label for flow plot y axis. Defaults to 'Flow'
        v_y : string, optional
            Label for volume plot y axis. Defaults to 'Volume'
        show : boolean, optional
            Flag to decide whether to show the plot or leave it so the user can add more to it. Defaults to True

        Returns
        -------
        None
        """
        if self.breaths is None:
            print("No breaths loaded")
        if index >= len(self.breaths):
            print("Index is larger than the number of breaths available")

        volume = [0]
        for f in self.flows[self.breaths[index].breath_start:self.breaths[index].breath_end]:
            volume.append(volume[-1] + f)

        x_values = np.array(range(0,self.breaths[index].breath_end - self.breaths[index].breath_start)) / self.freq
        plt.figure(figsize=(10,6))
        # Plot pressures
        plt.subplot(3,1,1)
        plt.title("Breath " + str(index))
        plt.plot(x_values,self.pressures[self.breaths[index].breath_start:self.breaths[index].breath_end], color="r")
        plt.ylim(bottom=0)
        plt.ylabel(p_y)
        for attr in attrs:
            xpos = x_values[getattr(self.breaths[index], attr) - self.breaths[index].breath_start]
            plt.axvline(x=xpos, color="k", linestyle="--")
            plt.text(xpos, random.uniform(plt.ylim()[0] * 0.9, plt.ylim()[1] * 0.9), attr)
        # Plot flows
        plt.subplot(3,1,2)
        plt.plot(x_values, self.flows[self.breaths[index].breath_start:self.breaths[index].breath_end], color="g")
        plt.axhline(y=0, color="k", linestyle="--")
        plt.ylabel(f_y)
        for attr in attrs:
            xpos = x_values[getattr(self.breaths[index], attr) - self.breaths[index].breath_start]
            plt.axvline(x=xpos, color="k", linestyle="--")
            plt.text(xpos, random.uniform(plt.ylim()[0] * 0.9, plt.ylim()[1] * 0.9), attr)
        # Plot volume
        plt.subplot(3,1,3)
        plt.plot(x_values, volume[1:], color="b")
        plt.xlabel("Time [s]")
        plt.ylabel(v_y)
        for attr in attrs:
            xpos = x_values[getattr(self.breaths[index], attr) - self.breaths[index].breath_start]
            plt.axvline(x=xpos, color="k", linestyle="--")
            plt.text(xpos, random.uniform(plt.ylim()[0] * 0.9, plt.ylim()[1] * 0.9), attr)

        plt.tight_layout()
        if show:
            plt.show()

    def plot_pv_loop(self, index, p_label="Pressure", v_label="Volume", show=True):
        """
        Plots the pressure-volume loop of the breath at index

        Parameters
        ----------
        index : int
            The index of the breath you wish to plot
        p_label : string, optional
            The label for the pressure axis. Defaults to 'Pressure'
        v_label : string, optional
            The label for the volume axis. Defaults to 'Volume'
        show : boolean, optional
            Flag for deciding whether to show the plot or leave it so the user can add more to it. Defaults to True

        Returns
        -------
        None
        """
        if self.breaths is None:
            print("No breaths loaded")
        if index >= len(self.breaths):
            print("Index is larger than the number of breaths available")
        volume = [0]
        for f in self.flows[self.breaths[index].breath_start:self.breaths[index].breath_end]:
            volume.append(volume[-1] + f)
        plt.plot(self.pressures[self.breaths[index].breath_start:self.breaths[index].breath_end], volume[1:])
        plt.xlabel(p_label)
        plt.ylabel(v_label)
        plt.title("Breath " + str(index))
        if show:
            plt.show()

    def plot_breaths_flow_phases(self,
                                 attrs=["inspiration_initiation_length", "peak_inspiratory_flow_length", "inspiration_termination_length",
                                        "inspiratory_hold_length", "expiration_initiation_length", "peak_expiratory_flow_length",
                                        "expiration_termination_length","expiratory_hold_length"]):
        """
        Plots a stacked barplot of each breath where the stack is the length of each phase of the breath

        Parameters
        ----------
        attrs : array like of string, optional
            List of breath phases you wish to plot out. Defaults to all breath phases (see BreathVariables for more information)

        Returns
        -------
        None
        """
        df = pd.DataFrame([vars(x) for x in self.breaths])
        order = np.argsort(np.array(df["breath_end"] - df["breath_start"]))
        df = df.iloc[order,:]
        bottom = np.array([0] * df.shape[0])
        ind = np.arange(df.shape[0])
        for attr in attrs:
            plt.bar(ind, df[attr], 1, bottom = bottom, label=attr)
            bottom += df[attr].values
        plt.legend()
        plt.xlabel("Breaths")
        plt.ylabel("Length")
        plt.show()

max_stars_repo_name: belteki/ventiliser

max_issues_repo_name: belteki/ventiliser

max_forks_repo_name: belteki/ventiliser