text:
from __future__ import division

import pandas as pd
import numpy as np
# import matplotlib.pyplot as plt
import time
import datetime as dt  # Imports dates library

from nupic.encoders import RandomDistributedScalarEncoder
from nupic.encoders.date import DateEncoder
from nupic.algorithms.spatial_pooler import SpatialPooler
from nupic.algorithms.temporal_memory import TemporalMemory

from scipy.stats import norm

class Encoder:  # Defines an encoder class to put into the variables of the
    def __init__(self, variable, encoders):
        self.variable = variable
        self.encoders = encoders

def multiencode(encoders, Data, iter):
    res = [0]
    for x in encoders:
        for y in x.encoders:
            eval("enc = " + y + ".encode(Data['" + x.variable + "'][iter])", locals(), globals())
            res = np.concatenate([res, enc])
    return res[1:]

# Index must be provided as another column in order to be considered a variable

def HTM_AD(Data='Test',
           vars={'value':['num']},
           prec_param=5,
           pooler_out=2024,  # Number of columns of the pooler output
           cell_col=5,  # HTM cells per column
           W=72,  # Window parameter
           W_prim=5,  # Local window for anomaly detection likelihood
           eps=1e-6,  # to Avoid by zero divisions
           athreshold=0.95):
    """
    This function performs HTM based anomaly detection on a time series provided
    :param Data:
    :param vars: Possible values: num, tod, weekend
    :param prec_param: A parameter that defines how much precision the number encoder has
        The encoder precision depends on the variability of the data,
        The real precision is computed taking into account both the precision parameter and data std
        A high precision might mean a high error at predicting the variable value in noisy variables
    :param pooler_out: Number of columns of the pooler output
    :param cell_col: HTM cells per column
    :param W: Window parameter
    :param W_prim: Local window for anomaly detection likelihood
    :param eps: to Avoid by zero divisions
    :param athreshold: To classify based on anomaly likelihood whether there is an anomaly or not
    :return: The Data + 3 columns
        Anomaly: indicates the error of within the value predicted by the HTM network
        Anomaly_likelihood: indicates the likelihood of the data into being anomalous
        Anomaly_flag: classifies the data in anomalous vs non anomalous
    """

    if Data == 'Test':  # If there is not data available, simply loads the temperature benchmark dataset
        # Import data
        Data = pd.read_csv('AT/Data/sample.csv',
                           parse_dates=True,
                           index_col='timestamp')
        Data = Data.resample('H').bfill().interpolate()

    TODE = DateEncoder(timeOfDay=(21, 1))
    WENDE = DateEncoder(weekend=21)

    var_encoders = set()
    # Spatial Pooler Parameters
    for x in vars:
        for y in vars[x]:
            if y == 'num':
                exec("RDSE_" + x + " = RandomDistributedScalarEncoder(resolution=Data['" + x + "'].std()/prec_param)",
                     locals(), globals())
                var_encoders.add(Encoder(x, ["RDSE_" + x]))
            elif y == 'weekend':
                var_encoders.add(Encoder(x, ["WENDE"]))
            elif y == 'tod':
                var_encoders.add(Encoder(x, ["TODE"]))
            else:
                return {"error": "Variable encoder type is not recognized "}

    encoder_width = 0  # Computes encoder width
    for x in var_encoders:
        for y in x.encoders:
            exec("s = " + y + ".getWidth()", locals(), globals())
            encoder_width += s

    SP = SpatialPooler(inputDimensions=encoder_width,
                       columnDimensions=pooler_out,
                       potentialPct=0.8,
                       globalInhibition=True,
                       numActiveColumnsPerInhArea=pooler_out//50,  # Gets 2% of the total area
                       boostStrength=1.0,
                       wrapAround=False)
    TM = TemporalMemory(columnDimensions=(pooler_out,),
                        cellsPerColumn=cell_col)

    Data['Anomaly'] = 0.0
    Data['Anomaly_Likelihood'] = 0.0

    # Train Spatial Pooler
    print("Spatial pooler learning")

    start = time.time()

    active_columns = np.zeros(pooler_out)

    for x in range(len(Data)):
        encoder = multiencode(var_encoders, Data, x)
        SP.compute(encoder, True, active_columns)

    end = time.time()
    print(end - start)

    # Temporal pooler
    print("Temporal pooler learning")

    start = time.time()

    A_score = np.zeros(len(Data))
    for x in range(len(Data)):
        encoder = multiencode(var_encoders, Data, x)
        SP.compute(encoder, False, active_columns)
        col_index = active_columns.nonzero()[0]
        TM.compute(col_index, learn=True)
        if x > 0:
            inter = set(col_index).intersection(Prev_pred_col)
            inter_l = len(inter)
            active_l = len(col_index)
            A_score[x] = 1 - (inter_l / active_l)
            Data.iat[x, -2] = A_score[x]
        Prev_pred_col = list(set(x // cell_col for x in TM.getPredictiveCells()))

    end = time.time()
    print(end - start)

    AL_score = np.zeros(len(Data))
    # Computes the likelihood of the anomaly
    for x in range(len(Data)):
        if x > 0:
            W_vec = A_score[max(0, x-W): x]
            W_prim_vec = A_score[max(0, x-W_prim): x]
            AL_score[x] = 1 - 2*norm.sf(abs(np.mean(W_vec)-np.mean(W_prim_vec))/max(np.std(W_vec), eps))
            Data.iat[x, -1] = AL_score[x]

    Data['Anomaly_flag'] = athreshold < Data['Anomaly_Likelihood']

    return Data


text:
import pandas as pd
import numpy as np

def renamer(k):
    return k.lower().replace(' ', '_').replace('1st', 'first')

def readaqs(cfg, verbose=1):
    aqspath = cfg['obs_path']
    obs_key = cfg["obs_key"]
    obs_defn = cfg["obs_defn"]
    aqsdf = pd.read_csv(aqspath)
    aqsdf['date'] = pd.to_datetime(aqsdf['Date Local'])
    # aqsdf = aqsdf[
    #     (aqsdf['date'].dt.month == 1) & (aqsdf['date'].dt.day == 15)
    # ].copy()
    # Archive testdata, remove derived columns
    # aqsdf.drop('date', axis=1).to_csv(
    #     'input/daily_88101_20160115.csv', index=False
    # )

    aqsdf.rename(columns=renamer, inplace=True)
    aqsdf.eval(f'{obs_key} = {obs_defn}', inplace=True)
    isnegative = aqsdf.loc[:, obs_key] <= 0
    if isnegative.sum() > 0:
        if verbose > 0:
            print(
                'Removing negative values:',
                isnegative.sum(), np.where(isnegative)
            )
        aqsdf.drop(aqsdf[isnegative].index, inplace=True)
        if verbose > 0:
            print('Done', aqsdf.shape)

    return aqsdf

ext: py

max_stars_repo_name: barronh/cmaqkrig

max_issues_repo_name: barronh/cmaqml

max_forks_repo_name: barronh/cmaqml

reason: import numpy


text:
import json
import pandas as pd
import numpy as np
import pandas_bokeh
import os
import pytest

directory = os.path.dirname(__file__)
os.makedirs(os.path.join(directory, "Plots"), exist_ok=True)

@pytest.fixture
def spark():
    # Start PySpark
    from pyspark.sql import SparkSession
    spark = SparkSession.builder.getOrCreate()
    yield spark
    spark.stop()

def test_basic_lineplot_pyspark(spark):
    """Test for basic lineplot with Pyspark"""

    # Create basic lineplot:
    np.random.seed(42)
    df = pd.DataFrame(
        {"Google": np.random.randn(1000) + 0.2,
         "Apple": np.random.randn(1000) + 0.17},
        index=pd.date_range("1/1/2000", periods=1000),
    )
    df.index.name = "Date"
    df = df.cumsum()
    df = df + 50
    df = spark.createDataFrame(df.reset_index())
    p_basic_lineplot = df.plot_bokeh(kind="line", x="Date", show_figure=False)
    p_basic_lineplot_accessor = df.plot_bokeh.line(x="Date", show_figure=False)

    # Output plot as HTML:
    output = pandas_bokeh.row([p_basic_lineplot, p_basic_lineplot_accessor])
    with open(os.path.join(directory, "Plots", "Basic_lineplot_PySpark.html"), "w") as f:
        f.write(pandas_bokeh.embedded_html(output))

    assert True

max_stars_repo_name: MJuddBooth/Pandas-Bokeh

max_issues_repo_name: MJuddBooth/Pandas-Bokeh

max_forks_repo_name: MJuddBooth/Pandas-Bokeh


text:
import torch
import torch.nn.functional as F
import numpy as np

from src.agent import Agent
from src.metrics import Metrics
from src.model import Actor, Critic
from src.replay import ReplayBuffer, Experience

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

class DDPGConfig:
    def __init__(self,
                 buffer_size=100000,
                 batch_size=256,
                 learning_rate=0.0001,
                 tau=0.001,
                 gamma=0.99,
                 fc1_units=128,
                 fc2_units=128):
        self.buffer_size = buffer_size
        self.batch_size = batch_size
        self.learning_rate = learning_rate
        self.tau = tau
        self.gamma = gamma
        self.fc1_units = fc1_units
        self.fc2_units = fc2_units

class DDPGAgent(Agent):
    def __init__(self, env, config: DDPGConfig):
        super().__init__(env)
        self.config = config
        self.replay_buffer = ReplayBuffer(config.buffer_size, config.batch_size)

        # Actor
        self.actor_current = Actor(env.state_size, env.action_size, config.fc1_units, config.fc2_units).to(device)
        self.actor_target = Actor(env.state_size, env.action_size, config.fc1_units, config.fc2_units).to(device)
        self.actor_optimizer = torch.optim.Adam(self.actor_current.parameters(), lr=config.learning_rate)

        # Critic
        self.critic_current = Critic(env.state_size, env.action_size, config.fc1_units, config.fc2_units).to(device)
        self.critic_target = Critic(env.state_size, env.action_size, config.fc1_units, config.fc2_units).to(device)
        self.critic_optimizer = torch.optim.Adam(self.critic_current.parameters(), lr=config.learning_rate)

        self.metrics = Metrics()

    def restore(self, actor_file, critic_file):
        self.actor_current.load_state_dict(torch.load(actor_file))
        self.critic_current.load_state_dict(torch.load(critic_file))

    def compute_action(self, state, epsilon=0):
        action = self.actor_current.action_values_for(state)
        if np.random.random() < epsilon:
            action += np.random.randn(self.env.action_size) * epsilon
            action = np.clip(action, -1, 1)
        return action

    def train(self, n_steps, update_every, print_every, epsilon_init=1.0, epsilon_decay=0.995, epsilon_min=0.01):
        epsilon = epsilon_init
        state = self._warmup(epsilon)
        self.metrics.plot()

        for t_step in range(1, n_steps + 1):
            state = self._step(state, epsilon)
            epsilon = max(epsilon_min, epsilon * epsilon_decay)

            if t_step % update_every == 0:
                self._batch_train()
                if self._check_solved():
                    break

            if t_step % print_every == 0:
                print(f"Step #{t_step}" +
                      f", Running score {self.metrics.running_score():.2f}" +
                      f", Total episodes {self.metrics.episode_count}")

    def _warmup(self, epsilon):
        state = self.env.reset(train_mode=True)
        needed_experiences = max(0, self.replay_buffer.batch_size - len(self.replay_buffer))
        for i in range(needed_experiences):
            state = self._step(state, epsilon)
        return state

    def _step(self, state, epsilon):
        action = self.compute_action(state, epsilon)
        next_state, reward, done = self.env.step(action)
        self.replay_buffer.add(Experience(state, action, reward, next_state, done))
        self.metrics.on_step(reward, done)
        if done:
            return self.env.reset(train_mode=True)
        return next_state

    def _batch_train(self):
        states, actions, rewards, next_states, dones = self.replay_buffer.sample()

        # Update Critic

        target_actions_next = self.actor_target(next_states)
        target_values_next = self.critic_target(next_states, target_actions_next).detach().max(1)[0].unsqueeze(1)
        target_values = rewards + (self.config.gamma * target_values_next * (1 - dones))
        expected_values = self.critic_current(states, actions)

        critic_loss = F.mse_loss(expected_values, target_values)
        self.critic_optimizer.zero_grad()
        critic_loss.backward()
        self.critic_optimizer.step()
        self.critic_target.soft_update(self.critic_current, self.config.tau)

        # Update Actor

        current_actions = self.actor_current(states)
        actor_loss = -self.critic_current(states, current_actions).mean()
        self.actor_optimizer.zero_grad()
        actor_loss.backward()
        self.actor_optimizer.step()
        self.actor_target.soft_update(self.actor_current, self.config.tau)

    def _check_solved(self):
        if self.metrics.running_score() >= 30:
            print(f"\nEnvironment solved in {self.metrics.episode_count} episodes!\t" +
                  f"Average Score: {self.metrics.running_score():.2f}")
            torch.save(self.actor_current.state_dict(), "actor_model.pt")
            torch.save(self.critic_current.state_dict(), "critic_model.pt")
            return True

        return False

max_stars_repo_name: imed-bh/udacity-drl-continuous-control

max_issues_repo_name: imed-bh/udacity-drl-continuous-control

max_forks_repo_name: imed-bh/udacity-drl-continuous-control


text:
import os
import re
import gensim
import nltk
import numpy as np
import pandas as pd
from gensim.parsing.preprocessing import STOPWORDS
from gensim.utils import simple_preprocess
from nltk.stem import SnowballStemmer, WordNetLemmatizer
from nltk.stem.porter import *
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer

import processor._fix_dictionary as mydictionary

nltk.download('wordnet')

sid_obj = SentimentIntensityAnalyzer()
save_path = 'data\\senti_results\\'

#class
class SentiProcess:
    """
    The class process the raw tweets and define if one is positive or negative
    then it counts the hourly positive, negative and all tweets
    it saved the results by the keyword it was searched from sraper_main
    """
    def __init__(self,key_w):
        self.key_word = key_w
        # load the main dictionary
        self.pos_dic,self.neg_dic = mydictionary.TwitterDict.new_dict()
        # load the pre filter dictionary
        self.pre_dic,self.pre_neg =  mydictionary.TwitterDict.pre_dict()

    def spliter(self,txt):
        """
        Filter out the unrelated $tickers
        """
        spliter = self.key_word.upper()
        txt = spliter+txt.upper().split(spliter)[-1].split("$")[0]
        return txt

    def effective_ttr(self,xfile,thd):
        """
        Filter out the tweets that are effective by our definition
        """
        if len(xfile)==0:return []
        xfile["Datetime"] =pd.to_datetime(xfile.Created)
        xfile["Hour"] = xfile.Datetime.dt.hour
        xfile=xfile.sort_values(by="Datetime")
        # filter the effective user twitter
        is_effec = xfile.User_flr>=thd #using follower number to filter
        # use frequency twitter
        # is_effec = list(map(myfilter.Filter.freq_filter,xfile.User_id))
        x_effc = xfile[is_effec]

        x_effc.index = range(len(x_effc))
        # split the $ sign and get only the key word
        x_effc["sText"] = list(map(self.spliter,x_effc.Text))
        return x_effc

    def get_senti(self,rawtxt):
        """for a sentence, it gives its sentiment
        """
        txt_list= rawtxt.split(" ")
        pos_count,neg_count =0,0
        for i in txt_list:
            if i in self.pos_dic:pos_count+=1
            if i in self.neg_dic:neg_count+=1
        # positive
        if pos_count>neg_count:senti = 1
        elif pos_count<neg_count:senti = -1
        else: senti = 0
        return senti

    def pre_filter(self,rawtxt):
        is_pos = sum([keyw in rawtxt for keyw in self.pre_dic])
        is_neg = sum([keyw in rawtxt for keyw in self.pre_neg])
        # if they are both zero or both not 0, which mean it is vague
        if not is_pos and not is_neg: return 0
        elif is_pos and is_neg:return 0
        # if one of them are zero one is not
        elif is_pos and not is_neg:return 1
        elif is_neg and not is_pos:return -1

    def senti_count(self,e_file,log_flag):
        """count how many positive or negative in a file named e_file
            log_flag: whether or not scale the count into log
        """
        if len(e_file)==0:
            print("file is empty")
            return None
        # initialize the sentiment list for each tweet
        sentis =  []
        for x in e_file.sText:
            #calculcate the compound score of each tweet first
            pre_senti = self.pre_filter(x)
            # let presenti filter first, if zero, use vader
            if pre_senti == 0:
                vader_senti = sid_obj.polarity_scores(x)['compound']
                if vader_senti==0:
                    dict_senti = self.get_senti(x)
                    sentis.append(dict_senti)
                else:
                    sentis.append(vader_senti)
            # if pre filter sentiment result is not zero, use it directly
            else:
                sentis.append(pre_senti)

        #return the sentiment number of each tweet to tweet file
        e_file["Sentiment"] = sentis
        s_file = e_file.copy()
        s_file.index = s_file.Datetime

        # add all sentis get a count
        s_file["All_counts"] = [1]*len(s_file)
        # count the hourly negative or positive ttr
        hour_count = s_file.loc[:,['Sentiment','All_counts']].resample('1H').sum()
        #scale it
        if log_flag:hour_count = np.log(hour_count+1)
        # show positive or negative
        hour_count['Positive'] = hour_count[hour_count.Sentiment>0].Sentiment
        hour_count['Negative'] = hour_count[hour_count.Sentiment<0].Sentiment
        hour_count=hour_count.fillna(0)
        # save negative or positive tweets v5/5/20:
        save_file = s_file.loc[:,["Sentiment","User_flr","Text","User_name"]]
        return hour_count,save_file

    @staticmethod
    def analysis_news(kw_list,filename):
        save_path = 'data\\news\\'
        ifile = pd.read_csv(f'{save_path}{filename}.csv',index_col = 0)
        """There should be index instead of Created
        """
        ifile.index = pd.to_datetime(ifile.loc[:,'Created'])
        ifile["datehour"] = list(map(lambda x:f'{x.date()} {x.hour}',ifile.index))
        hourly_data =  ifile.groupby("datehour")['Text'].apply(lambda x: x.sum()).sort_index()
        # count news numbers
        count = np.zeros(len(hourly_data))
        for kw in kw_list:
            count += np.array(list(map(lambda x:x.upper().count(kw),hourly_data.values)))

        count_series = pd.Series(count,index=hourly_data.index.copy())
        all_sentis = count_series.fillna(0)
        # change from UTC time to EST
        all_sentis.index = pd.to_datetime(list(map(lambda x:x+':00:00',all_sentis.index)))

        return all_sentis

    @staticmethod
    def _utc_to_est(df):
        """It convert the index of dataframe from utc time zone to est time zone
        """
        df["EST"] = [i.tz_localize('UTC').tz_convert('US/Eastern') for i in df.index]
        df.index = list(map(lambda x:x.replace(tzinfo=None),df["EST"]))
        df = df.drop(columns=['EST'])
        return df

    @staticmethod
    def _stemmer(text):
        '''
        get out all the stop words
        stem the word
        '''
        stemmer = PorterStemmer()
        result = []
        for token in gensim.utils.simple_preprocess(text):
            if token not in gensim.parsing.preprocessing.STOPWORDS:
                stemmed = stemmer.stem(WordNetLemmatizer().lemmatize(token, pos='v'))
                result.append(stemmed)
        return result

    @staticmethod
    def _only_letter(text):
        '''
        get only letter, no punctuation
        '''
        return re.sub('[^a-zA-Z]+', ' ', text).split(' ')

    def get_all_senti(self,files,thres,is_log,is_save_senti):
        key_word = self.key_word
        #make other functions use these variables
        dates = [i[-14:-4] for i in files]
        # count sentiments
        all_sentis,all_tweets = pd.DataFrame(),pd.DataFrame()
        for i in range(len(dates)):
            idate = dates[i]
            ifile = files[i]
            xfile=pd.read_csv(ifile,)
            # step 1 filter out all the unqualified ones, if empty return none
            e_file = self.effective_ttr(xfile,thres)
            # if empty, goes to next date file
            if len(e_file)==0:
                print("file is empty for {0}".format(idate))
                continue
            # step 2
            isenti_hourly,itweets_single = self.senti_count(e_file,is_log)
            # add today's senti to all
            all_sentis = pd.concat([all_sentis,isenti_hourly])
            # all_tweets file is the file contains all individual tweets
            all_tweets = pd.concat([all_tweets,itweets_single],axis=0,sort=False)

        all_sentis = all_sentis.replace([np.inf,-np.inf],[np.nan,np.nan])
        # make the index the EST time instead of original twitter UCT time
        # all_sentis.index = pd.to_datetime(all_sentis.index)
        all_sentis = SentiProcess._utc_to_est(all_sentis)
        # if the file is empty, then raise exception
        if len(all_sentis)==0:
            raise Exception('There are not enough sentiments to show.')

        # save the files if necessary 'results/tickername/file.csv'
        if is_save_senti ==1:
            tic_path = f'{save_path}{key_word}\\'
            if not os.path.exists(tic_path):os.makedirs(tic_path)
            # transfer the time zone
            # make the index the EST time instead of original twitter UCT time
            # all_tweets.index = pd.to_datetime(all_tweets.index)
            all_tweets = SentiProcess._utc_to_est(all_tweets)
            all_tweets.to_csv(f'{tic_path}{key_word}_{thres}.csv')
            print("sentiment files are saved successfully")
        # all_sentis is the file that contains all HOURLY sentiment data
        return all_sentis

    def get_accountsinfo(self,files):
        """
        read the raw tweets return the accounts that are interested in this topic
        """
        key_word = self.key_word
        #make other functions use these variables
        dates = [i[-14:-4] for i in files]
        # count sentiments
        all_sentis,all_tweets = pd.DataFrame(),pd.DataFrame()
        for i in range(len(dates)):
            idate = dates[i]
            ifile = files[i]
            xfile=pd.read_csv(ifile,)
            # step 1 filter out all the unqualified ones, if empty return none
            e_file = xfile.groupby('User_name').count().sort_values('User_flr',ascending=False)
            all_tweets = pd.concat([all_tweets,xfile],axis=0)
        #
        all_accounts = all_tweets.groupby('User_name').count().ID
        # add the followers info
        all_flr = all_tweets.groupby('User_name').sum().User_flr
        # if the file is empty, then raise exception
        all_info = pd.concat([all_accounts,all_flr],axis=1)
        #
        all_info.columns = ['Freq','Score']
        all_info['Flr'] = all_info.Score/all_info.Freq
        #
        all_info.sort_values('Freq', ascending=False, inplace=True)
        if len(all_info)==0:
            raise Exception('There are not enough sentiments to show.')
        #
        return all_info

max_stars_repo_name: WencaiZheng/TwitterAnalysisWithEarnings

max_issues_repo_name: WencaiZheng/TwitterAnalysisWithEarnings

max_forks_repo_name: WencaiZheng/TwitterAnalysisWithEarnings


text:
#importing the cv2 library
import cv2

#importing numPy library
import numpy as np

# as all the images are saved as arrays in the computer thus an array
# of zeroes represent a black image.
img=np.zeros([256,256,4],np.uint8)

# Drawng a line on the image it takes 5 major arguments
# image source, starting point, ending point, color int BGR format, thickness
img=cv2.line(img,(0,0),(128,128),(0,0,255),5)

# Drawing an arrowed line, takes the same arguments as the normal line
img=cv2.arrowedLine(img,(0,256),(128,128),(82,168,50),5)

# Drawing a rectangle, the argument it takes are similar to the that of a line
# but the point of difference is that the starting point is the top-left
# corner of the rectangle and the ending point is the bottom-right
# corner of the rectangle
img=cv2.rectangle(img,(160,0),(255,80),(157,50,168),-1)

# Drawing a circle, the argument it takes are as follows :-
# image source, center co-ordinate, radius, Colour in BGR format, Thickness
# NOTE:- that he thickness=-1 will fill up the figure
img=cv2.circle(img,(170,170),30,(5,234,255),-1)

# Putting up a text on the image, the arguments are:-
# image source, text to be shown, location, font style, font size,
# colour in BGR format, thickness, line style
img=cv2.putText(img,"IKUSEI ROCKS !!",(5,100),
        cv2.FONT_HERSHEY_DUPLEX,1,(255,47,5),4,cv2.LINE_AA)
cv2.imshow("image",img)
cv2.waitKey(0)
cv2.destroyAllWindows()

max_stars_repo_path: 4 geometric_shapes.py

max_stars_repo_name: rupinderjdn/OpenCV

max_issues_repo_path: 4 geometric_shapes.py

max_issues_repo_name: rupinderjdn/OpenCV

max_forks_repo_path: 4 geometric_shapes.py

max_forks_repo_name: rupinderjdn/OpenCV


text:
import numpy as np

def crd2grid(y, x):
    ux, indX0, indX = np.unique(x, return_index=True, return_inverse=True)
    uy, indY0, indY = np.unique(y, return_index=True, return_inverse=True)

    minDx = np.min(ux[1:] - ux[0:-1])
    minDy = np.min(uy[1:] - uy[0:-1])
    maxDx = np.max(ux[1:] - ux[0:-1])
    maxDy = np.max(uy[1:] - uy[0:-1])
    if maxDx > minDx * 2:
        print("skipped rows")
    #     indMissX=np.where((ux[1:]-ux[0:-1])>minDx*2)[0]
    #     insertX=(ux[indMissX+1]+ux[indMissX])/2
    #     ux=np.insert(ux,indMissX,insertX)
    if maxDy > minDy * 2:
        print("skipped coloums")
    #     indMissY=np.where((uy[1:]-uy[0:-1])>minDy*2)
    #     raise Exception('skipped coloums or rows')

    uy = uy[::-1]
    ny = len(uy)
    indY = ny - 1 - indY
    return (uy, ux, indY, indX)

def array2grid(data, *, lat, lon):
    (uy, ux, indY, indX) = crd2grid(lat, lon)
    ny = len(uy)
    nx = len(ux)
    if data.ndim == 2:
        nt = data.shape[1]
        grid = np.full([ny, nx, nt], np.nan)
        grid[indY, indX, :] = data
    elif data.ndim == 1:
        grid = np.full([ny, nx], np.nan)
        grid[indY, indX] = data
    return grid, uy, ux

max_stars_repo_name: csiro-hydroinformatics/hydroDL

max_issues_repo_name: csiro-hydroinformatics/hydroDL

max_forks_repo_name: csiro-hydroinformatics/hydroDL


text:
from operator import methodcaller

import numpy as np
import pandas as pd
import pandas.testing as tm
import pytest

from ibis import literal as L

pytest.importorskip('clickhouse_driver')
pytestmark = pytest.mark.clickhouse

@pytest.mark.parametrize(
    ('reduction', 'func_translated'),
    [
        ('sum', 'sum'),
        ('count', 'count'),
        ('mean', 'avg'),
        ('max', 'max'),
        ('min', 'min'),
        ('std', 'stddevSamp'),
        ('var', 'varSamp'),
    ],
)
def test_reduction_where(con, alltypes, translate, reduction, func_translated):
    template = '{0}If(`double_col`, `bigint_col` < 70)'
    expected = template.format(func_translated)

    method = getattr(alltypes.double_col, reduction)
    cond = alltypes.bigint_col < 70
    expr = method(where=cond)

    assert translate(expr) == expected

def test_std_var_pop(con, alltypes, translate):
    cond = alltypes.bigint_col < 70
    expr1 = alltypes.double_col.std(where=cond, how='pop')
    expr2 = alltypes.double_col.var(where=cond, how='pop')

    assert translate(expr1) == 'stddevPopIf(`double_col`, `bigint_col` < 70)'
    assert translate(expr2) == 'varPopIf(`double_col`, `bigint_col` < 70)'
    assert isinstance(con.execute(expr1), np.float)
    assert isinstance(con.execute(expr2), np.float)

@pytest.mark.parametrize('reduction', ['sum', 'count', 'max', 'min'])
def test_reduction_invalid_where(con, alltypes, reduction):
    condbad_literal = L('T')

    with pytest.raises(TypeError):
        fn = methodcaller(reduction, where=condbad_literal)
        fn(alltypes.double_col)

@pytest.mark.parametrize(
    ('func', 'pandas_func'),
    [
        (
            lambda t, cond: t.bool_col.count(),
            lambda df, cond: df.bool_col.count(),
        ),
        (
            lambda t, cond: t.bool_col.approx_nunique(),
            lambda df, cond: df.bool_col.nunique(),
        ),
        (
            lambda t, cond: t.double_col.sum(),
            lambda df, cond: df.double_col.sum(),
        ),
        (
            lambda t, cond: t.double_col.mean(),
            lambda df, cond: df.double_col.mean(),
        ),
        (
            lambda t, cond: t.int_col.approx_median(),
            lambda df, cond: np.int32(df.int_col.median()),
        ),
        (
            lambda t, cond: t.double_col.min(),
            lambda df, cond: df.double_col.min(),
        ),
        (
            lambda t, cond: t.double_col.max(),
            lambda df, cond: df.double_col.max(),
        ),
        (
            lambda t, cond: t.double_col.var(),
            lambda df, cond: df.double_col.var(),
        ),
        (
            lambda t, cond: t.double_col.std(),
            lambda df, cond: df.double_col.std(),
        ),
        (
            lambda t, cond: t.double_col.var(how='sample'),
            lambda df, cond: df.double_col.var(ddof=1),
        ),
        (
            lambda t, cond: t.double_col.std(how='pop'),
            lambda df, cond: df.double_col.std(ddof=0),
        ),
        (
            lambda t, cond: t.bool_col.count(where=cond),
            lambda df, cond: df.bool_col[cond].count(),
        ),
        (
            lambda t, cond: t.double_col.sum(where=cond),
            lambda df, cond: df.double_col[cond].sum(),
        ),
        (
            lambda t, cond: t.double_col.mean(where=cond),
            lambda df, cond: df.double_col[cond].mean(),
        ),
        (
            lambda t, cond: t.float_col.approx_median(where=cond),
            lambda df, cond: df.float_col[cond].median(),
        ),
        (
            lambda t, cond: t.double_col.min(where=cond),
            lambda df, cond: df.double_col[cond].min(),
        ),
        (
            lambda t, cond: t.double_col.max(where=cond),
            lambda df, cond: df.double_col[cond].max(),
        ),
        (
            lambda t, cond: t.double_col.var(where=cond),
            lambda df, cond: df.double_col[cond].var(),
        ),
        (
            lambda t, cond: t.double_col.std(where=cond),
            lambda df, cond: df.double_col[cond].std(),
        ),
        (
            lambda t, cond: t.double_col.var(where=cond, how='sample'),
            lambda df, cond: df.double_col[cond].var(),
        ),
        (
            lambda t, cond: t.double_col.std(where=cond, how='pop'),
            lambda df, cond: df.double_col[cond].std(ddof=0),
        ),
    ],
)
def test_aggregations(alltypes, df, func, pandas_func, translate):
    table = alltypes.limit(100)
    count = table.count().execute()
    df = df.head(int(count))

    cond = table.string_col.isin(['1', '7'])
    mask = cond.execute().astype('bool')
    expr = func(table, cond)

    result = expr.execute()
    expected = pandas_func(df, mask)

    np.testing.assert_allclose(result, expected)

@pytest.mark.parametrize(
    'op',
    [
        methodcaller('sum'),
        methodcaller('mean'),
        methodcaller('min'),
        methodcaller('max'),
        methodcaller('std'),
        methodcaller('var'),
    ],
)
def test_boolean_reduction(alltypes, op, df):
    result = op(alltypes.bool_col).execute()
    assert result == op(df.bool_col)

def test_anonymus_aggregate(alltypes, df, translate):
    t = alltypes
    expr = t[t.double_col > t.double_col.mean()]
    result = expr.execute().set_index('id')
    expected = df[df.double_col > df.double_col.mean()].set_index('id')
    tm.assert_frame_equal(result, expected, check_like=True)

def test_boolean_summary(alltypes):
    expr = alltypes.bool_col.summary()
    result = expr.execute()
    expected = pd.DataFrame(
        [[7300, 0, 0, 1, 3650, 0.5, 2]],
        columns=[
            'count',
            'nulls',
            'min',
            'max',
            'sum',
            'mean',
            'approx_nunique',
        ],
    )
    tm.assert_frame_equal(
        result, expected, check_column_type=False, check_dtype=False
    )

max_stars_repo_name: jreback/ibis

max_issues_repo_name: jreback/ibis

max_forks_repo_name: jreback/ibis


text:
import os
import os.path as op

import numpy as np
import pandas as pd
import nibabel as nib

def write_workflow_report(workflow_name, report_template, report_dict):
    """Generic function to take write .rst files and convert to pdf/html.

    Accepts a report template and dictionary. Writes rst once with
    full paths for image files and generates a pdf, then strips
    leading path components and writes again, generating an html
    file that exepects to live in the same directory as report images.

    """
    from os.path import exists, basename
    from subprocess import check_output

    # Plug the values into the template for the pdf file
    report_rst_text = report_template % report_dict

    # Write the rst file and convert to pdf
    report_pdf_rst_file = "%s_pdf.rst" % workflow_name
    report_pdf_file = op.abspath("%s_report.pdf" % workflow_name)
    open(report_pdf_rst_file, "w").write(report_rst_text)
    check_output(["rst2pdf", report_pdf_rst_file, "-o", report_pdf_file])
    if not exists(report_pdf_file):
        raise RuntimeError

    # For images going into the html report, we want the path to be relative
    # (We expect to read the html page from within the datasink directory
    # containing the images.  So iteratate through and chop off leading path.
    for k, v in report_dict.items():
        if isinstance(v, str) and v.endswith(".png"):
            report_dict[k] = basename(v)

    # Write the another rst file and convert it to html
    report_html_rst_file = "%s_html.rst" % workflow_name
    report_html_file = op.abspath("%s_report.html" % workflow_name)
    report_rst_text = report_template % report_dict
    open(report_html_rst_file, "w").write(report_rst_text)
    check_output(["rst2html.py", report_html_rst_file, report_html_file])
    if not exists(report_html_file):
        raise RuntimeError

    # Return both report files as a list
    return [report_pdf_file, report_html_file]

max_stars_repo_name: sgagnon/lyman

max_issues_repo_name: sgagnon/lyman

max_forks_repo_name: sgagnon/lyman


text:
# MIT License

# Copyright (c) 2021 Taiki Miyagawa and Akinori F. Ebihara

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#
# ==============================================================================

from __future__ import absolute_import, division, print_function
import os, shutil, datetime
import numpy as np
import tensorflow as tf

def checkpoint_logger(global_step, model, optimizer, flag_resume, root_ckptlogs,
    subproject_name, exp_phase, comment, time_stamp, path_resume=None,
    max_to_keep=3, config_path=None):
    """Make ckpt and manager objects, and restore the latest checkpoint if necessary.
    Args:
        global_step: A tf.Variable Tensor with dtype=tf.int32. tf.int64 didn't work...
        model: A tf.keras.Model object.
        optimizer: An optimizer object such as tf.optimizers.Adam(0.1)
        flag_resume: A boolean. Whether to resume training from the latest ckpt.
        root_ckptlogs: A string. Used for path to ckpts.
        subproject_name: A string. Used for path to ckpts.
        comment: A string. Used for path to ckpts.
        time_stamp: A string. Used for path to ckpts.
        path_resume: A string or None. The path to ckpt logs to be resumed.
            path_resume is ignored if flag_resume=False.
        max_to_keep: An int. Set max_to_keep=0 or None to keep all the ckpts.
        config_path: A string, where config file is saved for reference.
    Returns:
        ckpt: tf.train.Checkpoint object.
        ckpt_manager: tf.train.CheckpointManager object.
    Remark:
        Path to checkpoint files is
            'root_ckptlogs'/'subproject_name'_'exp_phase'/'comment'_'time_stamp'/ckptXXX
    """
    # Naming rule
    dir_ckptlogs = "{}/{}_{}/{}_{}".format(
        root_ckptlogs, subproject_name, exp_phase, comment, time_stamp)

    if not(config_path is None):
        dir_configs = dir_ckptlogs + "/configs"

        if not os.path.exists(dir_configs):
            os.makedirs(dir_configs)

        now = datetime.datetime.now().strftime("%Y%m%d_%H%M%S%f")[:-3]
        shutil.copyfile(config_path, dir_configs + "/config_{}.yaml".format(now))

    # Create ckpt
    ckpt = tf.train.Checkpoint(step=global_step, optimizer=optimizer, net=model)

    # If resume
    if flag_resume:
        assert os.path.exists(path_resume), "Not exist: path_ckpt {}".format(
            path_resume)

        # Create ckpt and manager for restore
        ckpt_manager_restore = tf.train.CheckpointManager(
            ckpt, path_resume, max_to_keep=max_to_keep)

        # Restore the latest ckpt log.
        ckpt.restore(ckpt_manager_restore.latest_checkpoint)
        print("Restored from {}".format(ckpt_manager_restore.latest_checkpoint))

    # Create manager
    ckpt_manager = tf.train.CheckpointManager(
        ckpt, dir_ckptlogs, max_to_keep=max_to_keep)

    return ckpt, ckpt_manager

max_stars_repo_name: TaikiMiyagawa/MSPRT-TANDEM

max_issues_repo_name: TaikiMiyagawa/MSPRT-TANDEM

max_forks_repo_name: TaikiMiyagawa/MSPRT-TANDEM


text:
import pandas as pd
from typing import Dict, List
import logging
from src.definitions import DIR_OUTPUT
import numpy as np

def process_clean(
    data: Dict[str, List[Dict[str, str]]], *, save_pickle=False
) -> Dict[str, pd.DataFrame]:
    """
    Takes a dict with values representing types of data (cases, deaths, tests...). Each value is a list of ordered
    dictionaries county-by-county data for that data type.

    This func converts each list of ordered dicts into a pandasDataFrame and formats columnnames and sets datatypes. E

    Args:
        data (Dict[List[Dict[str, str]]]): A dict of lists of ordered dicts.
        save_pickle (bool): Save each dataframe as a pickel in the root directory

    Return:
        A dictionary of pandas dataframes with cleaned data.
    """
    clean_data = {}
    for data_type, data_dict in data.items():
        logging.info(
            f"Cleaning '{data_type}' data and converting to Pandas DataFrame..."
        )
        df = pd.DataFrame(data_dict)
        # df.columns = map(str.lower, df.columns)  # set col names to lowercase
        df = df.rename(columns={"Date": "date"})
        for col in df.columns:
            if "date" in col:
                df["date"] = pd.to_datetime(df["date"])
            else:
                # replace empty cells with 0s
                df[col] = df[col].replace(r"^\s*$", 0, regex=True)
                # coerce any cells that include non-numeric numbers into NaN, datatype is now float
                df[col] = pd.to_numeric(df[col], errors="coerce")
                df = df.replace(np.nan, 0, regex=True)  # replace NaNs with 0s
                # now we've handled potentially invalid input, convert dtype to int
                df[col] = df[col].astype(int)

        # add to payload dict
        clean_data[data_type] = df

        # save pickel
        if save_pickle:
            df.to_pickle(f"{DIR_OUTPUT}/{data_type}.pkl")

    return clean_data

max_stars_repo_name: spotlightpa/covid-alerts-emailer

max_issues_repo_name: spotlightpa/covid-alerts-emailer

max_forks_repo_name: spotlightpa/covid-alerts-emailer


text:
""" $lic$
Copyright (c) 2016-2021, Mingyu Gao

This program is free software: you can redistribute it and/or modify it under
the terms of the Modified BSD-3 License as published by the Open Source
Initiative.

This program is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
PARTICULAR PURPOSE. See the BSD-3 License for more details.

You should have received a copy of the Modified BSD-3 License along with this
program. If not, see <https://opensource.org/licenses/BSD-3-Clause>.
"""

import unittest
import numpy as np
import matplotlib
from matplotlib import pyplot as plt

from easypyplot import format as fmt  # avoid conflict with built-in.

from . import sin_plot, skip_if_without_tex
from . import image_comparison

@image_comparison(baseline_images=['format_turn_off_box'])
def test_turn_off_box():
    ''' format turn off box. '''
    fig = plt.figure()
    ax = fig.gca()

    fmt.turn_off_box(ax)

    sin_plot(ax)

@image_comparison(baseline_images=['format_turn_off_box_twinx'])
def test_turn_off_box_twinx():
    ''' format turn off box with twinx. '''
    fig = plt.figure()
    ax = fig.gca()
    ax2 = ax.twinx()

    fmt.turn_off_box(ax, twinx_axes=ax2)

    sin_plot(ax2)

@image_comparison(baseline_images=['format_paper_plot'])
def test_paper_plot():
    ''' format paper plot. '''
    # Must be before figure creation.
    fmt.paper_plot(font='default')

    fig = plt.figure()
    ax = fig.gca()

    sin_plot(ax, remove_text=False)
    sin_plot(ax, phi=0.5 * np.pi, fmt='--', remove_text=False)
    sin_plot(ax, phi=1 * np.pi, fmt='-.', remove_text=False)
    sin_plot(ax, phi=1.5 * np.pi, fmt=':', remove_text=False)

    x = np.linspace(0, 6, 13)
    y = x / -6
    ax.plot(x, y, 's')

    ax.scatter([1, 2, 3], [0, -0.5, 0.5])

    ax.set_xlabel('My x label')
    ax.set_ylabel('My y label')
    ax.set_xticks(np.linspace(0, 6, 7, endpoint=True))
    ax.set_yticks(np.linspace(-1, 1, 9, endpoint=True))
    ax.set_xlim([0, 6])
    ax.set_ylim([-1, 1])
    ax.yaxis.grid(True)
    ax.legend(['My data 1-1', 'My data 1-2', 'My data 1-3', 'My data 1-4',
               'My data 2', 'My points'])

@image_comparison(baseline_images=['format_paper_plot_mono'])
def test_paper_plot_mono():
    ''' format paper plot with monospace font. '''
    # Must be before figure creation.
    fmt.paper_plot(font=('monospace', 'DejaVu Sans Mono'))

    fig = plt.figure()
    ax = fig.gca()

    sin_plot(ax, remove_text=False)

    ax.set_xlabel('My x label')
    ax.set_ylabel('My y label')

@image_comparison(baseline_images=['format_paper_plot_mathtext'],
                  remove_text=False)
def test_paper_plot_mathtext():
    ''' format paper plot and use mathtext. '''
    # Must be before figure creation.
    fmt.paper_plot(font='default')

    fig = plt.figure()
    ax = fig.gca()

    sin_plot(ax)
    ax.set_yticks(np.linspace(-100, 100, 11, endpoint=True) / 100.)

    ax.set_yticklabels([r'$e^{' + '{}'.format(x) + r'}$' for x in range(11)])

@image_comparison(baseline_images=['format_resize_ax_box_w_r'])
def test_resize_ax_box_w_r():
    ''' format resize axes box, width to right. '''
    fig = plt.figure()
    ax = fig.gca()

    sin_plot(ax)

    fmt.resize_ax_box(ax, wratio=0.8, to_right=True)

@image_comparison(baseline_images=['format_resize_ax_box_w_l'])
def test_resize_ax_box_w_l():
    ''' format resize axes box, width to left. '''
    fig = plt.figure()
    ax = fig.gca()

    sin_plot(ax)

    fmt.resize_ax_box(ax, wratio=0.6, to_right=False)

@image_comparison(baseline_images=['format_resize_ax_box_h_b'])
def test_resize_ax_box_h_b():
    ''' format resize axes box, height to bottom. '''
    fig = plt.figure()
    ax = fig.gca()

    sin_plot(ax)

    fmt.resize_ax_box(ax, hratio=0.85, to_bottom=True)

@image_comparison(baseline_images=['format_resize_ax_box_h_t'])
def test_resize_ax_box_h_t():
    ''' format resize axes box, height to top. '''
    fig = plt.figure()
    ax = fig.gca()

    sin_plot(ax)

    fmt.resize_ax_box(ax, hratio=0.7, to_bottom=False)

@image_comparison(baseline_images=['format_resize_ax_box_both'])
def test_resize_ax_box_both():
    ''' format resize axes box, both width and height. '''
    fig = plt.figure()
    ax = fig.gca()

    sin_plot(ax)

    fmt.resize_ax_box(ax, wratio=0.6, hratio=0.8)

@image_comparison(baseline_images=['format_set_axis_to_percent'],
                  remove_text=False)
def test_set_axis_to_percent():
    ''' format set axis tick to percentage. '''
    fig = plt.figure()
    ax = fig.gca()

    sin_plot(ax)
    ax.set_yticks(np.linspace(-100, 100, 11, endpoint=True) / 100.)

    fmt.set_axis_to_percent(ax.yaxis)

@image_comparison(baseline_images=['format_set_axis_to_percent_pcs'],
                  remove_text=False)
def test_set_axis_to_percent_pcs():
    ''' format set axis tick to percentage with precision. '''
    fig = plt.figure()
    ax = fig.gca()

    sin_plot(ax)
    ax.set_yticks(np.linspace(-100, 100, 11, endpoint=True) / 100.)

    fmt.set_axis_to_percent(ax.yaxis, precision=2)

@image_comparison(baseline_images=['format_set_axis_to_percent_tex'],
                  extensions=['pdf'], remove_text=False)
def test_set_axis_to_percent_tex():
    ''' format set axis tick to percentage with Tex. '''
    skip_if_without_tex()
    # Must be before figure creation.
    matplotlib.rcParams['text.usetex'] = True

    fig = plt.figure()
    ax = fig.gca()

    sin_plot(ax)
    ax.set_yticks(np.linspace(-100, 100, 11, endpoint=True) / 100.)

    fmt.set_axis_to_percent(ax.yaxis)

@image_comparison(baseline_images=['format_set_axis_to_percent_pplot'],
                  remove_text=False)
def test_set_axis_to_percent_pplot():
    ''' format set axis tick to percentage with paper plot. '''
    fmt.paper_plot()

    fig = plt.figure()
    ax = fig.gca()

    sin_plot(ax)
    ax.set_yticks(np.linspace(-100, 100, 11, endpoint=True) / 100.)

    fmt.set_axis_to_percent(ax.yaxis)

@image_comparison(baseline_images=['format_set_group_xticklabels'],
                  remove_text=False)
def test_set_group_xticklabels():
    ''' format set group labels along x axis. '''
    fig = plt.figure()
    ax = fig.add_subplot(211)

    gnum = 4
    xnum = 12
    xs = list(range(xnum))
    y = 0.2

    ax.set_ylim([0, 1])
    ax.axhline(y, xmin=xs[0], xmax=xs[-1])

    ax.set_xticks(xs)

    grouplabels = ['g{}'.format(i) for i in range(gnum)]
    xvals = range((xnum // gnum - 1) // 2, xnum, xnum // gnum)
    fmt.set_group_xticklabels(ax, grouplabels, xvals, -y)

@image_comparison(baseline_images=['format_set_group_xticklabels'],
                  remove_text=False, save_suffix='_orig_xvals')
def test_set_group_xticklabels_orig_xvals():
    ''' format set group labels along x axis, with original xvals. '''
    # pylint: disable=invalid-name
    fig = plt.figure()
    ax = fig.add_subplot(211)

    gnum = 4
    xnum = 12
    xs = list(range(xnum))
    y = 0.2

    ax.set_ylim([0, 1])
    ax.axhline(y, xmin=xs[0], xmax=xs[-1])

    ax.set_xticks(xs)

    grouplabels = ['g{}'.format(i) for i in range(gnum)]
    fmt.set_group_xticklabels(ax, grouplabels, xs, -y)

class TestFormat(unittest.TestCase):
    ''' Tests for format module. '''

    def test_get_fig_dims(self):
        ''' get_fig_dims(). '''
        figsize = fmt.get_fig_dims(100)
        self.assertEqual(len(figsize), 2)
        self.assertAlmostEqual(figsize[0], 1.3889, places=4)
        self.assertAlmostEqual(figsize[1] / figsize[0], 0.618, places=3)

        figsize = fmt.get_fig_dims(250)
        self.assertEqual(len(figsize), 2)
        self.assertAlmostEqual(figsize[0], 3.4722, places=4)
        self.assertAlmostEqual(figsize[1] / figsize[0], 0.618, places=3)

    def test_paper_plot_invalid_font(self):
        ''' paper_plot invalid font. '''
        with self.assertRaisesRegexp(ValueError, r'\[format\] .*font.*'):
            fmt.paper_plot(font='DejaVu Serif')

    def test_set_group_xticklabels_invalid_xvals(self):
        ''' set_group_xticklabels invalid xvals. '''
        # pylint: disable=invalid-name
        fig = plt.figure()
        ax = fig.gca()

        with self.assertRaisesRegexp(ValueError, r'\[format\] .*xvals.*'):
            fmt.set_group_xticklabels(ax, ['a', 'b'], [0, 1, 2], -1)

max_stars_repo_name: leepoly/easypyplot

max_issues_repo_name: leepoly/easypyplot

max_forks_repo_name: leepoly/easypyplot


text:
import pickle

import jax.numpy as jnp
import lab.jax as B
import wbml.out
from plum import Dispatcher
from probmods import Model, cast, fit, instancemethod, priormethod
from stheno.jax import Normal
from varz import Vars
from stheno import GP

from .approx import MeanField, Structured
from .util import (
    closest_psd,
    estimate_psd,
    summarise_samples,
    min_phase as transform_min_phase,
)

__all__ = ["AbstractGPCM"]

_dispatch = Dispatcher()

class AbstractGPCM(Model):
    """GPCM model.

    Args:
        scheme (str, optional): Approximation scheme. Must be one of `structured`,
            `mean-field-ca`, `mean-field-gradient`, `mean-field-collapsed-gradient`,
            `mean-field-ca-gradient`, or `mean-field-ca-collapsed-gradient`.
            Defaults to `structured`.
    """

    @_dispatch
    def __init__(self, scheme: str = "structured"):
        self.vs = Vars(jnp.float64)
        self.scheme = scheme.lower()

    def __prior__(self):
        # Construct kernel matrices.
        self.K_z = self.compute_K_z()
        self.K_z_inv = B.pd_inv(self.K_z)
        self.K_u = self.compute_K_u()
        self.K_u_inv = B.pd_inv(self.K_u)

        # Construct priors.
        self.p_u = Normal(self.K_u_inv)
        self.p_z = Normal(self.K_z_inv)

        # Construct approximation scheme.
        if self.scheme == "structured":
            self.approximation = Structured(self)
        elif self.scheme == "mean-field":
            # Use the best mean-field scheme.
            self.approximation = MeanField(self, fit="ca-collapsed-bfgs")
        elif self.scheme == "mean-field-ca":
            self.approximation = MeanField(self, fit="ca")
        elif self.scheme == "mean-field-gradient":
            self.approximation = MeanField(self, fit="bfgs")
        elif self.scheme == "mean-field-collapsed-gradient":
            self.approximation = MeanField(self, fit="collapsed-bfgs")
        elif self.scheme == "mean-field-ca-gradient":
            self.approximation = MeanField(self, fit="ca-bfgs")
        elif self.scheme == "mean-field-ca-collapsed-gradient":
            self.approximation = MeanField(self, fit="ca-collapsed-bfgs")
        else:
            raise ValueError(
                f'Invalid value "{self.scheme}" for the approximation scheme.'
            )

    def __condition__(self, t, y):
        self.approximation.condition(t, y)

    @instancemethod
    @cast
    def elbo(self, *args, **kw_args):
        return self.approximation.elbo(*args, **kw_args)

    @instancemethod
    @cast
    def predict(self, *args, **kw_args):
        return self.approximation.predict(*args, **kw_args)

    @instancemethod
    def predict_kernel(self, t_k=None, num_samples=1000):
        """Predict kernel and normalise prediction.

        Args:
            t_k (vector, optional): Inputs to sample kernel at. Will be automatically
                determined if not given.
            num_samples (int, optional): Number of samples to use. Defaults to `1000`.

        Returns:
            :class:`collections.namedtuple`: The prediction.
        """
        return summarise_samples(*self.sample_kernel(t_k=t_k, num_samples=num_samples))

    @instancemethod
    def sample_kernel(self, t_k=None, num_samples=1000):
        """Predict kernel and normalise prediction.

        Args:
            t_k (vector, optional): Inputs to sample kernel at. Will be automatically
                determined if not given.
            num_samples (int, optional): Number of samples to use. Defaults to `1000`.

        Returns:
            tuple[vector, tensor]: Tuple containing the inputs of the samples and the
                samples.
        """
        if t_k is None:
            t_k = B.linspace(self.dtype, 0, self.extent, 300)

        ks = self.approximation.sample_kernel(t_k, num_samples=num_samples)

        # Normalise predicted kernel.
        var_mean = B.mean(ks[:, 0])
        wbml.out.kv("Mean variance of kernel samples", var_mean)

        return t_k, ks

    @instancemethod
    def predict_psd(self, t_k=None, num_samples=1000):
        """Predict the PSD in dB.

        Args:
            t_k (vector, optional): Inputs to sample kernel at. Will be automatically
                determined if not given.
            num_samples (int, optional): Number of samples to use. Defaults to `1000`.

        Returns:
            :class:`collections.namedtuple`: Predictions.
        """
        if t_k is None:
            t_k = B.linspace(self.dtype, 0, 2 * self.extent, 1000)
        t_k, ks = self.sample_kernel(t_k, num_samples=num_samples)

        # Estimate PSDs.
        freqs, psds = zip(*[estimate_psd(t_k, k, db=False) for k in ks])
        freqs = freqs[0]
        psds = B.stack(*psds, axis=0)

        return summarise_samples(freqs, psds, db=True)

    @instancemethod
    def predict_fourier(self, num_samples=1000):
        """Predict Fourier features.

        Args:
            num_samples (int, optional): Number of samples to use. Defaults to `1000`.

        Returns:
            tuple: Marginals of the predictions.
        """
        return self.approximation.predict_z(num_samples=num_samples)

    @instancemethod
    def predict_filter(self, t_h=None, num_samples=1000, min_phase=True):
        """Predict the learned filter.

        Args:
            t_h (vector, optional): Inputs to sample filter at.
            num_samples (int, optional): Number of samples to use. Defaults to `1000`.
            min_phase (bool, optional): Predict a minimum-phase version of the filter.
                Defaults to `True`.

        Returns:
            :class:`collections.namedtuple`: Predictions.
        """
        if t_h is None:
            t_h = B.linspace(self.dtype, -self.extent, self.extent, 601)

        @B.jit
        def sample_h(state):
            state, u = self.approximation.p_u.sample(state)
            u = B.mm(self.K_u, u)  # Transform :math:`\hat u` into :math:`u`.
            h = GP(self.k_h())
            h = h | (h(self.t_u), u)  # Condition on sample.
            state, h = h(t_h).sample(state)  # Sample at desired points.
            return state, B.flatten(h)

        # Perform sampling.
        state = B.global_random_state(self.dtype)
        samples = []
        for _ in range(num_samples):
            state, h = sample_h(state)

            # Transform sample according to specification.
            if min_phase:
                h = transform_min_phase(h)

            samples.append(h)
        B.set_global_random_state(state)

        if min_phase:
            # Start at zero.
            t_h = t_h - t_h[0]
        return summarise_samples(t_h, B.stack(*samples, axis=0))

    @instancemethod
    @cast
    def kernel_approx(self, t1, t2, u):
        """Kernel approximation using inducing variables :math:`u` for the
        impulse response :math:`h`.

        Args:
            t1 (vector): First time input.
            t2 (vector): Second time input.
            u (vector): Values of the inducing variables.

        Returns:
            tensor: Approximation of the kernel matrix broadcasted over `t1` and `t2`.
        """
        # Construct the first part.
        part1 = self.compute_i_hx(t1[:, None], t2[None, :])

        # Construct the second part.
        L_u = B.cholesky(self.K_u)
        inv_L_u = B.trisolve(L_u, B.eye(L_u))
        prod = B.mm(inv_L_u, B.uprank(u, rank=2))
        I_ux = self.compute_I_ux(t1, t2)
        trisolved = B.mm(inv_L_u, I_ux, inv_L_u, tr_c=True)
        part2 = B.trace(trisolved) - B.trace(B.mm(prod, trisolved, prod, tr_a=True))

        return part1 - part2

    @priormethod
    @cast
    def sample(self, t, normalise=False):
        """Sample the kernel then the function.

        Args:
            t (vector): Time points to sample the function at.
            normalise (bool, optional): Normalise the sample of the kernel.
                Defaults to `False`.

        Returns:
            tuple: Tuple containing the kernel matrix and the function.
        """
        u = B.sample(self.compute_K_u())[:, 0]
        K = self.kernel_approx(t, t, u)
        if normalise:
            K = K / K[0, 0]
        f = B.sample(closest_psd(K))[:, 0]
        y = f + B.sqrt(self.noise) * B.randn(f)
        return K, y

    def save(self, path):
        """Save model and inference results to a file.

        Args:
            path (str): Path to save to.
        """
        data = {name: B.to_numpy(B.dense(self.vs[name])) for name in self.vs.names}
        with open(path, "wb") as f:
            pickle.dump(data, f)

    def load(self, path):
        """Load model from a file.

        Args:
            path (str): Path to load from.
        """
        with open(path, "rb") as f:
            data = pickle.load(f)
        for name, value in data.items():
            if name in self.vs:
                # Overwrite existing values.
                self.vs.assign(name, value)
            else:
                # Assign an invisible bounded variable: we lost the information about
                # the constraints.
                self.vs.unbounded(init=value, visible=False, name=name)

@fit.dispatch
def fit(model: AbstractGPCM, t, y, **kw_args):
    fit(model, t, y, model().approximation, **kw_args)

max_stars_repo_name: magnusross/gpcm

max_issues_repo_name: magnusross/gpcm

max_forks_repo_name: magnusross/gpcm

reason: import jax


text:
from collections import OrderedDict
from itertools import compress

try:
    import itertools.izip as zip
except ImportError:
    pass

import numpy as np

from .interface import Interface
from ..dimension import Dimension
from ..element import Element, NdElement
from ..dimension import OrderedDict as cyODict
from ..ndmapping import NdMapping, item_check
from .. import util

class DictInterface(Interface):
    """
    Interface for simple dictionary-based dataset format. The dictionary
    keys correspond to the column (i.e dimension) names and the values
    are collections representing the values in that column.
    """

    types = (dict, OrderedDict, cyODict)

    datatype = 'dictionary'

    @classmethod
    def dimension_type(cls, dataset, dim):
        name = dataset.get_dimension(dim, strict=True).name
        return dataset.data[name].dtype.type

    @classmethod
    def init(cls, eltype, data, kdims, vdims):
        odict_types = (OrderedDict, cyODict)
        if kdims is None:
            kdims = eltype.kdims
        if vdims is None:
            vdims = eltype.vdims

        dimensions = [d.name if isinstance(d, Dimension) else
                      d for d in kdims + vdims]
        if isinstance(data, tuple):
            data = {d: v for d, v in zip(dimensions, data)}
        elif ((util.is_dataframe(data) and all(d in data for d in dimensions)) or
              (isinstance(data, NdElement) and all(d in data.dimensions() for d in dimensions))):
            data = {d: data[d] for d in dimensions}
        elif isinstance(data, np.ndarray):
            if data.ndim == 1:
                if eltype._auto_indexable_1d and len(kdims)+len(vdims)>1:
                    data = np.column_stack([np.arange(len(data)), data])
                else:
                    data = np.atleast_2d(data).T
            data = {k: data[:,i] for i,k in enumerate(dimensions)}
        elif isinstance(data, list) and data == []:
            data = OrderedDict([(d, []) for d in dimensions])
        elif isinstance(data, list) and np.isscalar(data[0]):
            data = {dimensions[0]: np.arange(len(data)), dimensions[1]: data}
        elif (isinstance(data, list) and isinstance(data[0], tuple) and len(data[0]) == 2
              and any(isinstance(v, tuple) for v in data[0])):
            dict_data = zip(*((util.wrap_tuple(k)+util.wrap_tuple(v))
                              for k, v in data))
            data = {k: np.array(v) for k, v in zip(dimensions, dict_data)}
        # Ensure that interface does not consume data of other types
        # with an iterator interface
        elif not any(isinstance(data, tuple(t for t in interface.types if t is not None))
                     for interface in cls.interfaces.values()):
            data = {k: v for k, v in zip(dimensions, zip(*data))}
        elif isinstance(data, dict) and not all(d in data for d in dimensions):
            dict_data = zip(*((util.wrap_tuple(k)+util.wrap_tuple(v))
                              for k, v in data.items()))
            data = {k: np.array(v) for k, v in zip(dimensions, dict_data)}

        if not isinstance(data, cls.types):
            raise ValueError("DictInterface interface couldn't convert data.""")
        elif isinstance(data, dict):
            unpacked = [(d, np.array(data[d])) for d in data]
            if not cls.expanded([d[1] for d in unpacked]):
                raise ValueError('DictInterface expects data to be of uniform shape.')
            if isinstance(data, odict_types):
                data.update(unpacked)
            else:
                data = OrderedDict([(d, np.array(data[d])) for d in dimensions])
        return data, {'kdims':kdims, 'vdims':vdims}, {}

    @classmethod
    def validate(cls, dataset):
        dimensions = dataset.dimensions(label='name')
        not_found = [d for d in dimensions if d not in dataset.data]
        if not_found:
            raise ValueError('Following dimensions not found in data: %s' % not_found)
        lengths = [len(dataset.data[dim]) for dim in dimensions]
        if len({l for l in lengths if l > 1}) > 1:
            raise ValueError('Length of columns do not match')

    @classmethod
    def unpack_scalar(cls, dataset, data):
        """
        Given a dataset object and data in the appropriate format for
        the interface, return a simple scalar.
        """
        if len(data) != 1:
            return data
        key = list(data.keys())[0]

        if len(data[key]) == 1 and key in dataset.vdims:
            return data[key][0]

    @classmethod
    def shape(cls, dataset):
        return cls.length(dataset), len(dataset.data),

    @classmethod
    def length(cls, dataset):
        return len(list(dataset.data.values())[0])

    @classmethod
    def array(cls, dataset, dimensions):
        if not dimensions:
            dimensions = dataset.dimensions(label='name')
        else:
            dimensions = [dataset.get_dimensions(d).name for d in dimensions]
        return np.column_stack(dataset.data[dim.name] for dim in dimensions)

    @classmethod
    def add_dimension(cls, dataset, dimension, dim_pos, values, vdim):
        dim = dimension.name if isinstance(dimension, Dimension) else dimension
        data = list(dataset.data.items())
        if isinstance(values, util.basestring) or not hasattr(values, '__iter__'):
            values = np.array([values]*len(dataset))
        data.insert(dim_pos, (dim, values))
        return OrderedDict(data)

    @classmethod
    def redim(cls, dataset, dimensions):
        return OrderedDict([(dimensions.get(k, dataset.get_dimension(k)).name, v)
                            for k,v in dataset.data.items()])

    @classmethod
    def concat(cls, dataset_objs):
        cast_objs = cls.cast(dataset_objs)
        cols = set(tuple(c.data.keys()) for c in cast_objs)
        if len(cols) != 1:
            raise Exception("In order to concatenate, all Dataset objects "
                            "should have matching set of columns.")
        concatenated = OrderedDict()
        for column in cols.pop():
            concatenated[column] = np.concatenate([obj[column] for obj in cast_objs])
        return concatenated

    @classmethod
    def sort(cls, dataset, by=[]):
        by = [dataset.get_dimension(d).name for d in by]
        if len(by) == 1:
            sorting = cls.values(dataset, by[0]).argsort()
        else:
            arrays = [dataset.dimension_values(d) for d in by]
            sorting = util.arglexsort(arrays)
        return OrderedDict([(d, v[sorting]) for d, v in dataset.data.items()])

    @classmethod
    def values(cls, dataset, dim, expanded=True, flat=True):
        dim = dataset.get_dimension(dim).name
        values = np.array(dataset.data.get(dim))
        if not expanded:
            return util.unique_array(values)
        return values

    @classmethod
    def reindex(cls, dataset, kdims, vdims):
        dimensions = [dataset.get_dimension(d).name for d in kdims+vdims]
        return OrderedDict([(d, dataset.dimension_values(d))
                            for d in dimensions])

    @classmethod
    def groupby(cls, dataset, dimensions, container_type, group_type, **kwargs):
        # Get dimensions information
        dimensions = [dataset.get_dimension(d) for d in dimensions]
        kdims = [kdim for kdim in dataset.kdims if kdim not in dimensions]
        vdims = dataset.vdims

        # Update the kwargs appropriately for Element group types
        group_kwargs = {}
        group_type = dict if group_type == 'raw' else group_type
        if issubclass(group_type, Element):
            group_kwargs.update(util.get_param_values(dataset))
            group_kwargs['kdims'] = kdims
        group_kwargs.update(kwargs)

        # Find all the keys along supplied dimensions
        keys = [tuple(dataset.data[d.name][i] for d in dimensions)
                for i in range(len(dataset))]

        # Iterate over the unique entries applying selection masks
        grouped_data = []
        for unique_key in util.unique_iterator(keys):
            mask = cls.select_mask(dataset, dict(zip(dimensions, unique_key)))
            group_data = OrderedDict(((d.name, dataset[d.name][mask]) for d in kdims+vdims))
            group_data = group_type(group_data, **group_kwargs)
            grouped_data.append((unique_key, group_data))

        if issubclass(container_type, NdMapping):
            with item_check(False):
                return container_type(grouped_data, kdims=dimensions)
        else:
            return container_type(grouped_data)

    @classmethod
    def select(cls, dataset, selection_mask=None, **selection):
        if selection_mask is None:
            selection_mask = cls.select_mask(dataset, selection)
        indexed = cls.indexed(dataset, selection)
        data = OrderedDict((k, list(compress(v, selection_mask)))
                           for k, v in dataset.data.items())
        if indexed and len(list(data.values())[0]) == 1 and len(dataset.vdims) == 1:
            return data[dataset.vdims[0].name][0]
        return data

    @classmethod
    def sample(cls, dataset, samples=[]):
        mask = False
        for sample in samples:
            sample_mask = True
            if np.isscalar(sample): sample = [sample]
            for i, v in enumerate(sample):
                name = dataset.get_dimension(i).name
                sample_mask &= (np.array(dataset.data[name])==v)
            mask |= sample_mask
        return {k: np.array(col)[mask]
                for k, col in dataset.data.items()}

    @classmethod
    def aggregate(cls, dataset, kdims, function, **kwargs):
        kdims = [dataset.get_dimension(d, strict=True).name for d in kdims]
        vdims = dataset.dimensions('value', label='name')
        groups = cls.groupby(dataset, kdims, list, OrderedDict)
        aggregated = OrderedDict([(k, []) for k in kdims+vdims])

        for key, group in groups:
            key = key if isinstance(key, tuple) else (key,)
            for kdim, val in zip(kdims, key):
                aggregated[kdim].append(val)
            for vdim, arr in group.items():
                if vdim in dataset.vdims:
                    if isinstance(function, np.ufunc):
                        reduced = function.reduce(arr, **kwargs)
                    else:
                        reduced = function(arr, **kwargs)
                    aggregated[vdim].append(reduced)
        return aggregated

    @classmethod
    def iloc(cls, dataset, index):
        rows, cols = index
        scalar = False
        if np.isscalar(cols):
            scalar = np.isscalar(rows)
            cols = [dataset.get_dimension(cols, strict=True)]
        elif isinstance(cols, slice):
            cols = dataset.dimensions()[cols]
        else:
            cols = [dataset.get_dimension(d, strict=True) for d in cols]

        if np.isscalar(rows):
            rows = [rows]

        new_data = OrderedDict()
        for d, values in dataset.data.items():
            if d in cols:
                new_data[d] = values[rows]

        if scalar:
            return new_data[cols[0].name][0]
        return new_data

Interface.register(DictInterface)

max_stars_repo_name: stonebig/holoviews

max_issues_repo_name: stonebig/holoviews

max_forks_repo_name: stonebig/holoviews


text:
"""
Homomorphisms of abelian groups

.. TODO::

    - there must be a homspace first

    - there should be hom and Hom methods in abelian group

AUTHORS:

- David Joyner (2006-03-03): initial version
"""

#*****************************************************************************
#   Copyright (C) 2006 David Joyner and William Stein <wstein@gmail.com>
#
#   Distributed under the terms of the GNU General Public License (GPL)
#
#                    http://www.gnu.org/licenses/
#*****************************************************************************
from __future__ import print_function

from sage.groups.perm_gps.permgroup import *
from sage.interfaces.gap import *
from sage.categories.morphism import *
from sage.categories.homset import *

from sage.misc.all import prod

def is_AbelianGroupMorphism(f):
    return isinstance(f, AbelianGroupMorphism)

class AbelianGroupMap(Morphism):
    """
    A set-theoretic map between AbelianGroups.
    """
    def __init__(self, parent):
        """
        The Python constructor.
        """
        Morphism.__init__(self, parent)

    def _repr_type(self):
        return "AbelianGroup"

class AbelianGroupMorphism_id(AbelianGroupMap):
    """
    Return the identity homomorphism from X to itself.

    EXAMPLES:
    """
    def __init__(self, X):
        AbelianGroupMorphism.__init__(self, X.Hom(X))

    def _repr_defn(self):
        return "Identity map of " + str(X)

class AbelianGroupMorphism(Morphism):
    """
    Some python code for wrapping GAP's GroupHomomorphismByImages
    function for abelian groups. Returns "fail" if gens does not
    generate self or if the map does not extend to a group
    homomorphism, self - other.

    EXAMPLES::

        sage: G = AbelianGroup(3,[2,3,4],names="abc"); G
        Multiplicative Abelian group isomorphic to C2 x C3 x C4
        sage: a,b,c = G.gens()
        sage: H = AbelianGroup(2,[2,3],names="xy"); H
        Multiplicative Abelian group isomorphic to C2 x C3
        sage: x,y = H.gens()

        sage: from sage.groups.abelian_gps.abelian_group_morphism import AbelianGroupMorphism
        sage: phi = AbelianGroupMorphism(H,G,[x,y],[a,b])

    TESTS::

        sage: G.<x,y> = AbelianGroup(2,[2,3])
        sage: H.<a,b,c> = AbelianGroup(3,[2,3,4])
        sage: phi = AbelianGroupMorphism(G,H,[x,y],[a,b])
        sage: Hom(G,H) == phi.parent()
        True

    AUTHORS:

    - David Joyner (2006-02)
    """

#    There is a homomorphism from H to G but not from G to H:
#
#        sage: phi = AbelianGroupMorphism_im_gens(G,H,[a*b,a*c],[x,y])
#------------------------------------------------------------
#Traceback (most recent call last):
#  File "<ipython console>", line 1, in ?
#  File ".abeliangp_hom.sage.py", line 737, in __init__
#    raise TypeError, "Sorry, the orders of the corresponding elements in %s, %s must be equal."%(genss,imgss)
#TypeError: Sorry, the orders of the corresponding elements in [a*b, a*c], [x, y] must be equal.
#
#        sage: phi = AbelianGroupMorphism_im_gens(G,H,[a*b,(a*c)^2],[x*y,y])
#------------------------------------------------------------
#Traceback (most recent call last):
#  File "<ipython console>", line 1, in ?
#  File ".abeliangp_hom.sage.py", line 730, in __init__
#    raise TypeError, "Sorry, the list %s must generate G."%genss
#TypeError: Sorry, the list [a*b, c^2] must generate G.

    def __init__(self, G, H, genss, imgss):
        from sage.categories.homset import Hom
        Morphism.__init__(self, Hom(G, H))
        if len(genss) != len(imgss):
            raise TypeError("Sorry, the lengths of %s, %s must be equal." % (genss, imgss))
        self._domain = G
        self._codomain = H
        if not(G.is_abelian()):
            raise TypeError("Sorry, the groups must be abelian groups.")
        if not(H.is_abelian()):
            raise TypeError("Sorry, the groups must be abelian groups.")
        G_domain = G.subgroup(genss)
        if G_domain.order() != G.order():
            raise TypeError("Sorry, the list %s must generate G." % genss)
        # self.domain_invs = G.gens_orders()
        # self.codomaininvs = H.gens_orders()
        self.domaingens = genss
        self.codomaingens = imgss
        for i in range(len(self.domaingens)):
            if (self.domaingens[i]).order() != (self.codomaingens[i]).order():
                raise TypeError("Sorry, the orders of the corresponding elements in %s, %s must be equal." % (genss, imgss))

    def _gap_init_(self):
        """
        Only works for finite groups.

        EXAMPLES::

            sage: G = AbelianGroup(3,[2,3,4],names="abc"); G
            Multiplicative Abelian group isomorphic to C2 x C3 x C4
            sage: a,b,c = G.gens()
            sage: H = AbelianGroup(2,[2,3],names="xy"); H
            Multiplicative Abelian group isomorphic to C2 x C3
            sage: x,y = H.gens()
            sage: phi = AbelianGroupMorphism(H,G,[x,y],[a,b])
            sage: phi._gap_init_()
            'phi := GroupHomomorphismByImages(G,H,[x, y],[a, b])'
        """
        G  = (self.domain())._gap_init_()
        H  = (self.codomain())._gap_init_()
        s3 = 'G:=%s; H:=%s' % (G, H)
        gap.eval(s3)
        gensG = self.domain().variable_names()  # the Sage group generators
        gensH = self.codomain().variable_names()
        s1 = "gensG := GeneratorsOfGroup(G)"    # the GAP group generators
        gap.eval(s1)
        s2 = "gensH := GeneratorsOfGroup(H)"
        gap.eval(s2)
        for i in range(len(gensG)):             # making the Sage group gens
           cmd = ("%s := gensG["+str(i+1)+"]") % gensG[i]  # correspond to the Sage group gens
           gap.eval(cmd)
        for i in range(len(gensH)):
           cmd = ("%s := gensH[" + str(i + 1) + "]") % gensH[i]
           gap.eval(cmd)
        args = str(self.domaingens) + "," + str(self.codomaingens)
        gap.eval("phi := GroupHomomorphismByImages(G,H," + args + ")")
        self.gap_hom_string = "phi := GroupHomomorphismByImages(G,H,"+args+")"
        return self.gap_hom_string

    def _repr_type(self):
        return "AbelianGroup"

    def kernel(self):
        """
        Only works for finite groups.

        .. TODO::

            not done yet; returns a gap object but should return a Sage group.

        EXAMPLES::

            sage: H = AbelianGroup(3,[2,3,4],names="abc"); H
            Multiplicative Abelian group isomorphic to C2 x C3 x C4
            sage: a,b,c = H.gens()
            sage: G = AbelianGroup(2,[2,3],names="xy"); G
            Multiplicative Abelian group isomorphic to C2 x C3
            sage: x,y = G.gens()
            sage: phi = AbelianGroupMorphism(G,H,[x,y],[a,b])
            sage: phi.kernel()
            'Group([  ])'

            sage: H = AbelianGroup(3,[2,2,2],names="abc")
            sage: a,b,c = H.gens()
            sage: G = AbelianGroup(2,[2,2],names="x")
            sage: x,y = G.gens()
            sage: phi = AbelianGroupMorphism(G,H,[x,y],[a,a])
            sage: phi.kernel()
            'Group([ f1*f2 ])'
        """
        cmd = self._gap_init_()
        gap.eval(cmd)
        return gap.eval("Kernel(phi)")

    def image(self, S):
        """
        Return the image of the subgroup ``S`` by the morphism.

        This only works for finite groups.

        INPUT:

        - ``S`` -- a subgroup of the domain group ``G``

        EXAMPLES::

            sage: G = AbelianGroup(2,[2,3],names="xy")
            sage: x,y = G.gens()
            sage: subG = G.subgroup([x])
            sage: H = AbelianGroup(3,[2,3,4],names="abc")
            sage: a,b,c = H.gens()
            sage: phi = AbelianGroupMorphism(G,H,[x,y],[a,b])
            sage: phi.image(subG)
            Multiplicative Abelian subgroup isomorphic to C2 generated by {a}
        """
        return self.codomain().subgroup([self(g) for g in S.gens()])

    def _call_(self, g):
        """
        Some python code for wrapping GAP's Images function but only for
        permutation groups. Returns an error if g is not in G.

        EXAMPLES::

            sage: H = AbelianGroup(3, [2,3,4], names="abc")
            sage: a,b,c = H.gens()
            sage: G = AbelianGroup(2, [2,3], names="xy")
            sage: x,y = G.gens()
            sage: phi = AbelianGroupMorphism(G,H,[x,y],[a,b])
            sage: phi(y*x)
            a*b
            sage: phi(y^2)
            b^2
        """
        # g.word_problem is faster in general than word_problem(g)
        gens = self.codomaingens
        return prod(gens[(self.domaingens).index(wi[0])]**wi[1]
                    for wi in g.word_problem(self.domaingens))

max_stars_repo_name: bopopescu/sage

max_issues_repo_name: bopopescu/sage

max_forks_repo_name: bopopescu/sage

reason: from sage


text:
# Copyright (c) OpenMMLab. All rights reserved.
import base64
import numpy as np
import os
import torch
from ts.torch_handler.base_handler import BaseHandler

from mmdet3d.apis import inference_detector, init_model
from mmdet3d.core.points import get_points_type

class MMdet3dHandler(BaseHandler):
    """MMDetection3D Handler used in TorchServe.

    Handler to load models in MMDetection3D, and it will process data to get
    predicted results. For now, it only supports SECOND.
    """
    threshold = 0.5
    load_dim = 4
    use_dim = [0, 1, 2, 3]
    coord_type = 'LIDAR'
    attribute_dims = None

    def initialize(self, context):
        """Initialize function loads the model in MMDetection3D.

        Args:
            context (context): It is a JSON Object containing information
                pertaining to the model artifacts parameters.
        """
        properties = context.system_properties
        self.map_location = 'cuda' if torch.cuda.is_available() else 'cpu'
        self.device = torch.device(self.map_location + ':' +
                                   str(properties.get('gpu_id')) if torch.cuda.
                                   is_available() else self.map_location)
        self.manifest = context.manifest

        model_dir = properties.get('model_dir')
        serialized_file = self.manifest['model']['serializedFile']
        checkpoint = os.path.join(model_dir, serialized_file)
        self.config_file = os.path.join(model_dir, 'config.py')
        self.model = init_model(self.config_file, checkpoint, self.device)
        self.initialized = True

    def preprocess(self, data):
        """Preprocess function converts data into LiDARPoints class.

        Args:
            data (List): Input data from the request.

        Returns:
            `LiDARPoints` : The preprocess function returns the input
                point cloud data as LiDARPoints class.
        """
        for row in data:
            # Compat layer: normally the envelope should just return the data
            # directly, but older versions of Torchserve didn't have envelope.
            pts = row.get('data') or row.get('body')
            if isinstance(pts, str):
                pts = base64.b64decode(pts)

            points = np.frombuffer(pts, dtype=np.float32)
            points = points.reshape(-1, self.load_dim)
            points = points[:, self.use_dim]
            points_class = get_points_type(self.coord_type)
            points = points_class(
                points,
                points_dim=points.shape[-1],
                attribute_dims=self.attribute_dims)

        return points

    def inference(self, data):
        """Inference Function.

        This function is used to make a prediction call on the
        given input request.

        Args:
            data (`LiDARPoints`): LiDARPoints class passed to make
                the inference request.

        Returns:
            List(dict) : The predicted result is returned in this function.
        """
        results, _ = inference_detector(self.model, data)
        return results

    def postprocess(self, data):
        """Postprocess function.

        This function makes use of the output from the inference and
        converts it into a torchserve supported response output.

        Args:
            data (List[dict]): The data received from the prediction
                output of the model.

        Returns:
            List: The post process function returns a list of the predicted
                output.
        """
        output = []
        for pts_index, result in enumerate(data):
            output.append([])
            if 'pts_bbox' in result.keys():
                pred_bboxes = result['pts_bbox']['boxes_3d'].tensor.numpy()
                pred_scores = result['pts_bbox']['scores_3d'].numpy()
            else:
                pred_bboxes = result['boxes_3d'].tensor.numpy()
                pred_scores = result['scores_3d'].numpy()

            index = pred_scores > self.threshold
            bbox_coords = pred_bboxes[index].tolist()
            score = pred_scores[index].tolist()

            output[pts_index].append({'3dbbox': bbox_coords, 'score': score})

        return output

max_stars_repo_name: konyul/monovideo

max_issues_repo_name: konyul/monovideo

max_forks_repo_name: konyul/monovideo


text:
"""
the diags_file contains the hits that dagchainer found, the all_file is the full list of blast hits.
this script goes through all of the hits in the dag file and adds any hit from the all_file that is within 'dist' of
any hit in the diag.
"""
from scipy.spatial import cKDTree
import numpy as np
import sys
from cdagline import DagLine
import collections

def read_dag_to_tree(all_hits):
    """create a ckdtree, using query as x, subject as y
    do this for all htis, then for each diag, do an query
    to find nearby
    """
    trees = collections.defaultdict(list)
    lines = collections.defaultdict(list)
    for sline in open(all_hits):
        if sline[0] == '#': continue
        d = DagLine(sline)

        if d.a_seqid < d.b_seqid:
            chrs = (d.a_seqid, d.b_seqid)
            trees[chrs].append((d.a_start, d.b_start))
            lines[chrs].append(d)
        else:
            chrs = (d.b_seqid, d.a_seqid)
            trees[chrs].append((d.b_start, d.a_start))
            lines[chrs].append(d)

    for chrs in trees:
        trees[chrs] = cKDTree(np.array(trees[chrs]), leafsize=16)
    return dict(trees), dict(lines)

def find_nearby(dist, diags, all_hits):
    """empty docstring"""
    trees, lines = read_dag_to_tree(all_hits)
    global_seen = {}
    seen = {}
    # need to save these and print at the end
    # of a block in oder to know the nubmer of total genes.
    current_lines = []

    for sline in open(diags):
        if sline[0] == '#':
            seen = {}
            if len(current_lines) > 1:
                header = current_lines[0].split("\t")
                # update the gene count.
#                header[-1] = str(len(current_lines) - 1)
                current_lines[0] = "\t".join(header)+"\t"+str(len(current_lines) - 1)
                # print the entire block.
                print "\n".join(current_lines)

            current_lines = [sline.strip()]
            continue

        d = DagLine(sline)
        chrs = (d.a_seqid, d.b_seqid)
        global_seen[(d.a_accn, d.b_accn)] = None

        if not (d.a_accn, d.b_accn) in seen:
            seen[(d.a_accn, d.b_accn)] = None
            current_lines.append(sline.rstrip())
        else:
            continue
        try:
            tree = trees[chrs]
        except:
            print >>sys.stderr, trees.keys()[:10]
            raise
        info_lines = lines[chrs]

        q, s = d.a_start, d.b_start

        dists, idxs = tree.query([q, s], p=2, distance_upper_bound=dist, k=64)
        # values set to tree.n are extras.
        idxs = idxs[idxs != tree.n]

        for i in idxs:
            d = info_lines[i]
            ikey = (d.a_accn, d.b_accn)
            if ikey in global_seen: continue
            if ikey in seen: continue
            current_lines.append(str(d))
            seen[ikey] = True

if __name__ == "__main__":
    import optparse
    parser = optparse.OptionParser()
    parser.add_option("-d", "--dist", dest="dist", help="distance around each pair to look for missed pairs")
    parser.add_option("--diags", dest="diags", help="the dagchainter output file")
    parser.add_option("--all", dest="all", help="the dag blast hit file containing all hits of q to s")

    (options, _) = parser.parse_args()
    if not (options.dist and options.diags and options.all):
        sys.exit(parser.print_help())

    find_nearby(int(options.dist), options.diags, options.all)

max_stars_repo_name: LyonsLab/coge

max_issues_repo_name: LyonsLab/coge

max_forks_repo_name: LyonsLab/coge

reason: import numpy,from scipy


text:
# The following exec statement (or something like it) is needed to
# prevent SyntaxError on Python < 2.5. Even though this is a test,
# SyntaxErrors are not acceptable; on Debian systems, they block
# byte-compilation during install and thus cause the package to fail
# to install.

import sys
if sys.version_info[:2] >= (2, 5):
    exec("""
from __future__ import with_statement
from numpy.core import *
from numpy.random import rand, randint
from numpy.testing import *

class TestErrstate(TestCase):
    def test_invalid(self):
        with errstate(all='raise', under='ignore'):
            a = -arange(3)
            # This should work
            with errstate(invalid='ignore'):
                sqrt(a)
            # While this should fail!
            try:
                sqrt(a)
            except FloatingPointError:
                pass
            else:
                self.fail("Did not raise an invalid error")

    def test_divide(self):
        with errstate(all='raise', under='ignore'):
            a = -arange(3)
            # This should work
            with errstate(divide='ignore'):
                a // 0
            # While this should fail!
            try:
                a // 0
            except FloatingPointError:
                pass
            else:
                self.fail("Did not raise divide by zero error")

    def test_errcall(self):
        def foo(*args):
            print(args)
        olderrcall = geterrcall()
        with errstate(call=foo):
            assert(geterrcall() is foo), 'call is not foo'
            with errstate(call=None):
                assert(geterrcall() is None), 'call is not None'
        assert(geterrcall() is olderrcall), 'call is not olderrcall'

""")

if __name__ == "__main__":
    run_module_suite()

reason: from numpy


text:
## relation extractor
## https://conceptnet.io/
## https://github.com/commonsense/conceptnet5/wiki/Downloads

import logging
logging.basicConfig(format='%(asctime)s - %(message)s',
                    datefmt='%d-%b-%y %H:%M:%S')
logging.getLogger().setLevel(logging.INFO)

import gzip
import os
import pandas as pd
import networkx as nx
import tqdm
from collections import Counter
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
import nltk

POSSIBLE_RELATIONS = ["is_a","part_of","subset_of","represents_a","serves_for","lives_in","works_for","represents_the","looks_like","synonym_for","similar_to","context_of","related_to","knows_of","knowledge_of","serves_to","serves_for","lives_near","parent_of","child_of","superior_to"]

class ConceptFeatures:
    """
    Core class describing sentence embedding methodology employed here.
    """
    def __init__(self,
                 max_features=10000,
                 targets=None,
                 knowledge_graph="../memory"):

        self.max_features = max_features
        self.targets = targets
        ## TODO -> download conceptnet if not present.

        self.knowledge_graph = knowledge_graph
        self.feature_names = None

    def get_grounded_from_path(self, present_tokens, graph_path):
        """
        Method which performs a very simple term grounding. This simply evaluates if both terms are present in the corpus.
        :param list present_tokens: The present tokens
        :param str graph_path: Path to the triplet base (compressed)
        """

        with gzip.open(graph_path, "rt", encoding="utf-8") as gp:
            for line in gp:
                subject, predicate, obj = line.strip().split("\t")
                if subject.lower() in present_tokens and obj.lower(
                ) in present_tokens:
                    if subject != obj:
                        yield (subject, predicate, obj)

    def add_triplet(self, tokens, index, relations=["is_a"]):

        token = tokens[index]
        for relation in relations:
            p1, p2 = relation.split("_")
            if token.lower() == p1 and tokens[index + 1].lower() == p2:
                if len(tokens[index - 1]) > 1 and len(tokens[index + 2]) > 1:
                    triplet_adhoc = (tokens[index - 1], relation,
                                     tokens[index + 2])
                    if triplet_adhoc[0] != triplet_adhoc[2]:
                        if len(triplet_adhoc[0]) > 3 and len(triplet_adhoc[2]) > 3:
                            yield triplet_adhoc

    def concept_graph(self, document_space, graph_path):
        """
        If no prior knowledge graph is supplied, one is constructed.
        :param document_space: The list of input documents
        :param graph_path: The path of the knowledge graph used.
        :return grounded: Grounded relations.
        """

        generic_triplets = []
        present_tokens = set()

        for document in document_space:
            tokens = nltk.word_tokenize(document)
            tokens = [word.lower() for word in tokens]
            for i, token in enumerate(tokens):
                if i > 0 and i < len(tokens) - 1:
                    found_triplets = self.add_triplet(tokens, i, relations=POSSIBLE_RELATIONS)
                    for triplet in found_triplets:
                        generic_triplets.append(triplet)

            for token in tokens:
                present_tokens.add(token)

        grounded = []

        if len(generic_triplets) == 0:
            logging.info("Generating the knowledge graph (ad hoc)")
            for document in document_space:
                tokens = nltk.word_tokenize(document)
                tokens = [str(x) for x in tokens]
                for enx, token in enumerate(tokens):
                    if enx > 1 and enx < len(tokens) - 2:
                        if len(tokens[enx - 1]) >= 2 and len(
                                tokens[enx + 1]) >= 2:
                            triplet_adhoc = (tokens[enx - 2], token,
                                             tokens[enx + 2])
                            if triplet_adhoc[0] != triplet_adhoc[2]:
                                generic_triplets.append(triplet_adhoc)

        logging.info(f"Found the following relations: {set([x[1] for x in generic_triplets])}")
        try:

            kg_sources = os.listdir(graph_path)
            full_paths = [os.path.join(graph_path, x) for x in kg_sources]

            for path in full_paths:
                triplet_generator = self.get_grounded_from_path(
                    present_tokens, path)

                for triplet in triplet_generator:
                    grounded.append(triplet)
                logging.info(f"Grounded in total: {len(grounded)} triplets (last added from {path}).")

        except Exception as es:
            logging.info(
                f"No knowledge graphs found in the default path: {graph_path}. Reverting to generic triplet extraction from the corpus alone. To use a knowledge graph, place a Gzipped triplet (tsv) database in {graph_path} folder. ({es})"
            )
            grounded = generic_triplets
            del generic_triplets

        logging.info(
            "The number of grounded relations in the input corpus is: {}".
            format(len(grounded)))

        if len(grounded) == 0:
            logging.info(
                "No grounded relations found, please recoonsider the used knowledge base."
            )

        return grounded

    def get_propositionalized_rep(self, documents):
        """
        The method for constructing the representation.

        :param documents: The input list of documents.
        """

        G = nx.DiGraph()
        for (s, p, o) in tqdm.tqdm(self.grounded_triplets):
            G.add_edge(s, o, type=p)
        numbered_links = {e: enx for enx, e in enumerate(G.edges())}

        if self.feature_names is None:
            self.feature_names = [
                e[2]['type'] + "(" + e[0] + "," + e[1] + ")"
                for e in G.edges(data=True)
            ]
            self.feature_types = [e[2]['type'] for e in G.edges(data=True)]
        rbags = []
        all_doc_sizes = []
        self.t_counter = Counter()
        for enx2, document in tqdm.tqdm(enumerate(documents),
                                        total=len(documents)):
            tokens = nltk.word_tokenize(document)
            tokens = [word.lower() for word in tokens]
            doc_tokens = set(tokens)
            present = doc_tokens.intersection(G.nodes())
            subgraph = G.subgraph(present)
            rb = []

            for edge in subgraph.edges(data=True):

                enx1 = numbered_links[(edge[0], edge[1])]
                fname = edge[2]['type'] + "(" + edge[0] + "," + edge[1] + ")"
                self.t_counter.update({fname: 1})
                enx1 = self.feature_names[enx1]
                rb.append(enx1)

            all_doc_sizes.append(len(rb))
            rbags.append(" ".join(rb))
        logging.info("Most common relations below.")
        logging.info(self.t_counter.most_common(10))
        logging.info(f"Average bag size: {np.mean(all_doc_sizes)}")
        return rbags

    def fit(self, text_vector, refit=False, knowledge_graph=None):
        """
        Fit the model to a text vector.

        :param text_vector: Input list of documents.
        """

        if self.knowledge_graph is None:
            self.knowledge_graph = knowledge_graph

        logging.info("Constructing the token graph.")
        sentences_separated = []
        for doc in text_vector:
            sentences = nltk.tokenize.sent_tokenize(doc)
            for els in sentences:
                sentences_separated.append(els)
        self.grounded_triplets = self.concept_graph(sentences_separated,
                                                    self.knowledge_graph)
        logging.info("Relation propositionalization ..")

        # this identifies relational bags that are grounded + stores the info
        self.conc_docs = self.get_propositionalized_rep(text_vector)
        logging.info("Concept-based features extracted.")

        self.concept_vectorizer = TfidfVectorizer(
            ngram_range=(1, 1),
            sublinear_tf=True,  ## for very sparse spaces
            max_features=self.max_features,
            token_pattern=r'\S+').fit(self.conc_docs)

    def transform(self, text_vector, use_conc_docs=False):
        """
        Transform the data into suitable form.
        """

        if use_conc_docs:
            text_vector = self.conc_docs
        else:
            text_vector = self.get_propositionalized_rep(text_vector)
        return self.concept_vectorizer.transform(text_vector)

    def get_feature_names(self):

        return self.concept_vectorizer.get_feature_names()

    def fit_transform(self, text_vector, b=None):
        """
        A classifc fit-transform method.

        :param text_vector: The input list of documents.
        :return transformedObj: Transformed texts (to features).
        """

        try:
            self.fit(text_vector, self.targets)
        except Exception as es:
            logging.info(es)

        return self.transform(text_vector,
                              use_conc_docs=True)  # use stored docs

if __name__ == "__main__":

    example_text = pd.read_csv("../data/depression/train.tsv", sep="\t")
    text = example_text['text_a']
    labels = example_text['label']
    rex = ConceptFeatures(knowledge_graph="./memory")
    m = rex.fit_transform(text)
    fnames = rex.get_feature_names()

    m = m.todense()
    dataframe = pd.DataFrame(m)
    dataframe.columns = fnames
    dataframe['target'] = labels
    dataframe.to_csv("patronizing.tsv", sep = "\t", index=False)

max_stars_repo_name: EMBEDDIA/autoBOT

max_issues_repo_name: EMBEDDIA/autoBOT

max_forks_repo_name: EMBEDDIA/autoBOT

reason: import numpy,import networkx