text:
"""
Various NLP-related utilities.

Licensed under the 3-clause BSD License:

Copyright (c) 2013, Neeraj Kumar (neerajkumar.org)
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in the
      documentation and/or other materials provided with the distribution.
    * Neither the name of the author nor the
      names of its contributors may be used to endorse or promote products
      derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL NEERAJ KUMAR BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""

import os, sys, time
from utils import *

STOP_WORDS = set(['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', 'her', 'hers', 'herself', 'it', 'its', 'itself', 'they', 'them', 'their', 'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', 'these', 'those', 'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', 'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 'while', 'of', 'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before', 'after', 'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each', 'few', 'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very', 's', 't', 'can', 'will', 'just', 'don', 'should', 'now'])

def fullnlp(s):
    """Tokenizes, tags, and runs named entity recognition on an input string.
    Returns tokens, tagged, entities, flatentities, candidates"""
    import nltk
    tokens = nltk.word_tokenize(s)
    tagged = nltk.pos_tag(tokens)
    entities = nltk.chunk.ne_chunk(tagged)
    flatents = [x for x in entities if not isinstance(x, tuple)]
    flatents = [(c.node, [l for l in c.leaves()]) for c in flatents]
    flatents = [(n, ' '.join([l[0] for l in c])) for n, c in flatents]
    candidates = [l[1] for l in flatents]
    return (tokens, tagged, entities, flatents, candidates)

def usefullnlp(s):
    """Uses the fullnlp function"""
    for r in ret:
        r['tokens'], r['tagged'], r['entities'], r['flatents'], r['candidates'] = extractnlp(r['titletext'])
        r['tagged'] = ' '.join(['%s/%s' % (s, tag) for s, tag in r['tagged']])
        r['entities'] = r['entities'].pprint()
        log('Got r: %s' % (r,))
        for c in r['candidates']:
            cands[c] += 1.0 - r['i']/20.0
    scores = sorted([(v, c) for c, v in cands.items()], reverse=1)
    for i, (s, c) in enumerate(scores):
        if c.lower() in x.q.lower():
            c += ' (In Query - IGNORED)'
        scores[i] = (s, c)

def ngramextractor(text, n=(1,2,3,4,5), ret=None, normfunc=None, lweight=0.0):
    """"Extracts n-grams of the given lengths from the given text.
    If the text is a string, then tokenizes it using nltk.
    Passes each token through normfunc, if given.
    If this function returns the empty string, that token is removed.
    If this function returns None, then no n-gram is allowed to span this break.
    Param 'lweight' weights each entry by n**lweight
    Returns (sentences, dict of counts). Sentences are lists of tokens (split by delimiters from normfunc).
    You can optionally pass in 'ret', a dict of existing counts, to append to.
    """
    import nltk
    if ret is None:
        ret = {}
    if isinstance(text, basestring):
        tokens = nltk.word_tokenize(text)
    else:
        tokens = text
    if normfunc:
        tokens = [normfunc(w) for w in tokens if normfunc(w) != '']
        sentences = []
        cur = []
        sentences.append(cur)
        for t in tokens:
            if t is None:
                if cur:
                    cur = []
                    sentences.append(cur)
            else:
                cur.append(t)
    else:
        sentences = [tokens]
    from utils import log
    #log('Converted %s to %s to %s' % (text, tokens, sentences))
    for sent in sentences:
        for i, w in enumerate(sent):
            for x in n:
                if i+x > len(sent): continue
                cur = ' '.join(sent[i:i+x])
                #log('  Got i %d, w %s, x %s, cur %s' % (i, w, x, cur))
                if cur not in ret:
                    ret[cur] = 0.0
                ret[cur] += 1*(x**lweight)
    #log('Got ret %s' % (sorted(ret.items(), key=lambda pair: pair[-1], reverse=1),))
    return sentences, ret

def matchstrings(a, b):
    """Matches strings in a very loose way.
    Returns (matches, score), where matches is a list of pairs of matching
    words from a and b, and score is the final normalized similarity score
    (higher is better)."""
    from utils import utf, strsim
    import numpy as np
    np.set_printoptions(precision=2, linewidth=200, suppress=1)
    def norm(s):
        """Normalizes a string"""
        s = utf(s).strip().lower()
        return s

    def split(s):
        """Splits a string into components, quite aggressively"""
        import re
        els = re.split('\W+', s)
        els = map(norm, els)
        els = [e for e in els if e and e not in STOP_WORDS]
        return els

    els1 = split(norm(a))
    els2 = split(norm(b))
    #print '%s -> %s' % (a, els1)
    #print '%s -> %s' % (b, els2)
    matches = []
    best = 0.0
    if not els1 or not els2: return (matches, best)
    m = np.zeros((len(els1), len(els2)))
    for i, e1 in enumerate(els1):
        for j, e2 in enumerate(els2):
            c = m[i,j] = strsim(e1, e2)
            #print '  %d,%d = %s vs %s = %s' % (i, j, e1, e2, c)
    while 1:
        n = np.argmax(m)
        i, j = loc = np.unravel_index([n], m.shape)
        s = m[loc]
        #print '  %s, %s, %s' % (n, loc, s)
        if s <= 0: break
        # if we're here, then we want to add this match
        #print 'got %s, %s, %s, %s, %s' % (n, i, j, els1, els2)
        matches.append((els1[i[0]], els2[j[0]]))
        best += s
        m[i,:] = -1
        m[:,j] = -1
        #print '    picking %s, %s' % (matches[-1], best)
        #print m
    if matches:
        best /= float(len(matches))
    return (matches, best)

@memoize
def splitted(s):
    return s.split()

@memoize
def hasWord(term, tag):
    """Returns true if the tag has the term as a subsequence of words."""
    tagwords = splitted(tag)
    termwords = splitted(term)
    nterms = len(termwords)
    term = ' '.join(termwords)
    #print termwords
    #print tagwords
    #print 'nterms:', nterms
    #print term
    #print (term in ' '.join(tagwords))
    # simple check to see if this could possibly contain the word
    if term not in ' '.join(tagwords): return 0
    #log('  Testing term %s (%d terms) with tag %s (%d words)' % (term, nterms, tagwords, len(tagwords)))
    for i in range(len(tagwords)):
        hyptag = ' '.join(tagwords[i:i+nterms])
        #log('    Checking i %d: %s vs %s' % (i, hyptag.encode('utf-8', 'ignore'), term.encode('utf-8', 'ignore')))
        if hyptag == term: return 1
    return 0

@memoize
def wordnetsim(a, b, pos=None):
    """Returns the similarity between the two given terms using wordnet.
    Optionally restricts to a given part of speech, one of 'noun', 'verb', 'adv', 'adj'.
    This searches over all possible pairs of synsets.
    It also takes the max of the wup_similarity and path_similarity score.
    Returns a score between 0 and 1.
    """
    from nltk.corpus import wordnet as wn
    from math import sqrt
    a = wn.morphy(a)
    b = wn.morphy(b)
    if not a or not b: return 0
    posmap = dict(noun=wn.NOUN, verb=wn.VERB, adj=wn.ADJ, adv=wn.ADV)
    alla = wn.synsets(a, pos=posmap.get(pos, None))
    allb = wn.synsets(b, pos=posmap.get(pos, None))
    #score1 = max(a.wup_similarity(b) for a in alla for b in allb)
    score = max(a.path_similarity(b) for a in alla for b in allb)
    if score is None: return 0
    score = sqrt(score)
    #print a, b, score1, score2
    return score

#@memoize
def getTagMatchFunc(matchtype, tags=None, layerkey=None):
    """Returns a function that takes in (queryterm, tag) and returns a score.
    The different matchtypes are:
           'exact': term exactly matches a tag
         'hasword': term is contained within a tag (respecting words)
        'contains': term is contained within a tag
       'unordered': term matches tag (in any order)
          'starts': term starts tag
            'ends': term ends tag
            'typo': term is misspelled version of a tag
            'path': term is close to another (by path similarity in wordnet)
         'synonym': term is a synonym of a tag
        'constant': returns 1
    You can optionally pass in a 'layerkey' as a unique id. This can be used for caches, etc.
    """
    import pickle
    from nkpylib.nlp import matchstrings
    if matchtype == 'constant': ret = lambda term, tag: 1
    elif matchtype == 'exact': ret = lambda term, tag: 1 if term == tag else 0
    elif matchtype == 'hasword': ret = hasWord
    elif matchtype == 'contains': ret = lambda term, tag: 1 if term and (term in tag or tag in term) else 0
    elif matchtype == 'unordered':
        def ret(term, tag):
            matches, s = matchstrings(term, tag)
            return s if s > 0.3 else 0

    elif matchtype == 'starts': ret = lambda term, tag: 1 if tag.startswith(term) else 0
    elif matchtype == 'ends': ret = lambda term, tag: 1 if tag.endswith(term) else 0
    elif matchtype == 'typo': ret = lambda term, tag: strsim(term, tag) if strsim(term, tag) > 0.7 else 0
    elif matchtype == 'path': ret = lambda term, tag: wordnetsim(term, tag) if wordnetsim(term, tag) > 0.3 else 0
    elif matchtype == 'synonym': ret = lambda term, tag: 0 #TODO fix
    else:
        raise NotImplementedError()
    return (ret)

#@timed
def scoreTermTags(term, tags, matchtypes, layerkey=None):
    """Scores the given term against a list of tags.
    'matchtypes' is a dict mapping different matching types to their weights:
        'constant': returns 1
           'exact': term exactly matches a tag
         'hasword': term is contained within a tag (respecting words)
        'contains': term is contained within a tag
       'unordered': term matches tag (in any order)
          'starts': term starts tag
            'ends': term ends tag
            'typo': term is misspelled version of a tag
            'path': term is close to another (by path similarity in wordnet)
         'synonym': term is a synonym of a tag

    Returns a list of scores, which is the value from the first matching matchtype * the given weight.
    The tags are checked in the order above.
    You can optionally pass in a 'layerkey' as a unique id. This can be used for caches, etc.
    """
    # setup
    ALL_MATCH_TYPES = 'constant exact hasword contains unordered starts ends typo path'.split()
    matchfuncs = dict((mt, getTagMatchFunc(mt, tags=tags, layerkey=layerkey)) for mt in ALL_MATCH_TYPES if mt in matchtypes)
    ret = []
    for tag in tags:
        for mt in ALL_MATCH_TYPES:
            if not tag.strip(): continue # this needs to be continue for the 'else' clause to kick in
            if mt not in matchfuncs: continue # this needs to be continue for the 'else' clause to kick in
            matchfunc = matchfuncs[mt]
            #log('Testing term "%s" vs tag "%s" with mt %s' % (term, tag, mt))
            score = matchfunc(term, tag) * matchtypes[mt]
            if score > 0:
                #print term, tag, mt, matchtypes[mt], score, layerkey
                ret.append(score)
                break
        else:
            ret.append(0)
    assert len(ret) == len(tags)
    return ret

def findall(s, sub):
    """Returns all indices of 'sub' within 's', as a list"""
    ret = []
    cur = 0
    while 1:
        n = s.find(sub, cur)
        if n < 0: return ret
        ret.append(n)
        cur = n+1
    return ret

def cutlabel(s, cuts):
    """Cuts a string s using a set of (n, label) cuts.
    Returns a list of (sub, label) pairs.
    If there was an initial part before the first cut, then it has a label of None.
    If there are no cuts, returns s as a single element, with label None.
    """
    cuts = sorted(cuts)
    # no cuts -> whole string is an element
    if not cuts: return [(s, None)]
    if cuts[0][0] != 0:
        cuts.insert(0, (0, None))
    if cuts[-1][0] < len(s)-1:
        cuts.append((len(s), None))
    locs, labels = zip(*cuts)
    ret = []
    for i, j, label in zip(locs, locs[1:], labels):
        ret.append((s[i:j], label))
    return ret

def nlpnorm(s):
    """Normalizes a token for NLP.
    Currently, this includes:
        - string.lower().strip()
        - if string is delimiter, then return None (forced n-gram break)
        - remove parens, braces, brackets, colons, commas, pipes, etc.
        - remove ...
        - if string contains . inside (but not at end), then force delimiter (assumes this is a website)
    """
    if not s: return ''
    s = s.lower().strip()
    if not isinstance(s, unicode):
        s = unicode(s, 'utf-8', 'ignore')
    for dlm in u'/ @ - | [ ] { } ; : vs. vs at versus « » › â –'.split():
        #log(u'|%s| (%s) vs |%s| (%s)' % (s, type(s), dlm, type(dlm)))
        if s == dlm: return None
    s = u''.join(c for c in s if c not in u"'|()[]{},;`\"«»›–â")
    s = s.replace('...','')
    if '.' in s and not s.endswith('.'): return None
    return s

def inany(el, seq):
    """Returns the first sequence element that el is part of, else None"""
    for item in seq:
        if el in item:
            return item
    return None

if __name__ == '__main__':
    a, b = sys.argv[1:3]
    print matchstrings(a, b)

max_stars_repo_name: neeraj-kumar/nkpylib

max_issues_repo_name: neeraj-kumar/nkpylib

max_forks_repo_name: neeraj-kumar/nkpylib


text:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

#global defn
dates = pd.date_range("20130101", periods=6)
df = pd.DataFrame(np.random.randn(6, 4), index=dates, columns=list("ABCD"))
df2 = pd.DataFrame(
    {
        "A": 1.0,
        "B": pd.Timestamp("20130102"),
        "C": pd.Series(1, index=list(range(4)), dtype="float32"),
        "D": np.array([3] * 4, dtype="int32"),
        "E": pd.Categorical(["test", "train", "test", "train"]),
        "F": "foo",
    }
)

#series
s = pd.Series([1, 3, 5, np.nan, 6, 8])

pd.date_range("20130101", periods=6, freq='Q')
dates = pd.date_range("20130101", periods=6)
dates

#df
np.random.randn(2)
np.random.randn(2, 3)
np.random.randn(2, 3, 4)
list("sadf")
df = pd.DataFrame(np.random.randn(6, 4), index=dates, columns=list("ABCD"))
df

range(4)
list(range(4))
df2 = pd.DataFrame(
    {
        "A": 1.0,
        "B": pd.Timestamp("20130102"),
        "C": pd.Series(1, index=list(range(4)), dtype="float32"),
        "D": np.array([3] * 4, dtype="int32"),
        "E": pd.Categorical(["test", "train", "test", "train"]),
        "F": "foo",
    }
)
df2

#view
df.index
df.columns
df.to_numpy()
df.describe()
df.T
df.sort_index(ascending=False)
df.sort_index(axis=1, ascending=False)

#select
df["A"]
df["20130102":"20130104"]
df.loc[dates[0]]
df.loc[dates[0], "A"]
df.at[dates[0], "A"]
df.iloc[3]
df.iloc[3:5, 0:2]
df.iloc[1:3, :]
df.iloc[1, 1]
df.iat[1, 1]
df[df["A"] > 1]
df[df > 0]
df2[df2["E"].isin(["two", "four"])]

#set
s1 = pd.Series([1, 2, 3, 4, 5, 6], index=pd.date_range("20130102", periods=6))
df.loc[:, "D"] = np.array([5] * len(df))
df[df < 0] = -df
df

#missing
df1 = df.reindex(index=dates[0:4], columns=list(df.columns) + ["E"])
df1.loc[dates[0] : dates[1], "E"] = 1
df1
df1.dropna(how="any")
df1.fillna(value=5)
pd.isna(df1)

#operations
df.mean()

s = pd.Series([1, 2, 3, np.nan, 5, 6], index=dates)
s
s.shift(2)#shift down in dataframe order
s.sort_index(ascending=False)
s.sort_index(ascending=False).shift(2)

s = pd.Series([1, 3, 5, np.nan, 6, 8], index=dates).shift(2)
df.sub(s, axis="index")
df.add(df)
df.sub(df)
df.mul(df)
df.div(df)
df.mod(df)
df.pow(df)

df.apply(np.cumsum)
df.apply(lambda x: x.max() - x.min())

s = pd.Series(np.random.randint(0, 7, size=10))
s.value_counts()

s = pd.Series(["A", "B", "C", "Aaba", "Baca", np.nan, "CABA", "dog", "cat"])
s.str.lower()

#merge
df = pd.DataFrame(np.random.randn(10, 4))
pieces = [df[:3], df[3:7], df[7:]]
pd.concat(pieces)

left = pd.DataFrame({"key": ["foo", "bar"], "lval": [1, 2]})
right = pd.DataFrame({"key": ["foo", "bar"], "rval": [4, 5]})
pd.merge(left, right, on="key")

#group
df = pd.DataFrame(
    {
        "A": ["foo", "bar", "foo", "bar", "foo", "bar", "foo", "foo"],
        "B": ["one", "one", "two", "three", "two", "two", "one", "three"],
        "C": np.random.randn(8),
        "D": np.random.randn(8),
    }
)
df.groupby("A").sum()
df.groupby(["A", "B"]).sum()

#reshape
tuples = list(
    zip(
        *[
            ["bar", "bar", "baz", "baz", "foo", "foo", "qux", "qux"],
            ["one", "two", "one", "two", "one", "two", "one", "two"],
        ]
    )
)
index = pd.MultiIndex.from_tuples(tuples, names=["first", "second"])
df = pd.DataFrame(np.random.randn(8, 2), index=index, columns=["A", "B"])
df2 = df[:4]
df2

stacked = df2.stack()
stacked
stacked.index

stacked.unstack()
stacked.unstack(0)
stacked.unstack(1)

df = pd.DataFrame(
    {
        "A": ["one", "one", "two", "three"] * 3,
        "B": ["A", "B", "C"] * 4,
        "C": ["foo", "foo", "foo", "bar", "bar", "bar"] * 2,
        "D": np.random.randn(12),
        "E": np.random.randn(12),
    }
)
df
pd.pivot_table(df, values="D", index=["A", "B"], columns=["C"])

#time series
np.random.randint(1, 10)
np.random.randint(1, 10, 5)
rng = pd.date_range("1/1/2012", periods=100, freq="S")
ts = pd.Series(np.random.randint(0, 500, len(rng)), index=rng)
ts.resample("1Min").sum()
ts.resample("5Min").sum()

rng = pd.date_range("3/6/2012 00:00", periods=5, freq="D")
ts = pd.Series(np.random.randn(len(rng)), rng)
ts

ts_utc = ts.tz_localize("UTC")
ts_utc
ts_utc.tz_convert("US/Eastern")

rng = pd.date_range("1/1/2012", periods=5, freq="M")
ts = pd.Series(np.random.randn(len(rng)), index=rng)
ts

ps = ts.to_period()
ps
ps.to_timestamp()

pd.period_range("1990Q1", "2000Q4", freq="Q-DEC")
prng = pd.period_range("1990Q1", "2000Q4", freq="Q-NOV")
ts = pd.Series(np.random.randn(len(prng)), prng)
ts.index = (prng.asfreq("M", "e") + 1).asfreq("H", "s") + 9
ts.head()

#cate
df = pd.DataFrame(
    {"id": [1, 2, 3, 4, 5, 6], "raw_grade": ["a", "b", "b", "a", "a", "e"]}
)
df["grade"] = df["raw_grade"].astype("category")
df["grade"]
df["grade"].cat.categories = ["good", "very good", "very bad"]
df["grade"] = df["grade"].cat.set_categories(
    ["very bad", "bad", "medium", "good", "very good"]
)
df["grade"]
df.sort_values(by="grade")
df.groupby("grade").size()

#plot
plt.close("all")
ts = pd.Series(np.random.randn(1000), index=pd.date_range("1/1/2000", periods=1000))
ts = ts.cumsum()
ts.plot()
plt.show()

df = pd.DataFrame(
    np.random.randn(1000, 4), index=ts.index, columns=["A", "B", "C", "D"]
)
df = df.cumsum()
plt.figure()
df.plot()
plt.legend(loc='best')
plt.show()
plt.close("all")

#i/o
df.to_csv("foo.csv")
pd.read_csv("foo.csv")
df.to_hdf("foo.h5", "df")
pd.read_hdf("foo.h5", "df")
df.to_excel("foo.xlsx", sheet_name="Sheet1")
pd.read_excel("foo.xlsx", "Sheet1", index_col=None, na_values=["NA"])

ext: py

reason: import numpy


text:
import numpy as np
from sklearn.base import BaseEstimator
from sklearn.utils import check_random_state
from sklearn.utils.validation import check_array, check_is_fitted
from scipy.spatial.distance import cdist

class ThreeWayCMeans(BaseEstimator):
    """ An estimator using Three-Way C-Means algorithm proposed in:
        "Zhang, K.: A three-way c-means algorithm."

    Parameters
    ----------
    n_iter : int, default = 30
        Parameter used as stop criteria in case the fit reaches the
        maximum number of iterations.
    n_clusters : int, default = 10
        Parameter used to indicate the number of centroids.
    init : {'k-means++', 'random'}, default = 'k-means++'
        Parameter used to indicate the centroid initialization method.
    threshold : float, default = 0.9
        Parameter used to know whether assign an object to the lower
        bound of the closest cluster or not.
    m : float, default = 2.0
        Parameter used as fuzzifier to compute membership degrees.
    random_state : int, default = 42
        Parameter used as seed to generate random values.

    """
    def __init__(self, n_iter=30, n_clusters=10, init='k-means++', threshold=0.9, m=2.0, random_state=42):
        self.n_iter = n_iter
        self.n_clusters = n_clusters
        self.init = init
        self.threshold = threshold
        self.m = m
        self.random_state = random_state

    def __cluster_init(self, X):
        """ Initialization of the centroids using the input dataset.

        Parameters
        ----------

        X : array-like with shape (n_samples, n_features)
            The input dataset.

        Returns
        -------
        centroids : array-like with shape (n_cluster, n_features)
            The starting centroids for the execution of K-Means-like algorithm.

        """
        rng = check_random_state(self.random_state)
        centroids = np.zeros(shape=(self.n_clusters, X.shape[1]))
        if self.init == 'k-means++':
            # applying k-means++ method to assign starting centroids
            centroids[0] = np.copy(X[rng.choice(X.shape[0], 1)])
            temp_center = np.tile(centroids[0], (X.shape[0], 1))
            curr_distances = np.linalg.norm(X - temp_center, axis = 1)
            dist = np.array([curr_distances])

            for i in range(1, self.n_clusters):
                # computing the minimum distance from each object to a cluster
                min_distance = np.min(dist, axis = 0)
                if np.equal(np.sum(min_distance), 0.0):
                    centroid_ind = rng.choice(X.shape[0], 1)
                else:
                    # each object has the probability to be chosen depending
                    # on its closest cluster
                    probas = min_distance / np.sum(min_distance)
                    centroid_ind = rng.choice(X.shape[0], 1, p = probas)
                centroids[i] = np.copy(X[centroid_ind])

                # appending distance from each object to the new centroid
                temp_center = np.tile(centroids[i], (X.shape[0], 1))
                curr_distances = np.linalg.norm(X - temp_center, axis = 1)
                dist = np.append(dist, np.array([curr_distances]))
        else:
            # taking randomly k objects as starting centroids
            centroid_ind = rng.choice(X.shape[0], self.n_clusters)
            for i in range(self.n_clusters):
                centroids[i] = np.copy(X[centroid_ind[i]])
        return centroids

    def __assign(self, X, mu):
        """ Implementation of the assignment of X to the available clusters.

        Parameters
        ----------
        X : array-like with shape (n_samples, n_features)
            The input samples to assign.
        mu : array-like with shape(n_samples, n_cluster)
            mu[x][k] containts the membership degree of x-th object to
            k-th cluster.

        Returns
        -------
        y : array-like with shape (n_samples, n_cluster, 2)
            The target values (array of (n_cluster, 2) values for each sample)
            y[x][k][0] = 1.0 if x-th sample is in the k-th cluster's core.
            y[x][k][1] = 1.0 if x-th sample is in the k-th cluster's frontier.

        """
        # contains the maximum (f_j) of relationship vector of each object
        maxim_vec = np.max(mu, axis=1).reshape(X.shape[0], 1)
        # computing ratio membership (g_ij) for each object to each cluster
        ratio_membership = mu / maxim_vec
        # computing belonging matrix (l_ij)
        belongs = np.where(ratio_membership > self.threshold, 1.0, 0.0)
        # performing evaluation function (v_ij) for each object to each cluster
        sum_belongs = np.sum(belongs, axis=1).reshape(X.shape[0], 1)
        evaluation = ratio_membership / sum_belongs
        alpha_j = 1.0
        beta_j = np.full((X.shape[0], self.n_clusters), self.threshold)
        beta_j /= sum_belongs
        difference_a = evaluation - alpha_j
        difference_b = evaluation - beta_j
        # if the difference between the evaluation value and the corresponding
        # beta is positive, then the object is considered in the cluster
        pos = np.where(difference_a >= 0.0, 1.0, 0.0)
        bnd = np.where(difference_b > 0.0, 1.0, 0.0)
        bnd *= (1 - pos)
        y = np.empty(shape=(X.shape[0], self.n_clusters, 2))
        y[:,:,0] = np.copy(pos)
        y[:,:,1] = np.copy(bnd)
        return y

    def __mu(self, X):
        """ Implementation of computing membership degrees for each
            object to each cluster.

        Parameters
        ----------
        X : array-like with shape (n_samples, n_features)
            Input dataset to assign membership degree.

        Returns
        -------
        mu : array-like with shape(n_samples, n_cluster)
            mu[x][k] containts the membership degree of x-th object to
            k-th cluster.

        """
        # calculating distance from each object to each cluster
        dist = cdist(self.cluster_centers_, X)

        # computing mu values as proposed in the paper
        mu = []
        for (ind, center) in zip(range(self.n_clusters), self.cluster_centers_):
            # dist_i contains the distance from each object to (center)
            dist_i = dist[ind].reshape(X.shape[0], 1)
            # corrected distances removing 0 distances
            corr_dist = np.where(np.equal(dist, 0.0),
                                 0.1 ** 10,
                                 dist).T
            # in order to prevent bugs (using dist instead of corr_dist was
            # still dividing by 0 sometimes)
            membership = np.where(np.equal(dist_i, 0.0),
                                  1,
                                  (dist_i / corr_dist) ** (2 / (self.m-1)))
            mu.append(1 / np.sum(membership, axis=1))
        mu = np.array(mu)
        # if distance from the object to the centroid is == 0.0, it must have
        # 1 of membership
        mu = np.where(np.equal(dist, 0.0),
                      1,
                      mu)
        return mu.T

    def __weights(self, X, y, mu):
        """ Implementation of computing weights for each object to each cluster.

        Parameters
        ----------
        X : array-like with shape (n_samples, n_features)
            Input dataset to assign weights.
        y : array-like with shape (n_samples, n_cluster, 2)
            The target values (array of (n_cluster, 2) values for each sample)
            y[x][k][0] = 1.0 if x-th sample is in the k-th cluster's core.
            y[x][k][1] = 1.0 if x-th sample is in the k-th cluster's frontier.
        mu : array-like with shape(n_samples, n_cluster)
            mu[x][k] containts the membership degree of x-th object to
            k-th cluster.

        Returns
        -------
        weights: array-like with shape(n_samples, n_cluster)
            weights[x][k] contains the weight of x-th object being assigned
            to k-th cluster.
        """
        # verifying removing mu that are actually not used (if the cluster
        # c_i is not in B_xj)
        using_mu = mu * np.sum(y, axis=2)
        # computing weights denominators and replicating them (n_cluster) times
        weights_den = np.sum(using_mu, axis = 1).reshape(X.shape[0], 1)
        # computing actual weights
        weights = using_mu / weights_den
        return weights

    def fit(self, X, y=None):
        """ Implementation of the fitting function.

        Parameters
        ----------
        X : array-like with shape (n_samples, n_features)
            The input samples to fit.
        y : object, default = None
            Actually not used, needed for the interface.

        Returns
        -------
        self: object
            Returns the model itself.

        """
        X = check_array(X)
        self.n_features_in_ = X.shape[1]
        stop = False

        # choosing starting centroids
        self.cluster_centers_ = self.__cluster_init(X)

        # assigning objects to each cluster based on starting centroids
        mu = self.__mu(X)
        y_fit = self.__assign(X, mu)
        weights = self.__weights(X, y_fit, mu)
        count_iter = 0
        while stop != True:
            count_iter += 1
            # saving previous centroids computed
            previous_centroids = np.copy(self.cluster_centers_)
            # computing new centroids based on last assignment
            for (center, ind) in zip(self.cluster_centers_, range(self.n_clusters)):
                center_num = np.sum(X * weights[:,ind].reshape(X.shape[0], 1),
                                    axis=0)
                center_den = np.sum(weights[:,ind])
                center = center_num / center_den
                self.cluster_centers_[ind] = center
            # assigning objects to each cluster based on new centroids
            mu = self.__mu(X)
            y_fit = self.__assign(X, mu)
            weights = self.__weights(X, y_fit, mu)
            # stop criteria:
            # - maximum number of iterations reached
            # - equal centroids
            if (count_iter == self.n_iter or
                np.equal(previous_centroids, self.cluster_centers_).all()):
                stop = True
        self.y_ = y_fit
        return self

    def fit_predict(self, X, y=None):
        self.fit(X)
        return self.y_

    def predict(self, X):
        """ Implementation of the prediction function.

        Parameters
        ----------
        X : array-like with shape (n_samples, n_features)
            The input samples to predict.

        Returns
        -------
        y : array-like with shape (n_samples, n_cluster, 2)
            The target values (array of (n_cluster, 2) values for each sample)
            y[x][k][0] = 1.0 if x-th sample is in the k-th cluster's core.
            y[x][k][1] = 1.0 if x-th sample is in the k-th cluster's frontier.

        """
        # checking whether the model is fitted or not
        check_is_fitted(self)

        # input validation
        X = check_array(X)
        mu = self.__mu(X)
        y = self.__assign(X, mu)
        return y

    def predict_proba(self, X):
        """ Implementation of the prediction probabilities function.

        Parameters
        ----------
        X : array-like with shape (n_samples, n_features)
            The input samples to predict.

        Returns
        -------
        y : array-like with shape (n_samples, n_cluster)
            The predicted probabilities (array of n_cluster values for each
            sample) y[x][k] indicates the probability that x-th sample is
            actually in the k-th cluster (using principle of indifference
            from prediction).

        """
        y = self.predict(X)

        y_proba = np.sum(y, axis=2) / np.sum(y, axis=(1,2)).reshape(y.shape[0],1)

        return y_proba

max_stars_repo_name: simonemallei/soft-clustering-models

max_issues_repo_name: simonemallei/soft-clustering-models

max_forks_repo_name: simonemallei/soft-clustering-models

reason: import numpy,from scipy


text:
#!/usr/bin/env python
# Licensed under a 3-clause BSD style license - see LICENSE.rst

# NOTE: The configuration for the package, including the name, version, and
# other information are set in the setup.cfg file. Here we mainly set up
# setup_requires and install_requires since these are determined
# programmatically.

import os

import ah_bootstrap  # noqa

from astropy_helpers.distutils_helpers import is_distutils_display_option
from astropy_helpers.setup_helpers import setup

import astropy  # noqa
min_numpy_version = 'numpy>=' + astropy.__minimum_numpy_version__

if is_distutils_display_option():
    # Avoid installing setup_requires dependencies if the user just
    # queries for information
    setup_requires = []
else:
    setup_requires = [min_numpy_version]

    # Make sure we have the packages needed for building astropy, but do not
    # require them when installing from an sdist as the c files are included
    # there.
    if not os.path.exists(os.path.join(os.path.dirname(__file__), 'PKG-INFO')):
        setup_requires.extend(['cython>=0.21', 'jinja2>=2.7'])

install_requires = [min_numpy_version]

setup(setup_requires=setup_requires,
      install_requires=install_requires)

reason: import astropy,from astropy


text:
import sys
import threading
import numpy
from queue import Queue
import logging
from neil_vst_gui.ui_logging import ProcessLogHandler
from multiprocessing import current_process
from PyQt5 import QtCore

class PlayPluginChain(QtCore.QObject):
    """docstring for PlayPluginChain"""

    progress_signal = QtCore.pyqtSignal(float)
    stop_signal = QtCore.pyqtSignal()

    def __init__(self, **kwargs):
        super().__init__()

        self.device = kwargs.get("device", None)
        self.filename = kwargs.get("filename", None)
        self.vst_host = kwargs.get("vst_host", None)
        self.vst_plugins_chain = kwargs.get("vst_plugins_chain", None)
        self.blocksize = kwargs.get("blocksize", None)
        self.buffersize = kwargs.get("buffersize", None)

        self.log_level = kwargs.get("log_level", logging.INFO)
        _logger = self._logger_init(kwargs.get("pipe", None))
        self.logger = kwargs.get("logger", _logger)

        self.stream = None
        self.stream_wait_data_cnt = 0

        self._is_active = False

    def _fill_queue_buffer(self, data):
        #
        if self.play_event.is_set():
            return

        if self.play_block_index >= self.buffersize:
            if not self.stream.active:
                self.logger.info("Audio stream is STARTING for [ %s ] ..." % self.filename)
                self.stream.start()
            timeout = ((self.blocksize * self.buffersize) / self.vst_host.sample_rate)
            try:
                self.play_queue.put(data, timeout=timeout*2)
                self.progress_signal.emit(self.play_block_index / self.play_max_block_index)
            except Exception as e:
                self.logger.debug("Audio stream buffer is full.")
        else:
            self.play_queue.put_nowait(data)

        self.play_block_index += 1

    def _play_callback(self, outdata, frames, time, status):
        # assert frames == self.blocksize
        if status.output_underflow:
            self.logger.debug('Audio stream buffer underflow: increase blocksize?')
            # raise sounddevice.CallbackAbort
        # assert not status

        try:
            data = self.play_queue.get_nowait()
            if len(data) < len(outdata):
                outdata[:len(data)] = data
                outdata[len(data):] = numpy.zeros( (len(outdata) - len(data), 2), dtype=numpy.float32 )
                # raise sounddevice.CallbackStop
            else:
                outdata[:] = data
        except Exception as e:
            self.logger.debug('Audio stream buffer is empty: increase buffersize?')
            outdata[:] = numpy.zeros( (len(outdata), 2), dtype=numpy.float32 )
            self.stream_wait_data_cnt += 1
            if self.stream_wait_data_cnt >= 10:
                self.stop()
            # raise sounddevice.CallbackAbort

    def _logger_init(self, pipe):
        # Create logger for process and connect it to common pipe
        self.logger = logging.getLogger('Audio Stream')
        self.logger.setLevel( self.log_level )
        # create custom handler with a higher log level
        self.handler = logging.StreamHandler()
        formatter = logging.Formatter( fmt='%(name)s - %(levelname)s - %(message)s', datefmt='%H:%M:%S' )
        self.handler.setFormatter(formatter)
        self.logger.addHandler(self.handler)

    # -------------------------------------------------------------------------

    def start(self, filename, audio_device, channels, vst_host, vst_plugins_chain, start):
        import sounddevice
        import soundfile

        self.play_queue = Queue(maxsize=self.buffersize)
        self.play_event = threading.Event()

        f = soundfile.SoundFile(filename)
        f.close()

        if self.stream is None:
            self.stream = sounddevice.OutputStream(
                samplerate=f.samplerate,
                blocksize=self.blocksize,
                device=audio_device,
                channels=channels,
                dtype='float32',
                callback=self._play_callback,
                finished_callback=self.play_event.set
            )

        self.play_block_index = self.stream_wait_data_cnt = 0
        self.play_max_block_index = f.frames / self.blocksize
        # print("self.play_max_block_index: ", self.play_max_block_index)
        if start > 0:
            start = (f.frames * start) - f.frames
            # print("start pos: %s, max_pos: %s" % (start, f.frames))
        # print("start=", start)

        self._is_active = True

        self.filename = filename
        self.vst_host = vst_host
        self.vst_host.process_chain_start(filename, f.channels, vst_plugins_chain, soundfile.blocks, self._fill_queue_buffer, frames=-1, start=int(start), stop=None)

    def stop(self):
        self.vst_host.process_chain_stop()
        self.stream.stop()
        self._is_active = False
        self.logger.info("Audio stream is STOPED.")
        self.stop_signal.emit()

    def is_active(self):
        return self._is_active

max_stars_repo_name: LeftRadio/py-neil-vst-gui

max_issues_repo_name: LeftRadio/py-neil-vst-gui

max_forks_repo_name: LeftRadio/py-neil-vst-gui


text:
import numpy as np

backtrack = 0

def check_full(p, solution):
    return all(np.array(p) == np.array(solution))

def col(p, i):
    return np.array(p[i::9])

def row(p, i):
    return np.array(p[i * 9:i * 9 + 9])

def section(p, i):
    s = i % 3 * 3 + i // 3 * 27
    return np.array(p[s + 0*0:s + 0*9 + 3] +
                    p[s + 1*9:s + 1*9 + 3] +
                    p[s + 2 * 9:s + 2 * 9 + 3])

def check(p):
    def _check(ck):
        if any(ck == 0):
            return len(np.unique(ck)) - np.count_nonzero(ck) == 1
        else:
            return len(np.unique(ck)) == 9

    for i in range(0, 9):
        if not (_check(col(p, i)) and
                _check(row(p, i)) and
                _check(section(p, i))):
            return False

    return True

def imply(p):
    changed = False
    p = p.copy()

    for i in range(0, 81):
        if p[i] != 0:
            continue
        a = {1, 2, 3, 4, 5, 6, 7, 8, 9}
        row_n = i // 9
        col_n = i % 9
        section_n = (row_n // 3) * 3 + col_n // 3

        prob = a.difference(row(p, row_n)) \
            .difference(col(p, col_n)) \
            .difference(section(p, section_n))
        if len(prob) == 1:
            changed = True
            p[i] = next(iter(prob))

    if changed:
        return imply(p)
    return p

def _solve(p, loc):
    global backtrack
    if not check(p):
        return None
    if loc >= 81:
        return p
    if p[loc] != 0:
        return _solve(p, loc+1)

    p_i = p.copy()
    for i in range(1, 10):
        p_i[loc] = i
        sub_p = _solve(imply(p_i), loc+1)
        if sub_p is not None:
            return sub_p
        backtrack += 1
    return None

def solve(puzzle, solution):
    global backtrack

    backtrack = 0
    puzzle = imply([int(x) for x in puzzle])
    solution = [int(x) for x in solution]

    sol = _solve(puzzle, 0)
    if check_full(sol, solution):
        return backtrack
    return None

# "Worlds Hardest" sudoku: https://www.telegraph.co.uk/news/science/science-news/9359579/Worlds-hardest-sudoku-can-you-crack-it.html
puzzle = imply(
    [int(x) for x in "800000000003600000070090200050007000000045700000100030001000068008500010090000400"])
print(np.reshape(_solve(puzzle, 0), (9, 9)))
print(backtrack)

# Reads in csv of puzzle/solution pairs from kaggle: https://www.kaggle.com/bryanpark/sudoku/discussion

# i = 0
# with open("sudoku.csv") as csv:
#     csv.readline()  # Headers
#     t = 0
#     mn = 99999
#     mx = -1
#     while True:  # and i < 500:
#         print("\r"+str(i), end="")
#         i += 1
#         if i < 5000:
#             continue
#         line = csv.readline()
#         if not line:
#             break
#         p_s = line.split(',')
#         s = solve(p_s[0].strip(), p_s[1].strip())
#         t += s
#         mn = min(mn, s)
#         mx = max(mx, s)
#         if i % 250 == 0:
#             print("\nMean: ", t / i)
#             print("Min: ", mn)
#             print("Max: ", mx)


text:
import numpy as np
import math
import time
import normalize

def complex_normalize1(complex_arr: np.ndarray):
    print("complex_normalize1")

    def f(arr: np.ndarray):
        sum_of_squares: float = 0
        for c in arr:
            sum_of_squares += abs(c) ** 2

        norm = math.sqrt(sum_of_squares)
        return arr / norm

    timeit(lambda: [f(complex_arr) for _ in range(int(10))])

def complex_normalize1_cython(complex_arr: np.ndarray):
    print("complex_normalize1_cython")

    timeit(lambda: [normalize.c_normalize1(complex_arr) for _ in range(int(10))])

def complex_normalize2(complex_arr: np.ndarray):
    print("complex_normalize2")

    def f(arr: np.ndarray):
        sum_of_squares: float = (np.abs(arr) ** 2).sum()
        norm = math.sqrt(sum_of_squares)
        return arr / norm

    timeit(lambda: [f(complex_arr) for _ in range(int(10))])

def complex_normalize2_cython(complex_arr: np.ndarray):
    print("complex_normalize2_cython")

    timeit(lambda: [normalize.c_normalize2(complex_arr) for _ in range(int(10))])

def complex_normalize3(complex_arr: np.ndarray):
    print("complex_normalize3")

    def f(arr: np.ndarray):
        sum_of_squares: float = np.linalg.norm(arr, ord=2, axis=-1, keepdims=True)
        norm = math.sqrt(sum_of_squares)
        return arr / norm

    timeit(lambda: [f(complex_arr) for _ in range(int(10))])

def timeit(callback) -> None:
    """
    時間を計測する関数
    :param callback:
    :param r:
    :return:
    """
    start_time = time.perf_counter()

    callback()

    end_time = time.perf_counter()
    elapsed_time = end_time - start_time
    print("elapsed_time: {0}".format(elapsed_time) + "[sec]")

if __name__ == "__main__":
    c1 = np.array([
        1.0, 1.0, 1.0, 1.0
    ], dtype=np.complex128)
    complex_normalize1(c1)
    complex_normalize1_cython(c1)
    complex_normalize2(c1)
    complex_normalize2_cython(c1)
    complex_normalize3(c1)

    c2 = np.random.randn(int(1e6)) + 1j * np.random.randn(int(1e6))
    complex_normalize1(c2)
    complex_normalize1_cython(c2)
    complex_normalize2(c2)
    complex_normalize2_cython(c2)
    complex_normalize3(c2)


text:
import os

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
import numpy as np
import time
import pandas as pd
import tensorflow as tf
from pathlib import Path
from sklearn.model_selection import StratifiedKFold
from sklearn import svm
import utils.gen_utils as utils
import utils.dataset_processors as dataset_processors
import utils.linguistic_features_utils as feature_utils

def get_inputs(dataset):
    """ Read data from metafeature files and prepare for training. """
    if dataset == 'essays':
        dump_data = dataset_processors.load_essays_df('../data/essays/essays.csv')
        trait_labels = ['EXT', 'NEU', 'AGR', 'CON', 'OPN']
    elif dataset == 'kaggle':
        dump_data = dataset_processors.load_Kaggle_df('../data/kaggle/kaggle.csv')
        trait_labels = ['E', 'N', 'F', 'J']
    print('dataset loaded! Getting psycholinguistic features...')
    inputs, full_targets, _, _ = feature_utils.get_psycholinguist_data(dump_data, dataset, feature_flags)
    inputs = np.array(inputs)
    full_targets = np.array(full_targets)

    return inputs, full_targets, trait_labels

def classification(X_train, X_test, y_train, y_test, file_name):
    """  Run classification algorithm (SVM)  """
    """ (commented lines can save SVM model) """
    classifier = svm.SVC(gamma="scale")
    classifier.fit(X_train, y_train)
    # model_name = file_name + '.joblib'
    # joblib.dump(classifier, model_name)
    acc = classifier.score(X_test, y_test)
    return acc

def training(dataset, inputs, full_targets, trait_labels):
    """ Train model for each trait on 10-fold corss-validtion. """
    n_splits = 10
    expdata = {}
    expdata['acc'], expdata['trait'], expdata['fold'] = [], [], []

    for trait_idx in range(full_targets.shape[1]):
        # convert targets to one-hot encoding
        targets = full_targets[:, trait_idx]
        expdata['trait'].extend([trait_labels[trait_idx]] * n_splits)
        expdata['fold'].extend(np.arange(1, n_splits + 1))

        skf = StratifiedKFold(n_splits=n_splits, shuffle=False)
        k = -1
        for train_index, test_index in skf.split(inputs, targets):
            x_train, x_test = inputs[train_index], inputs[test_index]
            y_train, y_test = targets[train_index], targets[test_index]
            k += 1
            acc = classification(x_train, x_test, y_train, y_test,
                                 'SVM-' + dataset + '-' + embed + '-' + str(k) + "_t" + str(trait_idx))
            print(acc)
            expdata['acc'].append(acc)
    print(expdata)
    df = pd.DataFrame.from_dict(expdata)

    return df

def logging(df, log_expdata=True):
    """ Save results and each models config and hyper parameters."""
    df['network'], df['dataset'], df['lr'], df['batch_size'], df['epochs'], df['model_input'], df['embed'], df['layer'], \
    df['mode'], df['embed_mode'], df['jobid'] = network, \
                                                dataset, lr, batch_size, epochs, MODEL_INPUT, embed, layer, mode, embed_mode, jobid

    pd.set_option('display.max_columns', None)
    print(df.head(5))

    # save the results of our experiment
    if (log_expdata):
        Path(path).mkdir(parents=True, exist_ok=True)
        if (not os.path.exists(path + 'expdata.csv')):
            df.to_csv(path + 'expdata.csv', mode='a', header=True)
        else:
            df.to_csv(path + 'expdata.csv', mode='a', header=False)

if __name__ == "__main__":
    inp_dir, dataset, lr, batch_size, epochs, log_expdata, embed, layer, mode, embed_mode, jobid = utils.parse_args()
    features_dim = 123
    layer = ''
    path = 'explogs/'
    n_classes = 2
    MODEL_INPUT = 'psycholinguist_features'
    network = 'SVM'
    print(network)

    nrc, nrc_vad, readability, mairesse = [True, True, True, True]
    feature_flags = [nrc, nrc_vad, readability, mairesse]

    np.random.seed(jobid)
    tf.random.set_seed(jobid)

    start = time.time()
    inputs, full_targets, trait_labels = get_inputs(dataset)
    print('starting k-fold cross validation...')

    df = training(dataset, inputs, full_targets, trait_labels)
    logging(df, log_expdata)

max_stars_repo_name: jtfields/personality-prediction

max_issues_repo_name: jtfields/personality-prediction

max_forks_repo_name: jtfields/personality-prediction


text:
r"""Continuous linear change"""
import numpy as np

from ruptures.base import BaseCost
from ruptures.costs import NotEnoughPoints

class CostCLinear(BaseCost):

    r"""Piecewise linear approximation with a continuity constraint."""

    model = "clinear"

    def __init__(self):
        """Initialize the object."""
        self.signal = None
        self.min_size = 3

    def fit(self, signal) -> "CostCLinear":
        """Set parameters of the instance.

        Args:
            signal (array): signal of shape (n_samples, n_dims) or (n_samples,)

        Returns:
            self
        """
        if signal.ndim == 1:
            self.signal = signal.reshape(-1, 1)
        else:
            self.signal = signal

        return self

    def error(self, start, end) -> float:
        """Return the approximation cost on the segment [start:end].

        Args:
            start (int): start of the segment
            end (int): end of the segment

        Returns:
            segment cost (float)

        Raises:
            NotEnoughPoints: when the segment is too short (less than `min_size`
                samples).
        """
        if end - start < self.min_size:
            raise NotEnoughPoints

        if start == 0:
            start = 1

        sub = self.signal[start:end]
        slope = (self.signal[end - 1] - self.signal[start - 1]) / (end - start)
        intercept = self.signal[start - 1]
        approx = slope.reshape(-1, 1) * np.arange(
            1, end - start + 1
        ) + intercept.reshape(-1, 1)
        return np.sum((sub - approx.transpose()) ** 2)

max_stars_repo_name: antoinedemathelin/ruptures

max_issues_repo_name: antoinedemathelin/ruptures

max_forks_repo_name: antoinedemathelin/ruptures


text:
# -*- coding: utf-8 -*-
"""
Sideplots and Moments
=====================
An example showing how to use moments and sideplots.
"""

import WrightTools as wt
import numpy as np
import matplotlib.pyplot as plt

def S(x):
    return (1 + np.exp(-1 * x)) ** -1

# instantiate data object
d1 = np.linspace(-1, 3, 101)[:, None]
d2 = np.linspace(-1, 3, 102)[None, :]
arr = S(10 * (d1 + 0.5)) * S(10 * d2) * S(-2 * d1) + 0.1 * S(5 * d1) * S(5 * (d2 - 1))
d = wt.Data(name="test")
d.create_variable("d1", values=d1, units="ps", label="1")
d.create_variable("d2", values=d2, units="ps", label="2")
d.create_channel("z", values=arr)
d.transform("d1", "d2")

# calculate moments
d.moment(axis="d1", channel=0, moment=0)  # integral
d["z_d1_moment_0"].normalize()
d.moment(axis="d1", channel=0, moment=1)  # center of mass
d.moment(axis="d2", channel=0, moment=0)  # integral
d["z_d2_moment_0"].normalize()
d.moment(axis="d2", channel=0, moment=1)  # center of mass
# create figure
fig, gs = wt.artists.create_figure(cols=[1, "cbar"])
ax = plt.subplot(gs[0])
# instantiate sideplots
axcorrx = ax.add_sideplot(along="x", pad=0.1, ymin=-0.1, ymax=1.1)
axcorry = ax.add_sideplot(along="y", pad=0.1, ymin=-0.1, ymax=1.1)
# plot data
ax.pcolor(d, autolabel="both")
# plot integral moments in sideplot
axcorrx.plot(d, channel="z_d2_moment_0", color="k", linewidth=3)
# this sideplot is uncouth.
# the independent axis is 'y' and the dependent is 'x'
y_ = d.d2.points
x_ = d["z_d1_moment_0"].points
axcorry.plot(x_, y_, color="k", linewidth=3)
# plot center of mass for each slice
ax.plot(d["z_d1_moment_1"].points, d.d2.points, label="COM along y")
ax.plot(d.d1.points, d["z_d2_moment_1"].points, label="COM along x")
ax.legend(fontsize=12, loc=4)
# grids
for ax_ in [ax, axcorrx, axcorry]:
    ax_.grid()
# plot colorbar
cax = plt.subplot(gs[-1])
wt.artists.plot_colorbar(cax=cax, label="amplitude")

max_stars_repo_name: untzag/WrightTools

max_issues_repo_name: untzag/WrightTools

max_forks_repo_name: untzag/WrightTools


text:
# -*- coding: utf-8 -*-
"""
Created on Mon Oct 15 01:21:15 2018

@author: chint
"""
# Logistic regression for detecting cat vs non-cat images
import numpy as np
import h5py
import helper_functions_nn as hf
import argparse
from load_data import load_data

parser = argparse.ArgumentParser(description="Run commands")
parser.add_argument('-t', '--train', default=True, type=bool,
                    help="Training mode, Train if true, test if false")
parser.add_argument('-l', '--layer_dims', type=str, required=True,
                    help="Model architecture parameters without input dimensions")
parser.add_argument('-m', '--model', default='',
                    help="Model name if pretrained model")
parser.add_argument('-a', '--activations', type=str, required=True,
                    help='Activation functions for each layer')

args = parser.parse_args()

is_train = args.train
architecture = [int(i) for i in (args.layer_dims).split(',')]
activations = (args.activations).split(',')
print(activations)

train_x, train_y, test_x, test_y, classes = load_data()
train_x_flatten = train_x.reshape(train_x.shape[0], -1).T
test_x_flatten = test_x.reshape(test_x.shape[0], -1).T

train_x = train_x_flatten/255.0
test_x = test_x_flatten/255.0

layer_dims = [train_x.shape[0]] + architecture

if is_train and args.model == '':
    parameters = hf.train(train_x, train_y, layer_dims, activations, print_cost=True, plot_cost=True)
    hf.save_model(parameters)

if args.model != '':
    parameters = hf.get_model(args.model)

train_acc = hf.predict(train_x, train_y, parameters, activations)
test_acc = hf.predict(test_x, test_y, parameters, activations)


text:
# -*- coding: utf-8 -*-
"""
Es 3
"""

import matplotlib.pyplot as plt
import numpy as np
import sympy as sym
import allM as fz
import math
import Funzioni_Integrazione as fzz

scelta = input("Seleziona integrale ")
x = sym.Symbol("x")
alpha = np.array([13./2., 5./2., 1./2.])
integrali = {
    "1" : [sym.cos(x), 0., 2.],
    "2" : [x * sym.exp(x) * sym.cos(x**2), 2 * -math.pi, 0],
    "3" : [sym.Pow(sym.sin(x), alpha[0]), 0, math.pi/2],
    "4" : [sym.Pow(sym.sin(x), alpha[1]), 0, math.pi/2],
    "5" : [sym.Pow(sym.sin(x), alpha[2]), 0, math.pi/2]
        }
fx, a, b = integrali.get(scelta)
Iesatto = float(sym.integrate(fx, (x,a,b)))
f = sym.utilities.lambdify(x, fx, np)
errRelT = []
errRelS = []
nIntT = []
nIntS = []
kT = []
kS = []
nValT = []
nValS = []
for k in range(4, 11):
    print(k)
    tol = (10.0 ** (-k))
    It, Nt = fz.trapTol(f, a, b, tol)
    Is, Ns = fz.simpTol(f, a, b, tol)
    if Nt > 0:
        nIntT.append(Nt)
        errRelT.append(abs(It - Iesatto) / abs(Iesatto))
        kT.append(k)
        nValT.append(Nt + 1)
    if Ns > 0:
        nIntS.append(Ns)
        errRelS.append(abs(Is - Iesatto) / abs(Iesatto))
        kS.append(k)
        nValS.append(2 * Ns + 1)
plt.semilogy(kT, errRelT, "b-o", kS, errRelS, "r-*")
plt.legend(["ErrRelTrap", "ErrRelSimp"])
plt.show()
plt.plot(kT, nIntT, "b-o", kS, nIntS, "r-*")
plt.legend(["NSottIntTrap", "NSottIntSimp"])
plt.show()
plt.plot(kT, nValT, "b-o", kS, nValS, "r-*")
plt.legend(["NValTrap", "NValSimp"])
plt.show()

reason: import numpy,import sympy


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys
import scipy as sp
import numpy as np
import pandas as pd
import chart_studio.plotly as py
import cufflinks as cf

# define a global dict to store items that not working pickled
pickling = {}

# simple return sys version
def version():
    return sys.version

# common helper for plotting functions in plotly
# using cufflinks, wrapped over erlang,
# return the url holding the plot to use in the client
def jun_iplot_dataframe(df, key, keywords=[]):
    if ( isinstance(df, pd.core.frame.DataFrame) ):
        # make dict from keywords even if empty!
        kwargs = dict(keywords)
        # IMPORTANT: check if columns has the x and y, otherwise remove to plot
        x = kwargs.get('x')
        y = kwargs.get('y')
        columns = list(df)
        if x not in columns:
          del kwargs['x']
        if y not in columns:
          del kwargs['y']
        # explicity execute the fun
        iplot = df.iplot(**kwargs)
        pickling[key] = iplot
        # use get data since this should be achive as figure
        return ('plotly.iplot', key)
    else:
        return 'error_format_data_frame_invalid'

def jun_iplot_plot(iplot, filename, keywords=[]):
    iplot = pickling[iplot]
    url = py.plot(iplot, filename=filename, auto_open=False)
    return url

def jun_iplot_extend(iplot_x, iplot_y, keywords=[]):
    iplot_yy = pickling[iplot_y]
    iplot_xx = pickling[iplot_x]
    iplot_yy.add_trace(iplot_xx['data'][0])
    pickling[iplot_y] = iplot_yy
    return ('plotly.iplot', iplot_y)

def jun_iplot_get_figure(url, key, keywords=[]):
    iplot = py.get_figure(url)
    pickling[key] = iplot
    return ('plotly.iplot', key)

max_stars_repo_name: prateek-rtk/jun

max_issues_repo_name: prateek-rtk/jun

max_forks_repo_name: prateek-rtk/jun

reason: import numpy,import scipy


text:
import numpy as np
import gym
from gym import spaces

from .locobot_interface import PybulletInterface

import pprint

class LocobotBaseEnv(gym.Env):
    metadata = {
        'render.modes': ['human', 'rgb_array'],
        'video.frames_per_second': 50
    }

    def __init__(self, **params):
        self.interface = PybulletInterface(**params)
        self.params = self.interface.params

        print()
        print("LocobotBaseEnv params:")
        pprint.pprint(dict(
            self=self,
            **self.params
        ))
        print()

        if "observation_space" in params:
            self.observation_space = params["observation_space"]
        else:
            observation_type = params["observation_type"]
            if observation_type == "image" and params.get("baselines", False):
                self.observation_space = spaces.Box(low=0, high=1., shape=(params["image_size"], params["image_size"], 3))
            else:
                s = {}
                if observation_type == "image":
                    # pixels taken care of by PixelObservationWrapper
                    pass
                elif observation_type == "state":
                    observation_high = np.ones(params["state_dim"])
                    s['state'] = spaces.Box(-observation_high, observation_high)
                else:
                    raise ValueError("Unsupported observation_type: " + str(params["observation_type"]))
                self.observation_space = spaces.Dict(s)

        if "action_space" in params:
            self.action_space = params["action_space"]
        else:
            self.action_dim = params["action_dim"]
            action_high = np.ones(self.action_dim)
            self.action_space = spaces.Box(-action_high, action_high)

        self.max_ep_len = self.params["max_ep_len"]
        self.num_steps = 0

    def render(self, mode='rgb_color', close=False):
        return self.interface.render_camera()

    def get_pixels(self, *args, **kwargs):
        return self.render(*args, **kwargs)


text:
import pandas as pd
import numpy as np
from scipy.stats import poisson
import os.path

file_path = os.path.dirname(__file__)
alphabeta = pd.read_csv(f'{file_path}/../output/alphabet.csv', index_col=[0])

class Match:
    def __init__(self, home_team, away_team, knockout=False):
        self.home_team = home_team
        self.away_team = away_team
        self.knockout = knockout

        self.home_alpha = self.get_home_alpha()
        self.home_beta = self.get_home_beta()
        self.away_alpha = self.get_away_alpha()
        self.away_beta = self.get_away_beta()

        self.home_xg = self.get_home_xg()
        self.away_xg = self.get_away_xg()

        self.penalties = [0, 0]
        self.result = self.simulate(num_sims=1)
        self.winner = [self.home_team, self.away_team][self.result.index(max(self.result))]

    def __str__(self):
        return f"{self.home_team:15} - {self.away_team:15}"

    def get_home_alpha(self) -> float:
        return alphabeta.loc[self.home_team, 'alpha']

    def get_home_beta(self) -> float:
        return alphabeta.loc[self.home_team, 'beta']

    def get_away_alpha(self) -> float:
        return alphabeta.loc[self.away_team, 'alpha']

    def get_away_beta(self) -> float:
        return alphabeta.loc[self.away_team, 'beta']

    def get_home_xg(self) -> float:
        home_xg = self.home_alpha * self.away_beta
        return home_xg

    def get_away_xg(self) -> float:
        away_xg = self.home_beta * self.away_alpha
        return away_xg

    def get_prob_home(self) -> float:
        mu_1 = self.home_xg
        mu_2 = self.away_xg

        home = sum(poisson.pmf(k, mu_1) * poisson.cdf(k - 1, mu_2) for k in range(10))
        return home

    def get_prob_draw(self) -> float:
        mu_1 = self.home_xg
        mu_2 = self.away_xg

        draw = sum(poisson.pmf(k, mu_1) * poisson.pmf(k, mu_2) for k in range(10))
        return draw

    def get_prob_away(self) -> float:
        mu_1 = self.home_xg
        mu_2 = self.away_xg

        away = sum(poisson.pmf(k, mu_2) * poisson.cdf(k - 1, mu_1) for k in range(10))
        return away

    def simulate_home(self, num_sims=1, extra_time=False) -> int:
        mu_1 = self.home_xg
        if extra_time:
            mu_1 /= 3
        home_goals = np.random.poisson(mu_1, num_sims)
        if num_sims == 1:
            home_goals = int(home_goals)
        return home_goals

    def simulate_away(self, num_sims=1, extra_time=False) -> int:
        mu_2 = self.away_xg
        if extra_time:
            mu_2 /= 3
        away_goals = np.random.poisson(mu_2, num_sims)
        if num_sims == 1:
            away_goals = int(away_goals)
        return away_goals

    def simulate(self, num_sims=1) -> list:
        """
        Simulate the number of home and away goals scored in a match.
        If the score is equal and the match in in the knockout stage,
        first play an extra 30 minutes and if the score remains equal,
        do a penalty shootout
        :param num_sims: the number of times to simulate the match
        :return: a list containing the total number of home and away goals, respectively
        """
        home = self.simulate_home(num_sims)
        away = self.simulate_away(num_sims)

        if self.knockout:
            # Simulate 30 minutes of a match for knockout matches that end equal
            # after 90 minutes
            home_extra = (home == away) * self.simulate_home(num_sims, extra_time=True)
            away_extra = (home == away) * self.simulate_away(num_sims, extra_time=True)

            home += home_extra
            away += away_extra

            # If after extra time the scores are still level, a penalty shootout
            # will decide the winner
            if num_sims > 1 or home == away:
                penalties = self.penalty_shootout(num_sims)
                home_penalties = (home == away) * penalties[self.home_team]
                away_penalties = (home == away) * penalties[self.away_team]

                home += home_penalties
                away += away_penalties

        return [home, away]

    def probabilities(self):
        """Compute the probabilities for home win, draw or away win and return as dict"""
        mu_1 = self.home_xg
        mu_2 = self.away_xg

        home = sum(poisson.pmf(k, mu_1) * poisson.cdf(k - 1, mu_2) for k in range(10))
        draw = sum(poisson.pmf(k, mu_1) * poisson.pmf(k, mu_2) for k in range(10))
        away = sum(poisson.pmf(k, mu_2) * poisson.cdf(k - 1, mu_1) for k in range(10))

        return {'home': home, 'draw': draw, 'away': away}

    def penalty_shootout(self, num_sims) -> dict:
        """
        Simulate a penalty shootout until there is a large enough difference
        to conclude a winner
        :param num_sims: number of shootouts to simulate
        :return: a dictionary containing for both teams the number of penalties scored
        """
        home_pens = np.zeros(num_sims)
        away_pens = np.zeros(num_sims)

        for sim in range(num_sims):
            penalty_nr = 0
            pens_diff = 0

            # Simulate the first five penalties until there is a large enough
            # difference between the number scored
            while penalty_nr + pens_diff <= 5 and penalty_nr < 5:
                home_pens[sim] += np.random.binomial(p=0.75, n=1)
                away_pens[sim] += np.random.binomial(p=0.75, n=1)

                pens_diff = abs(home_pens[sim] - away_pens[sim])
                penalty_nr += 1

            # Simulate round by round if the number of penalties scored
            # is level after five penalty kicks
            while pens_diff == 0:
                home_pens[sim] += np.random.binomial(p=0.75, n=1)
                away_pens[sim] += np.random.binomial(p=0.75, n=1)

                pens_diff = abs(home_pens[sim] - away_pens[sim])
                penalty_nr += 1

        if num_sims == 1:
            home_pens, away_pens = int(home_pens), int(away_pens)
            self.penalties = [home_pens, away_pens]

        result = {self.home_team: home_pens, self.away_team: away_pens}

        return result

    def print_stats(self):
        """Print the expected goals and win probabilities"""
        probs = self.probabilities()
        prob_home, prob_draw, prob_away = probs['home'], probs['draw'], probs['away']
        stats_string = (
            f"{self}\n"
            f"Home goals: {round(self.home_xg, 2)}\n"
            f"Away goals: {round(self.away_xg, 2)}\n"
            f"Prob home: {round(prob_home, 2)}\n"
            f"Prob draw: {round(prob_draw, 2)}\n"
            f"Prob away: {round(prob_away, 2)}\n")

        print(stats_string)

        return stats_string

    def print_result(self):
        """Print the result of the Match in the format
        'home_team - away_team: 2 - 3' or
        'home_team - away_team: 2 - 2 (4 - 3)' in case
        of a penalty shootout
        """
        if sum(self.penalties) == 0:
            result_string = (
                f"{self.home_team:15} - {self.away_team:15}: "
                f"{' - '.join([str(res) for res in self.result])}"
            )
        else:
            home_result = self.result[0] - self.penalties[0]
            away_result = self.result[1] - self.penalties[1]
            home_pens, away_pens = self.penalties[0], self.penalties[1]

            result_string = (
                f"{self.home_team:15} - {self.away_team:15}: "
                f"{home_result} - {away_result} ({home_pens} - {away_pens})"
            )

        print(result_string)

max_stars_repo_name: YpeZ/EK-Voetbal

max_issues_repo_name: YpeZ/EK-Voetbal

max_forks_repo_name: YpeZ/EK-Voetbal


text:
# The MIT License (MIT)
# Copyright (c) 2021 by the xcube development team and contributors
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of
# this software and associated documentation files (the "Software"), to deal in
# the Software without restriction, including without limitation the rights to
# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
# of the Software, and to permit persons to whom the Software is furnished to do
# so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import functools
import json
import math
from typing import Dict, Tuple, List, Set

import numpy as np

from xcube.constants import LOG
from xcube.core.geom import get_dataset_bounds
from xcube.core.normalize import DatasetIsNotACubeError
from xcube.core.timecoord import timestamp_to_iso_string
from xcube.util.assertions import assert_instance
from xcube.util.cmaps import get_cmaps
from xcube.util.tilegrid import TileGrid
from xcube.webapi.auth import assert_scopes
from xcube.webapi.auth import check_scopes
from xcube.webapi.context import ServiceContext
from xcube.webapi.controllers.places import GeoJsonFeatureCollection
from xcube.webapi.controllers.tiles import get_dataset_tile_url
from xcube.webapi.controllers.tiles import get_tile_source_options
from xcube.webapi.errors import ServiceBadRequestError

def get_datasets(ctx: ServiceContext,
                 details: bool = False,
                 client: str = None,
                 point: Tuple[float, float] = None,
                 base_url: str = None,
                 granted_scopes: Set[str] = None) -> Dict:
    granted_scopes = granted_scopes or set()

    dataset_configs = list(ctx.get_dataset_configs())

    dataset_dicts = list()
    for dataset_config in dataset_configs:

        ds_id = dataset_config['Identifier']

        if dataset_config.get('Hidden'):
            continue

        if 'read:dataset:*' not in granted_scopes:
            required_scopes = ctx.get_required_dataset_scopes(dataset_config)
            is_substitute = dataset_config \
                .get('AccessControl', {}) \
                .get('IsSubstitute', False)
            if not check_scopes(required_scopes,
                                granted_scopes,
                                is_substitute=is_substitute):
                continue

        dataset_dict = dict(id=ds_id)

        dataset_dict['title'] = ds_id
        if 'Title' in dataset_config:
            ds_title = dataset_config['Title']
            if ds_title and isinstance(ds_title, str):
                dataset_dict['title'] = ds_title

        if 'BoundingBox' in dataset_config:
            ds_bbox = dataset_config['BoundingBox']
            if ds_bbox \
                    and len(ds_bbox) == 4 \
                    and all(map(lambda c: isinstance(c, float)
                                          or isinstance(c, int),
                                ds_bbox)):
                dataset_dict['bbox'] = ds_bbox

        dataset_dicts.append(dataset_dict)

        # Important note:
        # the "point" parameter is used by
        # the CyanoAlert app only

        if details or point:
            filtered_dataset_dicts = []
            for dataset_dict in dataset_dicts:
                ds_id = dataset_dict["id"]
                try:
                    if point:
                        ds = ctx.get_dataset(ds_id)
                        if "bbox" not in dataset_dict:
                            dataset_dict["bbox"] = list(get_dataset_bounds(ds))
                    if details:
                        dataset_dict.update(
                            get_dataset(ctx, ds_id, client,
                                        base_url,
                                        granted_scopes=granted_scopes)
                        )
                    filtered_dataset_dicts.append(dataset_dict)
                except (DatasetIsNotACubeError, CubeIsNotDisplayable) as e:
                    LOG.warn(f'skipping dataset {ds_id}: {e}')
            dataset_dicts = filtered_dataset_dicts
    if point:
        is_point_in_dataset_bbox = functools.partial(
            _is_point_in_dataset_bbox, point
        )
        # noinspection PyTypeChecker
        dataset_dicts = list(filter(is_point_in_dataset_bbox, dataset_dicts))

    return dict(datasets=dataset_dicts)

def get_dataset(ctx: ServiceContext,
                ds_id: str,
                client=None,
                base_url: str = None,
                granted_scopes: Set[str] = None) -> Dict:
    granted_scopes = granted_scopes or set()

    dataset_config = ctx.get_dataset_config(ds_id)
    ds_id = dataset_config['Identifier']

    if 'read:dataset:*' not in granted_scopes:
        required_scopes = ctx.get_required_dataset_scopes(dataset_config)
        assert_scopes(required_scopes, granted_scopes or set())

    ml_ds = ctx.get_ml_dataset(ds_id)
    grid_mapping = ml_ds.grid_mapping
    if not grid_mapping.crs.is_geographic:
        raise CubeIsNotDisplayable(f'CRS is not geographic:'
                                   f' {grid_mapping.crs.srs}')
    if not math.isclose(grid_mapping.x_res,
                        grid_mapping.y_res,
                        abs_tol=0.01):
        raise CubeIsNotDisplayable(f'spatial resolutions are'
                                   f' different in x, y:'
                                   f' {grid_mapping.x_res}'
                                   f' and {grid_mapping.y_res}')
    try:
        # Make sure we have a valid tile grid
        # noinspection PyStatementEffect
        assert_instance(ml_ds.tile_grid, TileGrid)
    except ValueError as e:
        raise CubeIsNotDisplayable(f'could not create tile grid: {e}')

    ds = ml_ds.get_dataset(0)

    ds_title = dataset_config.get('Title',
                                  ds.attrs.get('title',
                                               ds.attrs.get('name',
                                                            ds_id)))
    dataset_dict = dict(id=ds_id, title=ds_title)

    if "bbox" not in dataset_dict:
        dataset_dict["bbox"] = list(get_dataset_bounds(ds))

    variable_dicts = []
    for var_name in ds.data_vars:
        var = ds.data_vars[var_name]
        dims = var.dims
        if len(dims) < 3 \
                or dims[0] != 'time' \
                or dims[-2] != 'lat' \
                or dims[-1] != 'lon':
            continue

        if 'read:variable:*' not in granted_scopes:
            required_scopes = ctx.get_required_variable_scopes(dataset_config,
                                                               var_name)
            if not check_scopes(required_scopes, granted_scopes):
                continue

        variable_dict = dict(id=f'{ds_id}.{var_name}',
                             name=var_name,
                             dims=list(dims),
                             shape=list(var.shape),
                             dtype=str(var.dtype),
                             units=var.attrs.get('units', ''),
                             title=var.attrs.get('title',
                                                 var.attrs.get('long_name',
                                                               var_name)))

        if client is not None:
            tile_grid = ctx.get_tile_grid(ds_id)
            tile_xyz_source_options = get_tile_source_options(
                tile_grid,
                get_dataset_tile_url(ctx, ds_id,
                                     var_name,
                                     base_url),
                client=client
            )
            variable_dict["tileSourceOptions"] = tile_xyz_source_options

        cmap_name, (cmap_vmin, cmap_vmax) = ctx.get_color_mapping(ds_id,
                                                                  var_name)
        variable_dict["colorBarName"] = cmap_name
        variable_dict["colorBarMin"] = cmap_vmin
        variable_dict["colorBarMax"] = cmap_vmax

        if hasattr(var.data, '_repr_html_'):
            # noinspection PyProtectedMember
            variable_dict["htmlRepr"] = var.data._repr_html_()

        variable_dict["attrs"] = {
            key: ("NaN" if isinstance(value, float)
                           and np.isnan(value) else value)
            for key, value in var.attrs.items()
        }

        variable_dicts.append(variable_dict)

    dataset_dict["variables"] = variable_dicts

    rgb_var_names, rgb_norm_ranges = ctx.get_rgb_color_mapping(ds_id)
    if any(rgb_var_names):
        rgb_schema = {'varNames': rgb_var_names, 'normRanges': rgb_norm_ranges}
        if client is not None:
            tile_grid = ctx.get_tile_grid(ds_id)
            tile_xyz_source_options = get_tile_source_options(
                tile_grid,
                get_dataset_tile_url(ctx, ds_id,
                                     'rgb',
                                     base_url),
                client=client
            )
            rgb_schema["tileSourceOptions"] = tile_xyz_source_options
        dataset_dict["rgbSchema"] = rgb_schema

    dim_names = ds.data_vars[list(ds.data_vars)[0]].dims \
        if len(ds.data_vars) > 0 else ds.dims.keys()
    dataset_dict["dimensions"] = [
        get_dataset_coordinates(ctx, ds_id, dim_name)
        for dim_name in dim_names
    ]

    dataset_dict["attrs"] = {
        key: ds.attrs[key]
        for key in sorted(list(ds.attrs.keys()))
    }

    dataset_attributions = dataset_config.get(
        'DatasetAttribution',
        ctx.config.get('DatasetAttribution')
    )
    if dataset_attributions is not None:
        if isinstance(dataset_attributions, str):
            dataset_attributions = [dataset_attributions]
        dataset_dict['attributions'] = dataset_attributions

    place_groups = ctx.get_dataset_place_groups(ds_id, base_url)
    if place_groups:
        dataset_dict["placeGroups"] = _filter_place_groups(place_groups,
                                                           del_features=True)

    return dataset_dict

def get_dataset_place_groups(ctx: ServiceContext, ds_id: str, base_url: str) -> List[GeoJsonFeatureCollection]:
    # Do not load or return features, just place group (metadata).
    place_groups = ctx.get_dataset_place_groups(ds_id, base_url, load_features=False)
    return _filter_place_groups(place_groups, del_features=True)

def get_dataset_place_group(ctx: ServiceContext, ds_id: str, place_group_id: str,
                            base_url: str) -> GeoJsonFeatureCollection:
    # Load and return features for specific place group.
    place_group = ctx.get_dataset_place_group(ds_id, place_group_id, base_url, load_features=True)
    return _filter_place_group(place_group, del_features=False)

def get_dataset_coordinates(ctx: ServiceContext, ds_id: str, dim_name: str) -> Dict:
    ds, var = ctx.get_dataset_and_coord_variable(ds_id, dim_name)
    values = list()
    if np.issubdtype(var.dtype, np.floating):
        converter = float
    elif np.issubdtype(var.dtype, np.integer):
        converter = int
    else:
        converter = timestamp_to_iso_string
    for value in var.values:
        values.append(converter(value))
    return dict(name=dim_name,
                size=len(values),
                dtype=str(var.dtype),
                coordinates=values)

# noinspection PyUnusedLocal
def get_color_bars(ctx: ServiceContext, mime_type: str) -> str:
    cmaps = get_cmaps()
    if mime_type == 'application/json':
        return json.dumps(cmaps, indent=2)
    elif mime_type == 'text/html':
        html_head = '<!DOCTYPE html>\n' + \
                    '\n' + \
                    '' + \
                    '<meta charset="UTF-8">' + \
                    '<title>xcube server color maps</title>' + \
                    '\n' + \
                    '\n'
        html_body = ''
        html_foot = '\n' \
                    '\n'
        for cmap_cat, cmap_desc, cmap_bars in cmaps:
            html_body += '
%s
\n' % cmap_cat
            html_body += '
%s
\n' % cmap_desc
            html_body += '
\n'
            for cmap_bar in cmap_bars:
                cmap_name, cmap_data = cmap_bar
                cmap_image = f''
                html_body += f'
{cmap_name}:' \
                             f'	{cmap_image}
\n'
            html_body += '
\n'
        return html_head + html_body + html_foot
    raise ServiceBadRequestError(f'Format {mime_type!r} not supported for color bars')

def _is_point_in_dataset_bbox(point: Tuple[float, float], dataset_dict: Dict):
    if 'bbox' not in dataset_dict:
        return False
    x, y = point
    x_min, y_min, x_max, y_max = dataset_dict['bbox']
    if not (y_min <= y <= y_max):
        return False
    if x_min < x_max:
        return x_min <= x <= x_max
    else:
        # Bounding box crosses antimeridian
        return x_min <= x <= 180.0 or -180.0 <= x <= x_max

def _filter_place_group(place_group: Dict, del_features: bool = False) -> Dict:
    place_group = dict(place_group)
    del place_group['sourcePaths']
    del place_group['sourceEncoding']
    if del_features:
        del place_group['features']
    return place_group

def _filter_place_groups(place_groups, del_features: bool = False) -> List[Dict]:
    if del_features:
        def __filter_place_group(place_group):
            return _filter_place_group(place_group, del_features=True)
    else:
        def __filter_place_group(place_group):
            return _filter_place_group(place_group, del_features=False)

    return list(map(__filter_place_group, place_groups))

class CubeIsNotDisplayable(ValueError):
    pass


text:
import os.path as osp
from math import ceil

import torch
import torch.nn.functional as F
from torch_geometric.datasets import TUDataset
import torch_geometric.transforms as T
from torch_geometric.data import DenseDataLoader
from torch_geometric.nn import DenseSAGEConv, dense_diff_pool
import IPython
import sys
torch.set_printoptions(threshold=sys.maxsize)
from examples.graph_learning import parse_log_file
import numpy as np
from torch_geometric.data import InMemoryDataset
from torch_geometric.data.data import Data
import pickle

load_data = True
variational = True

max_nodes = 17

def estimate_vars(all_link_features, all_link_adj, all_rewards):
  key_dict = {}
  for feat, adj, rew in zip(all_link_features, all_link_adj, all_rewards):
    key = (feat.tostring(), adj.tostring())
    try:
      key_dict[key].append(rew)
    except:
      key_dict[key] = [rew]

  std_dict = {}
  for key in key_dict:
    std_dict[key] = np.std(key_dict[key])
  #TODO:
  #1. Compute std of each key
  #2. Re-loop over every key feature in order (maybe store in a loop on the first forward pass) and then return the stds in the same order
  #3. In the evaluation section, return difference of output variance and the sample variance as a proxy for the true variance
  return std_dict

class MyFilter(object):
    def __call__(self, data):
        return data.num_nodes <= max_nodes

path = osp.join(osp.dirname(osp.realpath(__file__)), '..', 'data',
                'PROTEINS_dense')
dataset = TUDataset(path, name='PROTEINS', transform=T.ToDense(max_nodes),
                    pre_filter=MyFilter())

print('path = ', path)

num_channels = 31
if not load_data:
  all_link_features, all_link_adj, all_rewards = parse_log_file.main('flat_jan21.csv', 'data/designs/grammar_jan21.dot')
  std_dict = estimate_vars(all_link_features, all_link_adj, all_rewards)

  if variational:
    all_rewards = [(reward,) for reward in all_rewards]
  else:
    all_rewards = [(reward,) for reward in all_rewards]

  #xperimental postprocessing
  #step 1: make symmetric
  all_link_adj_symmetric = [link_adj + np.transpose(link_adj) for link_adj in all_link_adj]

  #step 2: Add blank rows, pad with 0s, and fill out mask:
  #max length:
  max_nodes = max([feat.shape[0] for feat in all_link_features])

  def pad(array, shape):
      """
      array: Array to be padded
      reference: Reference array with the desired shape
      offsets: list of offsets (number of elements must be equal to the dimension of the array)
      """
      # Create an array of zeros with the reference shape
      result = np.zeros(shape)
      if len(shape) == 1:
        result[:array.shape[0], :] = array # ERROR: why result is 2d
      elif len(shape) == 2:
        result[:array.shape[0], :array.shape[1]] = array
      else:
        raise Exception('only 1 and 2d supported for now')
      return result

  all_link_adj_symmetric_pad = [pad(adj, (max_nodes, max_nodes)) for adj in all_link_adj_symmetric]
  all_features_pad = [pad(feat, (max_nodes, feat.shape[1])) for feat in all_link_features]

  def create_mask(feat, max_nodes):
    return np.array([True if i < feat.shape[0] else False for i in range(max_nodes)])

  all_masks = [create_mask(feat, max_nodes) for feat in all_link_features]
  #num_channels = all_features_pad[0].shape[1]

  #step 3: Create dataset object
  data = [Data(adj=torch.from_numpy(adj).float(),
               mask=torch.from_numpy(mask),
               x=torch.from_numpy(x[:, :num_channels]).float(),
               y=torch.from_numpy(np.array([y])).float(),
               std=torch.from_numpy(np.array([std_dict[std]])).float() ) for adj, mask, x, y, std in zip(all_link_adj_symmetric_pad, all_masks, all_features_pad, all_rewards, std_dict)]
  import random
  random.shuffle(data)

  dataset = dataset.shuffle()
  n = (len(dataset) + 9) // 10
  test_dataset = data[:n]
  val_dataset = data[n:2 * n]
  train_dataset = data[2 * n:]

  with open('test_loader', 'wb') as test_file, open('val_loader', 'wb') as val_file, open('train_loader', 'wb') as train_file:
    pickle.dump(test_dataset, test_file)
    pickle.dump(train_dataset, train_file)
    pickle.dump(val_dataset, val_file)

else:
  with open('test_loader', 'rb') as test_file, open('val_loader', 'rb') as val_file, open('train_loader', 'rb') as train_file:
    test_dataset = pickle.load(test_file)
    train_dataset = pickle.load(train_file)
    val_dataset = pickle.load(val_file)

test_loader = DenseDataLoader(test_dataset, batch_size=20)
val_loader = DenseDataLoader(val_dataset, batch_size=20)
train_loader = DenseDataLoader(train_dataset, batch_size=20)

class GNN(torch.nn.Module):
    def __init__(self, in_channels, hidden_channels, out_channels,
                 normalize=False, add_loop=False, lin=True):
        super(GNN, self).__init__()

        self.add_loop = add_loop

        self.conv1 = DenseSAGEConv(in_channels, hidden_channels, normalize)
        self.bn1 = torch.nn.BatchNorm1d(hidden_channels)
        self.conv2 = DenseSAGEConv(hidden_channels, hidden_channels, normalize)
        self.bn2 = torch.nn.BatchNorm1d(hidden_channels)
        self.conv3 = DenseSAGEConv(hidden_channels, out_channels, normalize)
        self.bn3 = torch.nn.BatchNorm1d(out_channels)

        if lin is True:
            self.lin = torch.nn.Linear(2 * hidden_channels + out_channels,
                                       out_channels)
        else:
            self.lin = None

    def bn(self, i, x):
        batch_size, num_nodes, num_channels = x.size()

        x = x.view(-1, num_channels)
        x = getattr(self, 'bn{}'.format(i))(x)
        x = x.view(batch_size, num_nodes, num_channels)
        return x

    def forward(self, x, adj, mask=None):
        batch_size, num_nodes, in_channels = x.size()

        x0 = x
        #IPython.embed()
        x1 = self.bn(1, F.relu(self.conv1(x0, adj, mask, self.add_loop)))
        x2 = self.bn(2, F.relu(self.conv2(x1, adj, mask, self.add_loop)))
        x3 = self.bn(3, F.relu(self.conv3(x2, adj, mask, self.add_loop)))

        x = torch.cat([x1, x2, x3], dim=-1)

        if self.lin is not None:
            x = F.relu(self.lin(x))

        return x

class Net(torch.nn.Module):
    def __init__(self, variational=False):
        super(Net, self).__init__()

        self.variational = variational

        num_nodes = ceil(0.25 * max_nodes)
        self.gnn1_pool = GNN(num_channels, 64, num_nodes, add_loop=True)
        self.gnn1_embed = GNN(num_channels, 64, 64, add_loop=True, lin=False)

        num_nodes = ceil(0.25 * num_nodes)
        self.gnn2_pool = GNN(3 * 64, 64, num_nodes)
        self.gnn2_embed = GNN(3 * 64, 64, 64, lin=False)

        self.gnn3_embed = GNN(3 * 64, 64, 64, lin=False)

        self.lin1 = torch.nn.Linear(3 * 64, 64)

        out_channels = 2 if self.variational else 1

        self.lin2 = torch.nn.Linear(64, out_channels)

    def forward(self, x, adj, mask=None):

        s = self.gnn1_pool(x, adj, mask)
        x = self.gnn1_embed(x, adj, mask)

        x, adj, l1, e1 = dense_diff_pool(x, adj, s, mask)

        #print('time for gnn2')

        s = self.gnn2_pool(x, adj)
        x = self.gnn2_embed(x, adj)

        x, adj, l2, e2 = dense_diff_pool(x, adj, s)

        x = self.gnn3_embed(x, adj)

        x = x.mean(dim=1)
        x = F.relu(self.lin1(x))

        x = self.lin2(x)

        return x, l1 + l2, e1 + e2

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model = Net(variational).to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=0.0001)

def train(epoch):
    model.train()
    loss_all = 0

    for data in train_loader:
        data = data.to(device)
        optimizer.zero_grad()
        output, _, _ = model(data.x, data.adj, data.mask)
        if not model.variational:
          loss = F.mse_loss(output[:, 0], data.y.view(-1))
        else:
          means = output[:, 0]
          stds = torch.exp(output[:, 1])
          dist = torch.distributions.Normal(means, stds)

          regularizer = 0.5 * torch.mean(torch.exp(stds) + means**2 - 1.0 - stds)
          loss = torch.mean(-dist.log_prob(data.y.view(-1))) + 0.001 * regularizer
        loss.backward()
        loss_all += loss.item()
        optimizer.step()
    return loss_all / len(train_dataset)

@torch.no_grad()
def test(loader, size):
    model.eval()
    error = 0.
    for data in loader:
        data = data.to(device)
        pred = model(data.x, data.adj, data.mask)[0]
        error += F.mse_loss(pred[:, 0], data.y.view(-1))
        pred_std = pred[:, 1]
        true_std = data.std.view(-1)
        #print('predicted std is ', torch.exp(pred_std))
        #print('true std is ', true_std)
        #print('prediction is ', pred[:, 0])
        #print('truth is ', data.y.view(-1))
        #correct += pred.eq(data.y.view(-1)).sum().item()
    return error / size

best_val_acc = test_acc = 10000.0
for epoch in range(1, 151):
    train_loss = train(epoch)
    train_acc = test(train_loader, len(train_loader))
    val_acc = test(val_loader, len(val_loader))
    if val_acc < best_val_acc:
        test_acc = test(test_loader, len(test_loader))
        best_val_acc = val_acc
    print('Epoch: {:03d}, Train Loss: {:.7f}, '
          'Val Acc: {:.7f}, Test Acc: {:.7f}, '
          'Train Acc: {:.7f}'.format(epoch, train_loss,
                                                     val_acc, test_acc,
                                                                      train_acc))

max_stars_repo_name: ONLYA/RoboGrammar

max_issues_repo_name: ONLYA/RoboGrammar

max_forks_repo_name: ONLYA/RoboGrammar


text:
#import os
#from pprint import pprint
#
#from mpi4py import MPI
#import numpy as np
#from networkx import DiGraph
#
#class ClusterManager:
#    def __init__(self):
#        self.node_name = MPI.Get_processor_name()
#        self.intracomm = MPI.COMM_WORLD
#
#    def inspect(self):
#        print("----------------------")
#        print("CLUSTER MANAGER")
#        #print("----------------------")
#        #print(self.intracomm) #> <mpi4py.MPI.Intracomm object at 0x10ed94a70>
#        #print(dict(self.intracomm.info)) #> {'mpi_assert_no_any_source': 'false', 'mpi_assert_allow_overtaking': 'false'}
#        #print("----------------------")
#        print("   NODE NAME:", self.node_name) #> 'MJs-MacBook-Air.local'
#        print("   NODE RANK:", self.node_rank) #> 0
#        print("   CLUSTER SIZE:", self.cluster_size) #> 1
#        print("   MAIN NODE?:", self.is_main_node) #> True
#
#    @property
#    def node_rank(self):
#        return self.intracomm.Get_rank()
#
#    @property
#    def cluster_size(self):
#        return self.intracomm.Get_size()
#
#    @property
#    def is_main_node(self):
#        return self.node_rank + 1 == self.cluster_size
#
#if __name__ == "__main__":
#
#    manager = ClusterManager()
#    manager.inspect()
#
#
#    # results = []
#    # for i in range(0, manager.cluster_size - 1): # this is a no-op because cluster size is 1 on the main node
#    # # for i in range(0, 3): # this just hangs if running only on one node?
#    #     result = manager.intracomm.recv(source=i)
#    #     results.append(result)
#    # print(results)
#

reason: import numpy,from networkx