text:
#Motion prediction Web service
#Gets real-time motion values and then sends the prediction of motion for the next time instant
#Communication is via MQTT

import json
import time
import paho.mqtt.client as MQTT
import random
import fpformat
import numpy
from datetime import datetime

# It writes the weeks in a nested list called year[] with:
# num_weeks - number of weeks (for easier presentation we use 4, for the final product it would be 52)
# samples_week - how many samples per week (for easier presentation we use 20, for the final product it would be 336 - every half hour)
# new_week stores the values of motion in real time, got via an MQTT subscription

new_value = 5
#num_weeks = 52
#samples_week = 336
num_weeks = 4
samples_week = 20
new_week = []
prediction = []

#Making the matrix which will keep values for the whole year
year = []
for row in range(num_weeks):
    week = []
    for column in range(samples_week):
        week.append(random.randint(0, 1))
    year.append(week)
print year

#Defining the MQTT class with its methods. This code will subscribe to values from the motion sensor and publish prediction values.
class MyMQTT:
    def __init__(self, broker, port, notifier):
        self.broker = broker
        self.port = port
        self.notifier = notifier
        self._paho_mqtt = MQTT.Client("Prediction_alg_motion", False)
        self._paho_mqtt.on_connect = self.myOnConnect
        self._paho_mqtt.on_message = self.myOnMessageReceived

    def myOnConnect (self, paho_mqtt, userdata, flags, rc):
        print ("Connected to message broker with result code: "+str(rc))

    def myOnMessageReceived (self, paho_mqtt , userdata, msg):
        self.notifier.notify(msg.topic, msg.payload)
        print ("Message received. Message: " + msg.topic + " QoS: " + str(msg.qos) + " Payload: " + str(msg.payload))

    def myPublish(self,p,hid):
       print("barbastruzzo")
       js = {"data": "motion", "value": p}
       motion_topic = 'house/'+hid+'/prediction_local_controller/predict_motion'
       #motion_topic = 'house/' + hid + '/motion_local_controller/motion_status'
       self._paho_mqtt.publish(motion_topic, json.dumps(js), 2)

    def mySubscribe(self, topic, qos=2):
        self._paho_mqtt.subscribe(topic, qos)
        print "Subscription successful"

    def start(self):
        self._paho_mqtt.connect(self.broker, self.port)
        self._paho_mqtt.loop_start()

    def stop(self):
        self._paho_mqtt.loop_stop()

class DoSomething():
    def __init__(self):
    	print "hello"
        self.house_id = ""
        self.mqtt_broker = ""
        self.mqtt_port = 0
        self.RC_config_reader()
        self.myMqtt = MyMQTT(self.mqtt_broker, self.mqtt_port, self)
        #self.myMqtt = MyMQTT("192.168.1.105", self.mqtt_port, self)
        self.myMqtt.start()
        self.myMqtt.mySubscribe('house/'+self.house_id+'/motion_local_controller/motion_status',0)
        global new_week
        global samples_week
        global week
        global year
        global num_weeks
        global prediction
        self.count = 0

        print "Old year: ", year
        self.predict()

        while True:

            time.sleep(30)

            new_week.append(new_value)
            print "new week is: ", new_week
            if len(new_week) == samples_week:
                print "Weeks shift here. New_week becomes the last week and is then emptied."
                for i in range(num_weeks):
                    if (i == num_weeks-1):
                        year[i] = new_week
                        break
                    year[i] = year[i+1]

                print "New year: ", year
                obj_year = {"year": year}
                self.writejson("predictions.json", obj_year)
                #break
                new_week = []
                self.count=0

                #trying to get the predictions
                self.predict()

#We publish a predicted value every half-hour
            self.myMqtt.myPublish(prediction[self.count], self.house_id)
            print "I've published at time: ", str(datetime.now())
            self.count += 1

#With this method we predict the presence values based on values for that time of the day and week, throughout the whole year
#We do it using a weighted average algorithm in which we use a number of past values, where the more recent values have a higher weight in the prediction

    def predict(self):
        global prediction
        #emptying prediction vector
        prediction = []
        #making column vectors with the appropriate values
        columns = []
        for row in range(samples_week):
            help = []
            for column in range(num_weeks):
                help.append(0)
            columns.append(help)

        for i in range(samples_week):
            for j in range(num_weeks):
                columns[i][j] = year[j][i]

        print "columns: ", columns

        for m in range(samples_week):
            x = 0
            sum_weights = 0
            weight = 0.5
            for item in columns[m]:
                x += (item * weight)
                sum_weights += weight
                weight += 0.1

            result = x / sum_weights
            print "Result: ", result
            if (result > 0.49999):
                prediction.append(1)
            else:
                prediction.append(0)
        print "Prediction: ", prediction

    def writejson(self, filename, obj):
        myfile = open(filename, "w")
        obj_1 = obj
        json.dump(obj_1, myfile)
        myfile.close()

    def end(self):
        self.myMqtt.stop ()

    def RC_config_reader(self):

        json_file = open('prediction_config.json').read()
        local_config = json.loads(json_file)

        if local_config.get("RC_base_url"):
            self.rc_base_url = local_config["RC_base_url"]
        else:
            print "Problem in local json - Can't get RC url"
        if local_config.get("Central_config_base_url"):
            self.cc_base_url = local_config["Central_config_base_url"]
        else:
            print "Problem in local json - Can't get Central config url"
        if local_config.get("house_id"):
            self.house_id = local_config["house_id"]
        else:
            print "Problem in local json - Can't get house_id"
        if local_config.get("mqtt_broker"):
            self.mqtt_broker = local_config["mqtt_broker"]
        else:
            print "Problem in local json - Can't get mqtt_broker"
        if local_config.get("mqtt_port"):
            self.mqtt_port = local_config["mqtt_port"]
        else:
            print "Problem in local json - Can't get mqtt_port"

    def notify(self, topic, msg):
        global new_value
        global num_weeks
        global samples_week
        global prediction
        if topic=='house/'+self.house_id+'/motion_local_controller/motion_status':
            new_value=(json.loads(msg)['value'])
        print "I've received the new value and it is: ", new_value

if __name__ == "__main__":
    test = DoSomething()
    test.end()

max_stars_repo_name: cantiusdeepan/Cadrea

max_issues_repo_name: cantiusdeepan/Cadrea

max_forks_repo_name: cantiusdeepan/Cadrea


text:
#!/usr/bin/env python
# -*- encoding: utf-8 -*-

from typing import Union

import numpy as np

def gen_batch_vectors(
    n_batches: int,
    n_features: int,
    batch_scale: float,
    bio_batch_angle: Union[float, None] = None,
    projection_to_bio: Union[np.ndarray, None] = None,
) -> np.ndarray:
    """Generates a batch-effect vector for each batch, optionally with some
    relation to the biological space

    :param n_batches: number of batches
    :param n_features: number of features
    :param batch_scale: size of batch effect relative to data
    :param bio_batch_angle: angle of batch effect w/ bio subspace
    :param projection_to_bio: projection from latent into gene space
    :return: array of shape (n_batches, n_features)
    """

    norm = lambda X: np.linalg.norm(X, axis=1, keepdims=True)

    batch_vectors = np.random.randn(n_batches, n_features)
    batch_vectors = (
        batch_vectors / norm(batch_vectors) * np.mean(norm(expression)) * batch_scale
    )

    if bio_batch_angle is not None:
        v_projected = np.dot(batch_vectors, projection_to_bio)
        v_complement = batch_vectors - v_projected

        batch_vectors = norm(batch_vectors) * (
            np.sin(bio_batch_angle) * v_complement / norm(v_complement)
            + np.cos(bio_batch_angle) * v_projected / norm(v_projected)
        )

    return batch_vectors

def add_batch_vectors(
    expression: np.ndarray,
    batch: np.ndarray,
    batch_scale: Union[int, float],
    bio_batch_angle: Union[float, None],
    projection_to_bio: Union[np.ndarray, None],
    copy: bool = True,
) -> np.ndarray:
    """Generate batch-effect vectors and apply them to the expression data

    :param expression: array of true expression, in latent space
    :param batch: indicator of which obs belongs to which batch
    :param batch_scale: batch effect relative to data
    :param bio_batch_angle: angle of batch effect w/ bio subspace
    :param projection_to_bio: projection from latent into gene space
    :param copy: return a copy of the expression array or modify in-place
    :return: expression matrix with batch effect
    """
    if copy:
        expression = expression.copy()

    n_batches = len(np.unique(batch))

    # add batch vector
    batch_vectors = gen_batch_vectors(
        n_batches, expression.shape[1], batch_scale, bio_batch_angle, projection_to_bio
    )

    for i in range(n_batches):
        expression[batch == i, :] += batch_vectors[i, :]

    return expression

ext: py

max_stars_repo_name: czbiohub/simscity

max_issues_repo_name: czbiohub/simscity

max_forks_repo_name: czbiohub/simscity

reason: import numpy


text:
import numpy as np
import seaborn as sns
import matplotlib.pylab as plt

kl_list = ['2.0', '3.0', '4.0', '5.0', '6.0']
es_list = ['1.', '1.1', '1.2', '1.5']
nb_list = ['1', '10', '50']

file_path = '/scratch/mh740/models/relative-entropy-coding/imagenet32/resnet_vae/gaussian/blocks_24/beta_1.000_lamb_0.100/compressor_initialized_{}/results_{}_beam_{}_es.csv'
fig, axes = plt.subplots(3, len(kl_list), figsize=(20, 20))
for k, kl in enumerate(kl_list):
    mask = np.ones((len(es_list), len(nb_list)), dtype=np.int32)
    crash_mask = np.ones((len(es_list), len(nb_list)), dtype=np.int32)
    overhead = np.zeros((len(es_list), len(nb_list)), dtype=np.float32)
    time = np.zeros((len(es_list), len(nb_list)), dtype=np.float32)
    crashes = np.zeros((len(es_list), len(nb_list)), dtype=np.int32)
    for i, es in enumerate(es_list):
        for j, nb in enumerate(nb_list):
            try:
                with open(file_path.format(kl, nb, es), "r") as results:
                    line_list = [line.split(',') for line in results]
                if len(line_list) < 11:
                    continue
                n_crash = sum([1 if float(l[9]) == -1.0 else 0 for l in line_list[1:]])
                crashes[i, j] = n_crash
                if n_crash == 10:
                    crash_mask[i, j] = 0
                    continue
                avg_times = sum([float(l[9]) if float(l[9]) != -1.0 else 0. for l in line_list[1:]]) / (10 - n_crash)
                avg_elbo = sum([float(l[0]) + float(l[1]) if float(l[9]) != -1.0 else 0. for l in line_list[1:]]) / (10 - n_crash)
                avg_codelength = sum([float(l[4]) + float(l[5]) if float(l[9]) != -1.0 else 0. for l in line_list[1:]]) / (10 - n_crash)
                # print(n_crash)
                # print(avg_elbo)
                # print([float(l[0]) + float(l[1]) if float(l[9]) != -1.0 else 0. for l in line_list[1:]])
                overhead[i, j] = -1. + avg_codelength / avg_elbo
                time[i, j] = avg_times
                mask[i, j] = 0
                crash_mask[i, j] = 0
            except FileNotFoundError:
                continue

    sns.heatmap(overhead.transpose(), annot=True, mask=mask.transpose(), ax=axes[0, k], vmin=0, vmax=1., xticklabels=es_list, yticklabels=nb_list, fmt='.3f')
    axes[0, k].set_title('Overhead \n KL per partition={}'.format(kl))
    axes[0, k].set_xlabel('Oversampling')
    axes[0, k].set_ylabel('N beams')
    sns.heatmap(time.transpose(), annot=True, mask=mask.transpose(), ax=axes[1, k], vmin=0, vmax=1000, xticklabels=es_list, yticklabels=nb_list, fmt='.1f')
    axes[1, k].set_title('Runtime \n KL per partition={}'.format(kl))
    axes[1, k].set_xlabel('Oversampling')
    axes[1, k].set_ylabel('N beams')
    sns.heatmap(crashes.transpose(), annot=True, mask=crash_mask.transpose(), ax=axes[2, k], vmin=0, vmax=10, xticklabels=es_list, yticklabels=nb_list)
    axes[2, k].set_title('Number of crashes \n KL per partition={}'.format(kl))
    axes[2, k].set_xlabel('Oversampling')
    axes[2, k].set_ylabel('N beams')
plt.savefig('examples/figures/hyperparam_tuning.pdf')

max_stars_repo_name: gergely-flamich/relative-entropy-coding

max_issues_repo_name: gergely-flamich/relative-entropy-coding

max_forks_repo_name: gergely-flamich/relative-entropy-coding


text:
# Source: http://www.pyqtgraph.org/documentation/_modules/pyqtgraph/opengl/items/GLAxisItem.html

import numpy as np

from OpenGL.GL import *
import pyqtgraph.opengl as gl

class ColoredGLAxisItem(gl.GLAxisItem):
    ''' A sub-class of GLAxisItem with the ability to customize the axes colors. '''

    def __init__(self, x_color=(255, 0, 0), y_color=(0, 255, 0), z_color=(0, 0, 255), **kwargs):
        super().__init__(**kwargs)
        self.x_color = x_color
        self.y_color = y_color
        self.z_color = z_color

    def paint(self):
        self.setupGLState()

        if self.antialias:
            glEnable(GL_LINE_SMOOTH)
            glHint(GL_LINE_SMOOTH_HINT, GL_NICEST)

        glLineWidth(5)

        glBegin( GL_LINES )

        x, y, z = self.size()

        _z_color = (np.array(self.z_color) / 255).tolist()
        glColor4f(*_z_color, .6)  # z is blue by default
        glVertex3f(0, 0, 0)
        glVertex3f(0, 0, z)

        _y_color = (np.array(self.y_color) / 255).tolist()
        glColor4f(*_y_color, .6)  # y is green by default
        glVertex3f(0, 0, 0)
        glVertex3f(0, y, 0)

        _x_color = (np.array(self.x_color) / 255).tolist()
        glColor4f(*_x_color, .6)  # x is red by default
        glVertex3f(0, 0, 0)
        glVertex3f(x, 0, 0)
        glEnd()


text:
import numpy as np
import paddle as pdpd
from paddle.nn.functional import interpolate
from save_model import saveModel
import sys
pdpd.enable_static()

def run_and_save_model(input_x, name, feed, fetch_list, main_prog, start_prog):
    cpu = pdpd.static.cpu_places(1)
    exe = pdpd.static.Executor(cpu[0])
    exe.run(start_prog)
    outs = exe.run(
        feed={'x': input_x},
        fetch_list=fetch_list,
        program=main_prog)

    with pdpd.static.program_guard(main_prog, start_prog):
        saveModel(name, exe, feedkeys=['x'], fetchlist=fetch_list, inputs=[input_x],
                  outputs=[outs[0]], target_dir=sys.argv[1])

    return outs

def pdpd_interpolate(x, sizes=None, scale_factor=None, mode='nearest', align_corners=True,
                     align_mode=0, data_format='NCHW', name=None):
    pdpd.enable_static()
    main_program = pdpd.static.Program()
    startup_program = pdpd.static.Program()
    with pdpd.static.program_guard(main_program, startup_program):
        node_x = pdpd.static.data(name='x', shape=x.shape, dtype='float32')
        interp = interpolate(node_x, size=sizes, scale_factor=scale_factor,
                             mode=mode, align_corners=align_corners, align_mode=align_mode,
                             data_format=data_format, name=name)
        out = pdpd.static.nn.batch_norm(interp, use_global_stats=True, epsilon=0)
    outs = run_and_save_model(x, name, node_x, out, main_program, startup_program)
    return outs[0]

def resize_upsample_bilinear():
    data = np.array([[[
        [1, 2, 3, 4],
        [5, 6, 7, 8],
        [9, 10, 11, 12],
        [13, 14, 15, 16]
    ]]], dtype=np.float32)

    test_case = [{'name': 'bilinear_upsample_false_1', 'align_corners': False, 'align_mode': 1},
                 {'name': 'bilinear_upsample_false_0', 'align_corners': False, 'align_mode': 0},
                 {'name': 'bilinear_upsample_true_0', 'align_corners': True, 'align_mode': 0}]

    for test in test_case:
        pdpd_result = pdpd_interpolate(data, [64, 64], None, mode='bilinear', align_corners=test['align_corners'],
                                       align_mode=test['align_mode'], data_format='NCHW', name=test['name'])

def resize_downsample_bilinear():
    data = np.array([[[
        [1, 2, 3, 4],
        [5, 6, 7, 8],
        [9, 10, 11, 12],
        [13, 14, 15, 16]
    ]]], dtype=np.float32)
    data_28 = data.reshape([1, 1, 2, 8])
    test_case = [{'name': 'bilinear_downsample_false_1', 'align_corners': False, 'align_mode': 1},
                 {'name': 'bilinear_downsample_false_0', 'align_corners': False, 'align_mode': 0},
                 {'name': 'bilinear_downsample_true_0', 'align_corners': True, 'align_mode': 0}]

    for test in test_case:
        pdpd_result = pdpd_interpolate(data_28, [2, 4], None, mode='bilinear', align_corners=test['align_corners'],
                                       align_mode=test['align_mode'], data_format='NCHW', name=test['name'])

def resize_upsample_nearest():
    data = np.array([[[
        [1, 2, 3, 4],
        [5, 6, 7, 8],
        [9, 10, 11, 12],
        [13, 14, 15, 16]
    ]]], dtype=np.float32)

    test_case = [
        {'name': 'nearest_upsample_false_0', 'size': [64, 64], 'align_corners': False, 'align_mode': 0},
        {'name': 'nearest_upsample_false_1', 'size': [16, 64], 'align_corners': False, 'align_mode': 0}
    ]

    for test in test_case:
        pdpd_result = pdpd_interpolate(data, test['size'], None, mode='nearest', align_corners=test['align_corners'],
                                       align_mode=test['align_mode'], data_format='NCHW', name=test['name'])

def resize_downsample_nearest():
    data = np.arange(0, 4096).astype(np.float32)
    data_64 = data.reshape([1, 1, 64, 64])
    test_case = [
        {'name': 'nearest_downsample_false_0', 'size': [8, 8], 'align_corners': False, 'align_mode': 1},
        {'name': 'nearest_downsample_false_1', 'size': [4, 8], 'align_corners': False, 'align_mode': 1}
    ]

    for test in test_case:
        pdpd_result = pdpd_interpolate(data_64, test['size'], None, mode='nearest', align_corners=test['align_corners'],
                                       align_mode=test['align_mode'], data_format='NCHW', name=test['name'])

def nearest_upsample_tensor_size():
    data = np.array([[[
        [1, 2, 3, 4],
        [5, 6, 7, 8],
        [9, 10, 11, 12],
        [13, 14, 15, 16]
    ]]], dtype=np.float32)
    sizes = np.array([8, 8], dtype=np.int32)
    pdpd.enable_static()
    test_case = [{'name': 'nearest_upsample_tensor_size', 'align_corners': False, 'align_mode': 0}]
    for test in test_case:
        main_program = pdpd.static.Program()
        startup_program = pdpd.static.Program()
        with pdpd.static.program_guard(main_program, startup_program):
            node_x = pdpd.static.data(name='x', shape=data.shape, dtype='float32')
            node_sizes = pdpd.static.data(name='sizes', shape=sizes.shape, dtype='int32')
            interp = interpolate(node_x, size=node_sizes, scale_factor=None,
                                 mode='nearest', align_corners=test['align_corners'], align_mode=test['align_mode'],
                                 data_format='NCHW', name=test['name'])
            out = pdpd.static.nn.batch_norm(interp, use_global_stats=True, epsilon=0)
            cpu = pdpd.static.cpu_places(1)
            exe = pdpd.static.Executor(cpu[0])
            exe.run(startup_program)
            outs = exe.run(
                feed={'x': data, 'sizes': sizes},
                fetch_list=out,
                program=main_program)
            saveModel(test['name'], exe, feedkeys=['x', 'sizes'], fetchlist=out, inputs=[data, sizes], outputs=[outs[0]], target_dir=sys.argv[1])

def bilinear_upsample_tensor_size():
    data = np.array([[[
        [1, 2, 3, 4],
        [5, 6, 7, 8],
        [9, 10, 11, 12],
        [13, 14, 15, 16]
    ]]], dtype=np.float32)
    sizes = np.array([8, 8], dtype="int32")

    test_case = [{'name': 'bilinear_upsample_tensor_size', 'align_corners': False, 'align_mode': 1}]

    for test in test_case:
        main_program = pdpd.static.Program()
        startup_program = pdpd.static.Program()
        with pdpd.static.program_guard(main_program, startup_program):
            node_x = pdpd.static.data(name='x', shape=data.shape, dtype='float32')
            node_sizes = pdpd.static.data(name='sizes', shape=sizes.shape, dtype='int32')
            interp = interpolate(node_x, size=node_sizes, scale_factor=None,
                                 mode='bilinear', align_corners=test['align_corners'], align_mode=test['align_mode'],
                                 data_format='NCHW', name=test['name'])
            out = pdpd.static.nn.batch_norm(interp, use_global_stats=True, epsilon=0)
            cpu = pdpd.static.cpu_places(1)
            exe = pdpd.static.Executor(cpu[0])
            exe.run(startup_program)
            outs = exe.run(
                feed={'x': data, 'sizes': sizes},
                fetch_list=out,
                program=main_program)
            saveModel(test['name'], exe, feedkeys=['x', 'sizes'], fetchlist=out, inputs=[data, sizes], outputs=[outs[0]], target_dir=sys.argv[1])

def bilinear_upsample_scales():
    data = np.array([[[
        [1, 2, 3, 4],
        [5, 6, 7, 8],
        [9, 10, 11, 12],
        [13, 14, 15, 16]
    ]]], dtype=np.float32)

    test_case = [{'name': 'bilinear_upsample_scales', 'align_corners': False, 'align_mode': 1, "scales": 2},
                 {'name': 'bilinear_upsample_scales2', 'align_corners': False, 'align_mode': 1, "scales": [2, 2]}]

    for test in test_case:
        pdpd_result = pdpd_interpolate(data, None, 2, mode='bilinear', align_corners=test['align_corners'],
                                       align_mode=test['align_mode'], data_format='NCHW', name=test['name'])

if __name__ == "__main__":
    resize_downsample_bilinear()
    resize_upsample_bilinear()
    resize_downsample_nearest()
    resize_upsample_nearest()
    nearest_upsample_tensor_size()
    bilinear_upsample_tensor_size()
    bilinear_upsample_scales()

max_stars_repo_name: monroid/openvino

max_issues_repo_name: monroid/openvino

max_forks_repo_name: v-Golubev/openvino


text:
#!/usr/bin/env python

import collections
import functools
import itertools
import math
import numpy

import uproot

import lark

grammar = """
start:      (NEWLINE | ";")* (statement (NEWLINE | ";")+)* statement (NEWLINE | ";")*

statement:  assignment -> pass | funcassign -> pass
funcassign: CNAME "(" [CNAME ("," CNAME)*] ")" "=" block
assignment: CNAME "=" expression

patassign:  CNAME "=" expression
          | CNAME "~" expression -> symmetric
          | CNAME "~~" expression -> allsymmetric
          | CNAME "!~" expression -> asymmetric
          | CNAME "!~~" expression -> allasymmetric
pattern:    (NEWLINE | ";")* (patassign (NEWLINE | ";")+)* patassign (NEWLINE | ";")*
block:      (NEWLINE | ";")* (statement (NEWLINE | ";")+)* expression (NEWLINE | ";")*

expression: function   -> pass
function:   or         -> pass | paramlist "=>" block
or:         and        -> pass | and "or" and
and:        not        -> pass | not "and" not
not:        comparison -> pass | "not" not
comparison: arith      -> pass | arith "==" arith -> eq | arith "!=" arith -> ne
                               | arith ">" arith -> gt  | arith ">=" arith -> ge
                               | arith "<" arith -> lt  | arith "<=" arith -> le

arith:      term          -> pass | term "+" arith  -> add | term "-" arith  -> sub
term:       factor        -> pass | factor "*" term -> mul | factor "/" term -> div
factor:     pow           -> pass | "+" factor      -> pos | "-" factor      -> neg
pow:        call          -> pass | call "**" factor
call:       atom          -> pass
          | call "(" arglist ")"
          | call "[" slice "]" -> subscript
          | call "." CNAME -> attribute

atom:       CNAME -> symbol | INT -> int | FLOAT -> float
          | "(" block ")" -> pass
          | "{" pattern "}" -> pass
          | "join" "{" pattern "}" -> join

paramlist:  "(" [CNAME ("," CNAME)*] ")" | CNAME
arglist:    [expression] ("," expression)*
slice:      expression -> pass
          | expression ":" -> slice1 | ":" expression -> slice2 | expression ":" expression -> slice12

%import common.CNAME
%import common.INT
%import common.FLOAT
%import common.NEWLINE
%import common.WS
%ignore WS
"""

parser = lark.Lark(grammar)

class AST:
    _fields = ()

    def __init__(self, *args, line=None):
        self.line = line
        assert len(self._fields) == len(args)
        for n, x in zip(self._fields, args):
            setattr(self, n, x)
            if not isinstance(x, list):
                x = [x]
            for xi in x:
                if self.line is None:
                    self.line = getattr(xi, "line", None)

    def __repr__(self):
        return "{0}({1})".format(type(self).__name__, ", ".join(getattr(self, n) for n in self._fields))

class Module(AST):
    _fields = ("statements",)
    def __str__(self):
        return "\n".join(str(x) for x in self.statements)

class Assignment(AST):
    _fields = ("symbol", "operator", "expression")
    def __str__(self):
        return "{0} {1} {2}".format(self.symbol, self.operator, str(self.expression))

class Pattern(AST):
    _fields = ("assignments", "matching")
    def __init__(self, *args, **kwargs):
        super(Pattern, self).__init__(*args, **kwargs)
    def __str__(self):
        return "{" + "; ".join(str(x) for x in self.assignments) + "}"

class Join(AST):
    _fields = ("expression", "matching")
    def __str__(self):
        return "join " + str(self.expression)

class Function(AST):
    _fields = ("parameters", "body")
    def __str__(self):
        left, right = ("", "") if len(self.parameters) == 1 else ("(", ")")
        return "{0}{1}{2} => {3}".format(left, ", ".join(self.parameters), right, str(self.body))

class Block(AST):
    _fields = ("statements", "expression")
    def __str__(self):
        return "{" + "; ".join(str(x) for x in self.statements + [self.expression]) + "}"

class Call(AST):
    _fields = ("function", "arguments")
    def __str__(self):
        return "{0}({1})".format(str(self.function), ", ".join(str(x) for x in self.arguments))

class Subscript(AST):
    _fields = ("object", "index")
    def __str__(self):
        return "{0}[{1}]".format(str(self.object), str(self.index))

class Attribute(AST):
    _fields = ("object", "field")
    def __str__(self):
        return "{0}.{1}".format(str(self.object), self.field)

class Slice(AST):
    _fields = ("start", "stop")
    def __str__(self):
        return "{0}:{1}".format("" if self.start is None else str(self.start), "" if self.stop is None else str(self.stop))

class Symbol(AST):
    _fields = ("symbol",)
    def __str__(self):
        return self.symbol

class Literal(AST):
    _fields = ("value",)
    def __str__(self):
        return str(self.value)

opname = {
    "and": "and", "or": "or", "not": "not",
    "eq": "==", "ne": "!=", "gt": ">", "ge": ">=", "lt": "<", "le": "<=",
    "add": "+", "sub": "-", "mul": "*", "div": "/", "pos": "+", "neg": "-", "pow": "**",
    "symmetric": "~", "allsymmetric": "~~", "asymmetric": "!~", "allasymmetric": "!~~"
    }

def toast(node, matching):
    if node.data == "pass":
        return toast(node.children[0], matching)
    elif node.data == "start":
        return Module([toast(x, matching) for x in node.children if not isinstance(x, lark.lexer.Token)])
    elif node.data == "funcassign":
        return Assignment(str(node.children[0]), "=", Function(node.children[1:-1], toast(node.children[-1], matching)), line=node.children[0].line)
    elif node.data == "assignment" or node.data == "patassign":
        return Assignment(str(node.children[0]), "=", toast(node.children[1], matching), line=node.children[0].line)
    elif node.data == "symmetric" or node.data == "allsymmetric" or node.data == "asymmetric" or node.data == "allasymmetric":
        if matching is None:
            raise SyntaxError("cannot use {0} operator outside of a join".format(opname[node.data]))
        return Assignment(str(node.children[0]), opname[node.data], matching.replace(node.children[1], node.data == "symmetric" or node.data == "asymmetric"), line=node.children[0].line)
    elif node.data == "pattern":
        return Pattern([toast(x, matching) for x in node.children if not isinstance(x, lark.lexer.Token)], matching)
    elif node.data == "join":
        m = Matching()
        return Join(toast(node.children[0], m), m)
    elif node.data == "function":
        return Function(toast(node.children[0], matching), toast(node.children[1], None))
    elif node.data == "paramlist":
        return [str(x) for x in node.children]
    elif node.data == "block":
        items = [toast(x, matching) for x in node.children if not isinstance(x, lark.lexer.Token)]
        if len(items) == 1:
            return items[0]
        else:
            return Block(items[:-1], items[-1])
    elif node.data == "call":
        return Call(toast(node.children[0], matching), toast(node.children[1], matching))
    elif node.data == "arglist":
        return [toast(x, matching) for x in node.children]
    elif node.data == "subscript":
        return Subscript(toast(node.children[0], matching), toast(node.children[1], matching))
    elif node.data == "attribute":
        return Attribute(toast(node.children[0], matching), node.children[1])
    elif node.data == "slice1":
        return Slice(toast(node.children[0], matching), None)
    elif node.data == "slice2":
        return Slice(None, toast(node.children[0], matching))
    elif node.data == "slice12":
        return Slice(toast(node.children[0], matching), toast(node.children[1], matching))
    elif node.data == "symbol":
        return Symbol(str(node.children[0]), line=node.children[0].line)
    elif node.data == "int":
        return Literal(int(node.children[0]), line=node.children[0].line)
    elif node.data == "float":
        return Literal(float(node.children[0]), line=node.children[0].line)
    elif node.data in opname:
        return Call(Symbol(opname[node.data]), [toast(x, matching) for x in node.children])
    else:
        raise AssertionError(node.data)

def ast(source):
    return toast(parser.parse(source), None)

class SymbolTable:
    def __init__(self, parent, symbols):
        self.parent = parent
        self.symbols = symbols

    def __getitem__(self, symbol):
        if symbol in self.symbols:
            return self.symbols[symbol]
        elif self.parent is not None:
            return self.parent[symbol]
        else:
            raise KeyError(symbol)

    def __setitem__(self, symbol, value):
        if symbol in self.symbols:
            raise TypeError("symbol {0} has multiple definitions at this scope".format(repr(symbol)))
        else:
            self.symbols[symbol] = value

    def __delitem__(self, symbol):
        if symbol in self.symbols:
            del self.symbols[symbol]
        elif self.parent is not None:
            del self.parent[symbol]
        else:
            raise KeyError(symbol)

    def __str__(self):
        out = []
        for n, x in self.symbols.items():
            if not callable(x):
                first = (len(out) == 0)
                start = ("" if first else " ") + repr(n) + ": "
                if isinstance(x, list):
                    out.append(start + "[" + ("\n" + " " * len(start) + ("  " if first else " ")).join(repr(xi) for xi in x) + "]")
                else:
                    out.append(start + repr(x))
        return "{" + ",\n".join(out) + "}"

@functools.total_ordering
class ID:
    def __init__(self, n):
        self.n = n
    def __repr__(self):
        return "{0}({1})".format(type(self).__name__, self.n)
    def __hash__(self):
        return hash((type(self), self.n))
    def __eq__(self, other):
        return type(self) == type(other) and self.n is not None and self.n == other.n
    def __ne__(self, other):
        return not self == other
    def __lt__(self, other):
        if type(self) is type(other):
            if self.n is None:
                return True
            elif other.n is None:
                return False
            else:
                return self.n < other.n
        else:
            return type(self).__name__ < type(other).__name__

class obj:
    def __init__(self, id=None, **fields):
        self._id = id
        self._fields = collections.OrderedDict(fields.items())
        self._asymm = set()
    @property
    def id(self):
        if self._id is None:
            asymm, symm = [], []
            for n, x in self._fields.items():
                if isinstance(x, obj):
                    if n in self._asymm:
                        asymm.append(x.id)
                    else:
                        symm.append(x.id)
            if len(symm) + len(asymm) == 0:
                return ID(None)
            else:
                return tuple(sorted(symm) + asymm)
        else:
            return self._id
    def __contains__(self, n):
        return n in self._fields
    def __getitem__(self, n):
        return self._fields[n]
    def __setitem__(self, n, x):
        self._fields[n] = x
    def __getattr__(self, n):
        if n in self._fields:
            return self._fields[n]
        else:
            raise AttributeError("{0} object has no attribute {1}".format(repr(type(self).__name__), repr(n)))
    def __repr__(self):
        return "{0}({1}{2})".format(type(self).__name__, self.id, "".join(", {0}={1}".format(n, repr(x)) for n, x in self._fields.items()))

class Matching:
    class Placeholder:
        def __init__(self, m, n):
            self.m = m
            self.n = n
        def __repr__(self):
            return "Placeholder({0}, {1})".format(self.m, self.n)
        def value(self):
            return self.m.row[self.n]

    def __init__(self):
        self.collections = []
        self.unique = []

    def __repr__(self):
        return "<Matching at 0x{0:012x}>".format(id(self))

    def replace(self, node, unique):
        toasted = toast(node, self)
        if isinstance(toasted, Pattern):
            return toasted
        else:
            self.collections.append(toasted)
            self.unique.append(unique)
            return self.Placeholder(self, len(self.collections) - 1)

    def run(self, node, symbols):
        evaluated = [run(x, symbols) for x in self.collections]
        for row in itertools.product(*evaluated):
            for i, unique in enumerate(self.unique):
                if unique and row[i]._id in [row[j]._id for j in range(len(row)) if i != j]:
                    break
            else:
                self.row = row
                yield run(node, symbols)

class Method:
    def __init__(self, object, function):
        self.object = object
        self.function = function
    def __repr__(self):
        return "Method({0}, {1})".format(self.object, self.function)

class Evaluated:
    def __init__(self, value):
        self.value = value
    def __repr__(self):
        return "Evaluated({0})".format(self.value)

def run(node, symbols):
    if isinstance(node, Evaluated):
        return node.value

    elif isinstance(node, Module):
        for x in node.statements:
            run(x, symbols)

    elif isinstance(node, Assignment):
        if isinstance(node.expression, Matching.Placeholder):
            symbols[node.symbol] = node.expression.value()
        else:
            symbols[node.symbol] = run(node.expression, symbols)
        if node.operator == "!~" or node.operator == "!~~":
            symbols.symbols._asymm.add(node.symbol)

    elif isinstance(node, Join):
        node.matching.seen = set()
        out = []
        for result in node.matching.run(node.expression, symbols):
            if result.id not in node.matching.seen:
                out.append(result)
            node.matching.seen.add(result.id)
        return out

    elif isinstance(node, Pattern):
        out = obj()
        symbols = SymbolTable(symbols, out)
        for x in node.assignments:
            run(x, symbols)
        return out

    elif isinstance(node, Function):
        def function(arguments, symbols):
            if len(node.parameters) != len(arguments):
                raise TypeError("on line {0}, function expects {1} arguments, got {2}".format(node.line, len(node.parameters), len(arguments)))
            scope = {}
            for param, arg in zip(node.parameters, arguments):
                scope[param] = run(arg, symbols)
            return run(node.body, SymbolTable(symbols, scope))
        return function

    elif isinstance(node, Block):
        symbols = SymbolTable(symbols, {})
        for x in node.statements:
            run(x, symbols)
        return run(node.expression, symbols)

    elif isinstance(node, Call):
        x = run(node.function, symbols)
        if isinstance(x, Method):
            return x.function(x.object, node.arguments, symbols)
        else:
            try:
                return x(node.arguments, symbols)
            except Exception as err:
                raise RuntimeError("on line {0}, encountered {1}: {2}".format("???" if node.line is None else node.line, type(err).__name__, str(err)))

    elif isinstance(node, Subscript):
        try:
            return run(node.object, symbols)[run(node.index, symbols)]
        except Exception as err:
            raise RuntimeError("on line {0}, encountered {1}: {2}".format("???" if node.line is None else node.line, type(err).__name__, str(err)))

    elif isinstance(node, Attribute):
        x = run(node.object, symbols)
        if isinstance(x, list):
            return Method(x, listmethods[node.field])
        elif isinstance(x, obj):
            try:
                return getattr(x, node.field)
            except Exception as err:
                raise RuntimeError("on line {0}, encountered {1}: {2}".format("???" if node.line is None else node.line, type(err).__name__, str(err)))
        else:
            raise AssertionError(type(x))

    elif isinstance(node, Slice):
        return slice(None if node.start is None else run(node.start, symbols), None if node.stop is None else run(node.stop, symbols), None)

    elif isinstance(node, Symbol):
        try:
            return symbols[node.symbol]
        except Exception as err:
            raise RuntimeError("on line {0}, encountered {1}: {2}".format("???" if node.line is None else node.line, type(err).__name__, str(err)))

    elif isinstance(node, Literal):
        return node.value

    else:
        raise AssertionError(type(node))

builtins = SymbolTable(None, {
    "len":  lambda arguments, symbols: len(run(arguments[0], symbols)),
    "abs":  lambda arguments, symbols: abs(run(arguments[0], symbols)),

    "and":  lambda arguments, symbols: run(arguments[0], symbols) and run(arguments[1], symbols),
    "or":   lambda arguments, symbols: run(arguments[0], symbols) or run(arguments[1], symbols),
    "not":  lambda arguments, symbols: not run(arguments[0], symbols),

    "==":   lambda arguments, symbols: run(arguments[0], symbols) == run(arguments[1], symbols),
    "!=":   lambda arguments, symbols: run(arguments[0], symbols) != run(arguments[1], symbols),
    ">":    lambda arguments, symbols: run(arguments[0], symbols) > run(arguments[1], symbols),
    ">=":   lambda arguments, symbols: run(arguments[0], symbols) >= run(arguments[1], symbols),
    "<":    lambda arguments, symbols: run(arguments[0], symbols) < run(arguments[1], symbols),
    "<=":   lambda arguments, symbols: run(arguments[0], symbols) <= run(arguments[1], symbols),
    "+":    lambda arguments, symbols: +run(arguments[0], symbols) if len(arguments) == 1 else (run(arguments[0], symbols) + run(arguments[1], symbols)),
    "-":    lambda arguments, symbols: -run(arguments[0], symbols) if len(arguments) == 1 else (run(arguments[0], symbols) - run(arguments[1], symbols)),
    "*":    lambda arguments, symbols: run(arguments[0], symbols) * run(arguments[1], symbols),
    "/":    lambda arguments, symbols: float(run(arguments[0], symbols)) / float(run(arguments[1], symbols)),
    "**":   lambda arguments, symbols: run(arguments[0], symbols) ** run(arguments[1], symbols),

    "sqrt": lambda arguments, symbols: math.sqrt(run(arguments[0], symbols)),
    "exp":  lambda arguments, symbols: math.exp(run(arguments[0], symbols)),
    "sin":  lambda arguments, symbols: math.sin(run(arguments[0], symbols)),
    "cos":  lambda arguments, symbols: math.cos(run(arguments[0], symbols)),
    "tan":  lambda arguments, symbols: math.tan(run(arguments[0], symbols)),
    "sinh": lambda arguments, symbols: math.sinh(run(arguments[0], symbols)),
    "cosh": lambda arguments, symbols: math.cosh(run(arguments[0], symbols)),
    "tanh": lambda arguments, symbols: math.tanh(run(arguments[0], symbols)),
    })

def builtins_union(arguments, symbols):
    out = []
    ids = set()
    for arg in arguments:
        collection = run(arg, symbols)
        for x in collection:
            if x._id not in ids:
                out.append(x)
            ids.add(x._id)
    return out
builtins["union"] = builtins_union

listmethods = {}

def listmethods_filter(object, arguments, symbols):
    f = run(arguments[0], symbols)
    return [x for x in object if f([Evaluated(x)], symbols)]
listmethods["filter"] = listmethods_filter

def listmethods_sort(object, arguments, symbols):
    f = run(arguments[0], symbols)
    order = numpy.argsort([f([Evaluated(x)], symbols) for x in object])
    return numpy.array(object, dtype="O")[order].tolist()
listmethods["sort"] = listmethods_sort

def listmethods_top(object, arguments, symbols):
    n = run(arguments[0], symbols)
    f = run(arguments[1], symbols)
    order = numpy.argsort([f([Evaluated(x)], symbols) for x in object])[:n]
    return numpy.array(object, dtype="O")[order].tolist()
listmethods["top"] = listmethods_top

class LorentzVector(obj):
    def __init__(self, id, px, py, pz, E):
        super(LorentzVector, self).__init__(id)
        self._fields = {"px": px, "py": py, "pz": pz, "E": E}
    def __repr__(self):
        return "LorentzVector({0}, {1:g}, {2:g}, {3:g}, {4:g})".format(self.id, self.px, self.py, self.pz, self.E)
    @property
    def pt(self):
        return math.sqrt(self.px**2 + self.py**2)
    @property
    def mass(self):
        try:
            return math.sqrt(self.E**2 - self.px**2 - self.py**2 - self.pz**2)
        except ValueError:
            return float("nan")
    def __add__(self, other):
        return LorentzVector((self.id, other.id), self.px + other.px, self.py + other.py, self.pz + other.pz, self.E + other.E)

class L(ID): pass
symbols = SymbolTable(builtins, {"x": LorentzVector(L(0), 1, 2, 3, 4)})
run(ast("""
quad(x, y) = sqrt(x**2 + y**2)
q = quad(x.pz, x.E)
"""), symbols)
print(symbols)

class X(ID): pass
class Y(ID): pass
symbols = SymbolTable(builtins, {"x": [obj(X(0), a=0.0), obj(X(1), a=1.1), obj(X(2), a=2.2)],
                                 "y": [obj(Y(0), b="one"), obj(Y(1), b="two")]})
run(ast("""
z = join {
    xi ~ x
    yi ~ y
}
"""), symbols)
print(symbols)

events = uproot.open("http://scikit-hep.org/uproot/examples/HZZ.root")["events"]
Electron_Px, Electron_Py, Electron_Pz, Electron_E, Electron_Charge = events.arrays(["Electron_Px", "Electron_Py", "Electron_Pz", "Electron_E", "Electron_Charge"], outputtype=tuple, entrystop=33)
Muon_Px, Muon_Py, Muon_Pz, Muon_E, Muon_Charge = events.arrays(["Muon_Px", "Muon_Py", "Muon_Pz", "Muon_E", "Muon_Charge"], outputtype=tuple, entrystop=33)

higgs_finder = ast("""
higgs(flavor1, flavor2) =
    join {
        z1 ~ {
            lep1 ~ flavor1
            lep2 ~ flavor1
            mass = (lep1.p4 + lep2.p4).mass
        }
        z2 ~ {
            lep1 ~ flavor2
            lep2 ~ flavor2
            mass = (lep1.p4 + lep2.p4).mass
        }
    }.filter(h => h.z1.lep1.charge != h.z1.lep2.charge and
                  h.z2.lep1.charge != h.z2.lep2.charge)
     .sort(h => (h.z1.mass - 91)**2 + (h.z2.mass - 91)**2)
     [:1]

higgs4e    = higgs(electrons, electrons)
higgs4mu   = higgs(muons, muons)
higgs2e2mu = higgs(electrons, muons)
""")

class E(ID): pass
class M(ID): pass
for i in range(len(Muon_Px)):
    symbols = SymbolTable(builtins, {
        "electrons": [obj(E(j), p4=LorentzVector(E(j), Electron_Px[i][j], Electron_Py[i][j], Electron_Pz[i][j], Electron_E[i][j]), charge=Electron_Charge[i][j]) for j in range(len(Electron_Px[i]))],
        "muons": [obj(M(j), p4=LorentzVector(M(j), Muon_Px[i][j], Muon_Py[i][j], Muon_Pz[i][j], Muon_E[i][j]), charge=Muon_Charge[i][j]) for j in range(len(Muon_Px[i]))],
        })
    run(higgs_finder, symbols)

    del symbols["electrons"]
    del symbols["muons"]
    print(symbols)

max_stars_repo_name: diana-hep/gawk

max_issues_repo_name: diana-hep/gawk

max_forks_repo_name: diana-hep/gawk


text:
import numpy as np
import warnings
warnings.filterwarnings("ignore")
from pathlib import Path
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
from scipy.io import wavfile
import pyworld as pw
from scipy import signal
from util.utility import separate_speaker, get_separated_values
from LSTMLM.LSTMLM_generater import LSTMLM_generator
from StarGANVC_generator.StarGANVC_generator import StarGANVC_generator
from Basic_generator.AP_generator import AP_generator
from Basic_generator.F0_generator import F0_generator
from Basic_generator.MCEP_generator import MCEP_generator
from NPBDAA_generator.NPBDAA_generator import NPBDAA_generator
from NPBDAA_LM.Bigram_generator import Bigram_generator
from NPBDAA_LM.Unigram_generator import Unigram_generator

def denorm_mcep(generated_mcep, f0, domain_mean, domain_std):
    gmcep_mean = generated_mcep[f0!=0].mean(axis=0)
    gmcep_std = generated_mcep[f0!=0].std(axis=0)

    domain_mcep = domain_std / gmcep_std * (generated_mcep - gmcep_mean) + domain_mean
    return domain_mcep

def expanding_label(label_src, width=4):
    ones = np.ones((label_src.shape[0], width), dtype=int)
    labs = ones * label_src.reshape((-1, 1))
    return labs.reshape((-1, ))

parser = ArgumentParser(fromfile_prefix_chars='@', formatter_class=ArgumentDefaultsHelpFormatter)

parser.add_argument("--samplerate", type=int, default=48000)
parser.add_argument("--fftsize", type=int, default=1024)
parser.add_argument("--frame_period", type=float, default=5E-3)

parser.add_argument("--sentences_file", type=Path, required=True)

parser.add_argument("--letter_num", type=int, required=True)
parser.add_argument("--letter_stateseq", type=Path, required=True)
parser.add_argument("--ap", type=Path, required=True)
parser.add_argument("--f0", type=Path, required=True)

parser.add_argument("--flat_f0", action="store_true")
parser.add_argument("--flat_ap", action="store_true")

parser.add_argument("--parameter", type=Path)

parser.add_argument("--speaker_id", type=Path, required=True)
parser.add_argument("--mcep_norm_param", type=Path, nargs=2, required=True)

parser.add_argument("--target_speaker", type=str)
parser.add_argument("--stargan_generator", type=Path)

parser.add_argument("--output_prefix", type=Path)

parser.add_argument("--sentences", action="append", type=int, nargs="+")
parser.add_argument("--size", type=int, default=1)
parser.add_argument("--reinput", action="store_true")

parser.add_argument("--mode", choices=["ML", "RND"], default="ML")

parser.add_argument("--LM", choices=["LSTM", "Bigram", "Unigram"])
parser.add_argument("--unique", action="store_true")

parser.add_argument("--LSTM_model", type=Path)

args = parser.parse_args()

speakers, spkind_keys = separate_speaker(np.load(args.speaker_id))
letter_stateseq = np.load(args.letter_stateseq)
keys = sorted(list(letter_stateseq.keys()))
speaker_num = len(speakers)

target_idx = speakers.index(args.target_speaker)
src_letter_stateseq = get_separated_values(np.load(args.letter_stateseq), spkind_keys)[target_idx]
src_f0 = get_separated_values(np.load(args.f0), spkind_keys)[target_idx]
src_ap = get_separated_values(np.load(args.ap), spkind_keys)[target_idx]

mcep_mean = np.load(args.mcep_norm_param[0])[args.target_speaker]
mcep_std = np.load(args.mcep_norm_param[1])[args.target_speaker]

if args.sentences is None:
    if args.LM == "Unigram":
        snt_generator = Unigram_generator(args.sentences_file)
    elif args.LM == "Bigram":
        snt_generator = Bigram_generator(args.sentences_file, args.parameter)
    elif args.LM == "LSTM":
        snt_generator = LSTMLM_generator(args.LSTM_model, args.sentences_file)

ap_generator = AP_generator(args.letter_num, src_ap, letter_stateseq=src_letter_stateseq, flat=args.flat_ap, mode=args.mode)
f0_generator = F0_generator(args.letter_num, src_f0, letter_stateseq=src_letter_stateseq, flat=args.flat_f0, mode=args.mode)
feat_generator = NPBDAA_generator(args.parameter, mode=args.mode)

mcep_generator = StarGANVC_generator(args.stargan_generator, speaker_num)

if args.sentences is None:
    sentences = snt_generator.generate(size=args.size, unique=args.unique)
else:
    sentences = args.sentences
print(f"sentences: {sentences}")

for s, snt in enumerate(sentences):
    feature, gen_letter_stateseq = feat_generator.generate(snt)
    gen_letter_stateseq = expanding_label(gen_letter_stateseq)

    mcep = mcep_generator.generate(feature.astype(np.float32), target_idx, reinput=args.reinput)
    ap = ap_generator.generate(gen_letter_stateseq)
    f0 = f0_generator.generate(gen_letter_stateseq)
    f0[f0<0] = 0

    mcep = denorm_mcep(mcep, f0, mcep_mean, mcep_std)
    mcep = signal.medfilt(mcep, (5, 1))
    mcep = mcep.astype(float, order="C")

    decoded_sp = pw.decode_spectral_envelope(mcep, args.samplerate, args.fftsize)
    synthesized = pw.synthesize(f0, decoded_sp, ap, args.samplerate, frame_period=args.frame_period*1000)
    synthesized = synthesized / max(abs(synthesized)) * 30000

    args.output_prefix.parent.mkdir(parents=True, exist_ok=True)
    out_file = args.output_prefix.with_name(f"{args.output_prefix.name}_{s:02d}_({'_'.join(map(str, snt))}).wav")

    wavfile.write(out_file, args.samplerate, synthesized.astype(np.int16))

max_stars_repo_name: RyoOzaki/myResearch

max_issues_repo_name: RyoOzaki/myResearch

max_forks_repo_name: RyoOzaki/myResearch

reason: import numpy,from scipy


text:
from excel_io import get_original_data
import numpy as np
import matplotlib.pyplot as plt
import scipy.signal as signal
from scipy.signal import savgol_filter
import pickle
from example_load_original_data import read_data_example
from example_load_forces import read_position_and_force
from example_load_acceleration_data import example_load_pickle_acc
from compute_the_acce import filtering
import math
# counterclockwise is the positive direction

def findabsmax(l):

    max = 0
    idx =0
    max_id =-1
    for i in l:
        idx = idx+1
        if abs(i)>abs(max):
            max = i
            max_id = idx

    return max,max_id

def calForceX(m, GRFx, accx):
    return m*accx - GRFx

def calForceZ(m, GRFz, accz, g):
    return m*accz + m*g - GRFz

def calDistanceXZ(Point1, Point2):
    x = Point1[:-2,0]-Point2[:-2,0]
    z = Point1[:-2,0]-Point2[:-2,0]
    return abs(x/1000), abs(z/1000)

def calMoment(m, g, reactionForce, accdata, origianlData):
    ankle = origianlData["L_ankle"]
    footCG = origianlData["foot_CG"]
    GRF = reactionForce["Position"]
    dis_GRF_ankle_x,  dis_GRF_ankle_z = calDistanceXZ(GRF, ankle)
    dis_footCG_ankle_x, dis_footCG_ankle_z = calDistanceXZ(footCG, ankle)

    acc_foot = accdata["foot_CG"]
    acc_angle = accdata["Angle1"]
    I = 0.0035

    return -GRF[:-2,0]*dis_GRF_ankle_z - GRF[:-2,0]*dis_GRF_ankle_x + m*g*dis_footCG_ankle_x - acc_foot[:,0]/1000*m*dis_footCG_ankle_z - acc_foot[:,2]/1000*m*dis_footCG_ankle_x + I*acc_angle*math.pi/180

def getMaxMoment():
    originalData = read_data_example()

    file_path = '../data/reactionforce.forces'
    forceData = read_position_and_force(file_path, min_time=1.16, max_time=1.496)
    GRFx = forceData["Force"][:-2, 0]
    GRFz = forceData["Force"][:-2, 2]

    accData = example_load_pickle_acc()
    accx = accData['foot_CG'][:, 0]
    accz = accData['foot_CG'][:, 2]

    # moment
    moment = calMoment(0.76, 9.8, forceData, accData, originalData)

    # filtering for moment
    filtered_moment = filtering(moment)
    return findabsmax(filtered_moment)

if __name__=="__main__":

    # prepare data
    originalData = read_data_example()

    file_path = '../data/reactionforce.forces'
    forceData = read_position_and_force(file_path,min_time=1.16,max_time=1.496)
    GRFx = forceData["Force"][:-2,0]
    GRFz = forceData["Force"][:-2, 2]

    accData = example_load_pickle_acc()
    accx = accData['foot_CG'][:,0]
    accz = accData['foot_CG'][:, 2]

    # join force
    Fx = calForceX(0.76, GRFx, accx/1000)
    Fz = calForceZ(0.76, GRFz, accz/1000, 9.8)

    # moment
    moment = calMoment(0.76, 9.8, forceData, accData, originalData)

    # filtering for moment
    filtered_moment = filtering(moment)
    print(filtered_moment)

    #graph for moment
    x = [i for i in range(len(filtered_moment))]
    plt.title("Moment Graph")
    plt.plot(x, -filtered_moment)
    max,max_idx = findabsmax(-filtered_moment)

    plt.scatter(np.array([max_idx-1]),np.array([max]),c='none',marker='o',edgecolors='r')
    plt.text(max_idx-1,max-3,s="{}".format(max),color='red')

    plt.show()

max_stars_repo_name: Magicboomliu/Bio-SystemII-Tokyo-Tech

max_issues_repo_name: Magicboomliu/Bio-SystemII-Tokyo-Tech

max_forks_repo_name: Magicboomliu/Bio-SystemII-Tokyo-Tech

reason: import numpy,import scipy,from scipy


text:
""" Code for the MAML algorithm and network definitions. """
from __future__ import print_function
import numpy as np
import sys
import tensorflow as tf
try:
    import special_grads
except KeyError as e:
    print('WARN: Cannot define MaxPoolGrad, likely already defined for this version of tensorflow: %s' % e,
          file=sys.stderr)

from tensorflow.python.platform import flags
from utils import mse, xent, conv_block, normalize

FLAGS = flags.FLAGS

auto = True
encodeWeight = 0.3

class MAML:
    def __init__(self, dim_input=1, dim_output=1, test_num_updates=5):
        """ must call construct_model() after initializing MAML! """
        self.dim_input = dim_input
        self.dim_output = dim_output
        self.update_lr = FLAGS.update_lr
        self.meta_lr = tf.placeholder_with_default(FLAGS.meta_lr, ())
        self.auto_lr = tf.placeholder_with_default(FLAGS.auto_lr, ())

        self.classification = False
        self.test_num_updates = test_num_updates
        self.dim_auto = 2 #This should be able to be arbitrary
        if auto:
            self.real_input = 39 # This is square root of the total (its a kernel)
            #self.real_output = 40#self.dim_output
            self.real_output = 39*39 # This should be the complete dimension out.
            self.dim_input = 3*self.dim_auto #= 3*self.dim_auto
            self.dim_output = self.dim_auto
            #This is from each.
            #if auto: self.dim_input, self.dim_output = self.dim_auto, self.dim_auto #If auto, pass in/out the dimension of the latent (auto_
        if FLAGS.datasource == 'sinusoid':
            self.dim_hidden = [40, 40,40]
            self.loss_func = mse
            self.forward = self.forward_fc
            self.construct_weights = self.construct_fc_weights
        elif FLAGS.datasource == 'omniglot' or FLAGS.datasource == 'miniimagenet':
            self.loss_func = xent
            self.classification = True
            if FLAGS.conv:
                self.dim_hidden = FLAGS.num_filters
                self.forward = self.forward_conv
                self.construct_weights = self.construct_conv_weights
            else:
                self.dim_hidden = [256, 128, 64, 64]
                self.forward=self.forward_fc
                self.construct_weights = self.construct_fc_weights
            if FLAGS.datasource == 'miniimagenet':
                self.channels = 3
            else:
                self.channels = 1
            self.img_size = int(np.sqrt(self.dim_input/self.channels))
        else:
            raise ValueError('Unrecognized data source.')

    def construct_model(self, input_tensors=None, prefix='metatrain_'):
        # a: training data for inner gradient, b: test data for meta gradient
        if input_tensors is None:
            self.inputa1 = tf.placeholder(tf.float32)#,shape=[None,100,2,2])
            self.inputa2 = tf.placeholder(tf.float32)
            self.inputa3 = tf.placeholder(tf.float32)
            self.inputb1 = tf.placeholder(tf.float32)
            self.inputb2 = tf.placeholder(tf.float32)
            self.inputb3 = tf.placeholder(tf.float32)
            self.labela = tf.placeholder(tf.float32)
            self.labelb = tf.placeholder(tf.float32)
        else:
            self.inputa1 = input_tensors['inputa1']
            self.inputa2 = input_tensors['inputa2']
            self.inputa3 = input_tensors['inputa3']
            self.inputb1 = input_tensors['inputb1']
            self.inputb2 = input_tensors['inputb2']
            self.inputb3 = input_tensors['inputb3']
            self.labela = input_tensors['labela']
            self.labelb = input_tensors['labelb']

        with tf.variable_scope('model', reuse=None) as training_scope:
            if 'weights' in dir(self):
                training_scope.reuse_variables()
                weights = self.weights
                auto_weights = self.auto_weights
            else:
                # Define the weights
                self.weights = weights = self.construct_weights()
                self.auto_weights = auto_weights = self.construct_auto_weights()

            # outputbs[i] and lossesb[i] is the output and loss after i+1 gradient updates
            lossesa, outputas, lossesb, outputbs = [], [], [], []
            accuraciesa, accuraciesb = [], []
            num_updates = max(self.test_num_updates, FLAGS.num_updates)
            outputbs = [[]]*num_updates
            lossesb = [[]]*num_updates
            accuraciesb = [[]]*num_updates

            def task_metalearn(inp, reuse=True):
                """ Perform gradient descent for one task in the meta-batch. """
                inputa1,inputa2,inputa3, inputb1,inputb2,inputb3, labela, labelb = inp
                print("Input a: " , inputa1)
                task_outputbs, task_lossesb, auto_losses = [], [], []
                auto_loss = None

                #One for each.

                # This takes in the input and passes out the latent variables.
                temp_in_a_1 = self.encoder(inputa1,self.auto_weights, reuse=reuse)
                #Then transform it back, and take the loss
                temp_out_a_1 = self.decoder(temp_in_a_1,self.auto_weights, reuse=reuse)
                auto_out_a_1 = temp_out_a_1

                temp_in_a_2 = self.encoder(inputa2,self.auto_weights, reuse=reuse)
                #Then transform it back, and take the loss
                temp_out_a_2 = self.decoder(temp_in_a_2,self.auto_weights, reuse=reuse)
                auto_out_a_2 = temp_out_a_2

                temp_in_a_3 = self.encoder(inputa3,self.auto_weights, reuse=reuse)
                #Then transform it back, and take the loss
                temp_out_a_3 = self.decoder(temp_in_a_3,self.auto_weights, reuse=reuse)
                auto_out_a_3 = temp_out_a_3

               # This takes in the input and passes out the latent variables.
                temp_in_b_1 = self.encoder(inputb1,self.auto_weights, reuse=reuse)
                #Then transform it back, and take the loss
                temp_out_b_1 = self.decoder(temp_in_b_1,self.auto_weights, reuse=reuse)
                auto_out_b_1 = temp_out_b_1

                temp_in_b_2 = self.encoder(inputb2,self.auto_weights, reuse=reuse)
                #Then transform it back, and take the loss
                temp_out_b_2 = self.decoder(temp_in_b_2,self.auto_weights, reuse=reuse)
                auto_out_b_2 = temp_out_b_2

                temp_in_b_3 = self.encoder(inputb3,self.auto_weights, reuse=reuse)
                #Then transform it back, and take the loss
                temp_out_b_3 = self.decoder(temp_in_b_3,self.auto_weights, reuse=reuse)
                auto_out_b_3 = temp_out_b_3

                #print("temp out a: " , temp_out_a)
                la_1 = self.loss_func(temp_out_a_1,inputa1)
                la_2 = self.loss_func(temp_out_a_2,inputa2)
                la_3 = self.loss_func(temp_out_a_3,inputa3)

                lb_1 = self.loss_func(temp_out_b_1,inputb1)
                lb_2 = self.loss_func(temp_out_b_2,inputb2)
                lb_3 = self.loss_func(temp_out_b_3,inputb3)

                auto_loss = la_1+la_2+la_3

                print("Input a1: " , temp_in_a_1)
                inputa=tf.concat([temp_in_a_1, temp_in_a_2,temp_in_a_3],1)
                inputb=tf.concat([temp_in_b_1, temp_in_b_2,temp_in_b_3],1)
                print("Inputa: " , inputa)
                task_outputa = self.forward(inputa, weights, reuse=reuse)  # only reuse on the first iter
                temp_outputa = self.decoder(task_outputa,self.auto_weights,reuse=reuse)

                print("Task outputa: " , temp_outputa)
                print("Label a: " , labela)
                task_lossa = self.loss_func(temp_outputa, labela)

                grads = tf.gradients(task_lossa, list(weights.values()))
                if FLAGS.stop_grad:
                    grads = [tf.stop_gradient(grad) for grad in grads]
                gradients = dict(zip(weights.keys(), grads))
                fast_weights = dict(zip(weights.keys(), [weights[key] - self.update_lr*gradients[key] for key in weights.keys()]))
                output = self.forward(inputb, fast_weights, reuse=True)

                temp_outputb = self.decoder(output,self.auto_weights, reuse=True)
                output = temp_outputb

                task_outputbs.append(output)
                print("Output: " , output)
                print("Labels: " , labelb)
                task_lossesb.append(self.loss_func(output, labelb))
                print("Num updates is: " , num_updates-1)

                for j in range(num_updates - 1):
                    loss = self.loss_func(self.decoder(self.forward(inputa, fast_weights, reuse=True),self.auto_weights), labela)
                    grads = tf.gradients(loss, list(fast_weights.values()))
                    if FLAGS.stop_grad:
                        grads = [tf.stop_gradient(grad) for grad in grads]
                    gradients = dict(zip(fast_weights.keys(), grads))
                    fast_weights = dict(zip(fast_weights.keys(), [fast_weights[key] - self.update_lr*gradients[key] for key in fast_weights.keys()]))

                    output = self.forward(inputb, fast_weights, reuse=True)
                    output = self.decoder(output,self.auto_weights)

                    task_outputbs.append(output)
                    task_lossesb.append(self.loss_func(output, labelb))
                # Task Outputa  Direct output, non-decoded (only first step). This is the location (so it is in hyperparameter space)
                # Task_outputbs Output from complete system, for each step (first is after 1st step, 2nd is after next, etc.)
                # task_lossa    Loss of difference in first step (decoded)
                # task_lossesb   Losses from complete ssystem from each step
                # auto_loss      Sum of all auto-encoding loss of a's
                # auto_out_a_1    Auto-encoded a1 output
                # auto_out_a_2    Auto-encoded a2 output

                task_output = [temp_outputa, task_outputbs, task_lossa, task_lossesb, auto_loss,auto_out_a_1,auto_out_a_2]
                return task_output

            out_dtype = [tf.float32, [tf.float32]*num_updates, tf.float32, [tf.float32]*num_updates]

            out_dtype.extend([tf.float32,tf.float32,tf.float32])

            result = tf.map_fn(task_metalearn, elems=(self.inputa1,self.inputa2,self.inputa3,self.inputb1,self.inputb2,self.inputb3, self.labela, self.labelb), dtype=out_dtype, parallel_iterations=FLAGS.meta_batch_size)
            #In case you want to fetch it.
            #if auto:
                #auto_losses = [1,2,3,4]
            outputas, outputbs, lossesa, lossesb, autoloss, auto_out_a,auto_out_b  = result

            self.outputbs = outputbs
        #print("Meta batch: " , FLAGS.meta_batch_size)
        ## Performance & Optimization

        self.l1_regularizer_e = tf.contrib.layers.l1_regularizer(
           scale=FLAGS.encoderregularize_penal, scope=None
        )
        self.weights1 = tf.trainable_variables() # all vars of your graph
        regularization_penalty_e = tf.contrib.layers.apply_regularization(self.l1_regularizer_e, self.weights1)

        self.l1_regularizer_p = tf.contrib.layers.l1_regularizer(
           scale=FLAGS.predictregularize_penal, scope=None
        )

        regularization_penalty_p = tf.contrib.layers.apply_regularization(self.l1_regularizer_p, self.weights1)

        if 'train' in prefix:
            # auto_losses - mean of all auto-encoding loss of a's
            self.auto_losses = tf.reduce_sum(autoloss) / tf.to_float(FLAGS.meta_batch_size)
            # This is the loss a (before going into steps)
            self.total_loss1 = total_loss1 = tf.reduce_sum(lossesa) / tf.to_float(FLAGS.meta_batch_size)
            # This is the loss b (within the steps)
            self.total_losses2 = total_losses2 = [tf.reduce_sum(lossesb[j]) / tf.to_float(FLAGS.meta_batch_size) for j in range(num_updates)]

            self.outputas, self.outputbs = outputas, outputbs
            self.auto_out_a, self.auto_out_b = auto_out_a, auto_out_b

            #self.autotrain_op = tf.train.AdamOptimizer(self.auto_lr).minimize(self.auto_losses + regularization_penalty_e)
            self.autotrain_op = tf.train.AdamOptimizer(self.auto_lr).minimize(self.auto_losses)

            self.pretrain_op = tf.train.AdamOptimizer(self.meta_lr).minimize(total_loss1+encodeWeight*self.auto_losses+ regularization_penalty_e) #For pretraining as well.
            if FLAGS.metatrain_iterations > 0:
                #print("Meta train....")
                optimizer = tf.train.AdamOptimizer(self.meta_lr)
                lossPenal = self.total_losses2[FLAGS.num_updates-1]
                lossPenal = lossPenal + regularization_penalty_e + encodeWeight*self.auto_losses
                self.gvs = gvs = optimizer.compute_gradients(lossPenal)
                self.metatrain_op = optimizer.apply_gradients(gvs)
        else:
            print("Validation, Meta batch size: " , FLAGS.meta_batch_size)
            self.metaval_total_loss1 = total_loss1 = tf.reduce_sum(lossesa) / tf.to_float(FLAGS.meta_batch_size)
            self.metaval_total_losses2 = total_losses2 = [tf.reduce_sum(lossesb[j]) / tf.to_float(FLAGS.meta_batch_size) for j in range(num_updates)]
            #if self.classification:
            #    self.metaval_total_accuracy1 = total_accuracy1 = tf.reduce_sum(accuraciesa) / tf.to_float(FLAGS.meta_batch_size)
            #    self.metaval_total_accuracies2 = total_accuracies2 =[tf.reduce_sum(accuraciesb[j]) / tf.to_float(FLAGS.meta_batch_size) for j in range(num_updates)]

        ## Summaries
        tf.summary.scalar(prefix+'Pre-update loss', total_loss1)
        if self.classification:
            tf.summary.scalar(prefix+'Pre-update accuracy', total_accuracy1)

        for j in range(num_updates):
            tf.summary.scalar(prefix+'Post-update loss, step ' + str(j+1), total_losses2[j])
            if self.classification:
                tf.summary.scalar(prefix+'Post-update accuracy, step ' + str(j+1), total_accuracies2[j])

    ### Network construction functions (fc networks and conv networks)
    def construct_fc_weights(self):
        weights = {}
        weights['w1'] = tf.Variable(tf.truncated_normal([self.dim_input, self.dim_hidden[0]], stddev=0.01))
        weights['b1'] = tf.Variable(tf.zeros([self.dim_hidden[0]]))
        print("Weights w1: " , weights['w1'].shape)
        for i in range(1,len(self.dim_hidden)):
            weights['w'+str(i+1)] = tf.Variable(tf.truncated_normal([self.dim_hidden[i-1], self.dim_hidden[i]], stddev=0.01))
            weights['b'+str(i+1)] = tf.Variable(tf.zeros([self.dim_hidden[i]]))
        weights['w'+str(len(self.dim_hidden)+1)] = tf.Variable(tf.truncated_normal([self.dim_hidden[-1], self.dim_output], stddev=0.01))
        weights['b'+str(len(self.dim_hidden)+1)] = tf.Variable(tf.zeros([self.dim_output]))
        return weights

    # This constructs the autoencoder weights.
    def construct_auto_weights(self):
        weights = {}
        weights['encoder_a'] = tf.Variable(tf.truncated_normal([512,2],stddev=0.01))
        weights['encoder_bias'] = tf.Variable(tf.zeros([2]))

        print("Encoder a: " , weights['encoder_a'].shape)
        weights['decoder_a'] = tf.Variable(tf.truncated_normal([2,512],stddev=0.01))
        weights['decoder_bias'] = tf.Variable(tf.zeros([32]))
        dtype = tf.float32
        conv_initializer =  tf.contrib.layers.xavier_initializer_conv2d(dtype=dtype)

        channels = 1
        weights['conv1'] = tf.get_variable('conv1', [3, 3, channels, 32], dtype=dtype)
        weights['b1_e'] = tf.Variable(tf.zeros([32]))
        weights['conv2'] = tf.get_variable('conv2', [3, 3, 32, 32], initializer=conv_initializer, dtype=dtype)

        weights['b2_e'] = tf.Variable(tf.zeros([32]))
        weights['conv3'] = tf.get_variable('conv3', [3, 3, 32,32], initializer=conv_initializer, dtype=dtype)
        weights['b3_e'] = tf.Variable(tf.zeros([32]))

        weights['dec_t'] = tf.Variable(tf.zeros([1]))

        weights['conv1_t'] = tf.get_variable('conv1_t', [3, 3, 32, 32], initializer=conv_initializer, dtype=dtype)
        weights['b1_t'] = tf.Variable(tf.zeros([32]))
        weights['conv2_t'] = tf.get_variable('conv2_t', [3, 3, 32, 32], initializer=conv_initializer, dtype=dtype)

        weights['b2_t'] = tf.Variable(tf.zeros([32]))
        weights['conv3_t'] = tf.get_variable('conv3_t', [3, 3, 1,32], initializer=conv_initializer, dtype=dtype)
        weights['b3_t'] = tf.Variable(tf.zeros([39,39,1]))
        #weights['conv4'] = tf.get_variable('conv4', [k, k, self.dim_hidden, self.dim_hidden], initializer=conv_initializer, dtype=dtype)
        #weights['b4'] = tf.Variable(tf.zeros([32]))
        return weights

    #Take the entire and responds with the small latent.
    def encoder(self,inp,weights,reuse=False,scope=''):
        #print("Weights: " , weights.keys())
        #weights['encoder_a'] = weights['w1']
        print("Input: " , inp)

        my_inp = tf.reshape(inp,[-1,39,39,1])
        print("My input: " , my_inp)

        conv1 = normalize(tf.nn.conv2d(my_inp,weights['conv1'],[1,2,2,1],"VALID")+weights['b1_e'],activation=tf.nn.relu,reuse=reuse,scope=scope+"0")
        print("Conv", conv1)

        conv2 = normalize(tf.nn.conv2d(conv1,weights['conv2'],[1,2,2,1],"VALID")+weights['b2_e'],activation=tf.nn.relu,reuse=reuse,scope=scope+"1")
        print("Conv2: " , conv2)

        conv3 = normalize(tf.nn.conv2d(conv2,weights['conv3'],[1,2,2,1],"VALID")+weights['b3_e'],activation=tf.nn.relu,reuse=reuse,scope=scope+"2")
        print("Conv3 : " , conv3)
        #conv2 = tf.layers.conv2d(inputs=conv1,filters=32,kernel_size=[3,3],padding="valid",activation=tf.nn.relu,strides=[2,2])
        #conv3 = tf.layers.conv2d(inputs=conv2,filters=32,kernel_size=[3,3],padding="valid",activation=tf.nn.relu,strides=[2,2])
        print("Conv network: " , conv3)
        #os.exit()
        pool2_flat = tf.reshape(conv3, [-1, 512])

        print("Pool flat: " , pool2_flat)

        layer_2 = tf.matmul(pool2_flat,weights['encoder_a']) + weights['encoder_bias']
        print("Lay2: " , layer_2)
        #os.exit()

        #print("Pool 2 fat: " , pool2_flat)
        return layer_2

    #Takes in the small latent and replies with the entire.
    def decoder(self,inp,weights,reuse=False,scope=''):

        print("InputT: " , inp)

        layer_2 = tf.matmul(inp,weights['decoder_a']) + weights['dec_t']

        ## TODO ADD A WEIGHT IN THIS.
        print("Layer 2: " , layer_2)
        my_inp = tf.reshape(layer_2,[-1,4,4,32])

        #conv1 = normalize(tf.nn.conv2d_transpose(my_inp,weights['conv1_t'],[FLAGS.update_batch_size,9,9,32],[1,2,2,1],padding="VALID"),activation=tf.nn.relu,reuse=reuse,scope=scope+"0")

        conv1 = normalize(tf.nn.conv2d_transpose(my_inp,weights['conv1_t'],[FLAGS.update_batch_size,9,9,32],[1,2,2,1],padding="VALID")+ weights['b1_t'],activation=tf.nn.relu,reuse=reuse,scope=scope+"0")
        print("Conv 1: ", conv1)

        #conv2 = normalize(tf.nn.conv2d_transpose(conv1,weights['conv2_t'],[FLAGS.update_batch_size,19,19,32],[1,2,2,1],padding="VALID"),activation=tf.nn.relu,reuse=reuse,scope=scope+"1")
        conv2 = normalize(tf.nn.conv2d_transpose(conv1,weights['conv2_t'],[FLAGS.update_batch_size,19,19,32],[1,2,2,1],padding="VALID")+ weights['b2_t'],activation=tf.nn.relu,reuse=reuse,scope=scope+"1")
        print("Conv 2: " , conv2)

        #conv3 = normalize(tf.nn.conv2d_transpose(conv2,weights['conv3_t'],[FLAGS.update_batch_size,39,39,1],[1,2,2,1],padding="VALID"),activation=tf.nn.relu,reuse=reuse,scope=scope+"2")
        conv3 = normalize(tf.nn.conv2d_transpose(conv2,weights['conv3_t'],[FLAGS.update_batch_size,39,39,1],[1,2,2,1],padding="VALID")+ weights['b3_t'],activation=tf.nn.relu,reuse=reuse,scope=scope+"1")
        print("Conv 3: " , conv3)

        return conv3

    def forward_fc(self, inp, weights, reuse=False):
        print("Weights:", weights.keys())
        #new_in = self.encoder(inp,weights)

        hidden = normalize(tf.matmul(inp, weights['w1']) + weights['b1'], activation=tf.nn.relu, reuse=reuse, scope='0')
        print("Hidden: " , hidden)
        print("Weights w1: " , weights['w1'])
        for i in range(1,len(self.dim_hidden)):
            hidden = normalize(tf.matmul(hidden, weights['w'+str(i+1)]) + weights['b'+str(i+1)], activation=tf.nn.relu, reuse=reuse, scope=str(i+1))
        return tf.matmul(hidden, weights['w'+str(len(self.dim_hidden)+1)]) + weights['b'+str(len(self.dim_hidden)+1)]

    def construct_conv_weights(self):
        weights = {}

        dtype = tf.float32
        conv_initializer =  tf.contrib.layers.xavier_initializer_conv2d(dtype=dtype)
        fc_initializer =  tf.contrib.layers.xavier_initializer(dtype=dtype)
        k = 3

        weights['conv1'] = tf.get_variable('conv1', [k, k, self.channels, self.dim_hidden], initializer=conv_initializer, dtype=dtype)
        weights['b1'] = tf.Variable(tf.zeros([self.dim_hidden]))
        weights['conv2'] = tf.get_variable('conv2', [k, k, self.dim_hidden, self.dim_hidden], initializer=conv_initializer, dtype=dtype)
        weights['b2'] = tf.Variable(tf.zeros([self.dim_hidden]))
        weights['conv3'] = tf.get_variable('conv3', [k, k, self.dim_hidden, self.dim_hidden], initializer=conv_initializer, dtype=dtype)
        weights['b3'] = tf.Variable(tf.zeros([self.dim_hidden]))
        weights['conv4'] = tf.get_variable('conv4', [k, k, self.dim_hidden, self.dim_hidden], initializer=conv_initializer, dtype=dtype)
        weights['b4'] = tf.Variable(tf.zeros([self.dim_hidden]))
        if FLAGS.datasource == 'miniimagenet':
            # assumes max pooling
            weights['w5'] = tf.get_variable('w5', [self.dim_hidden*5*5, self.dim_output], initializer=fc_initializer)
            weights['b5'] = tf.Variable(tf.zeros([self.dim_output]), name='b5')
        else:
            weights['w5'] = tf.Variable(tf.random_normal([self.dim_hidden, self.dim_output]), name='w5')
            weights['b5'] = tf.Variable(tf.zeros([self.dim_output]), name='b5')
        return weights

    def forward_conv(self, inp, weights, reuse=False, scope=''):
        # reuse is for the normalization parameters.
        channels = self.channels
        inp = tf.reshape(inp, [-1, self.img_size, self.img_size, channels])

        hidden1 = conv_block(inp, weights['conv1'], weights['b1'], reuse, scope+'0')
        hidden2 = conv_block(hidden1, weights['conv2'], weights['b2'], reuse, scope+'1')
        hidden3 = conv_block(hidden2, weights['conv3'], weights['b3'], reuse, scope+'2')
        hidden4 = conv_block(hidden3, weights['conv4'], weights['b4'], reuse, scope+'3')
        if FLAGS.datasource == 'miniimagenet':
            # last hidden layer is 6x6x64-ish, reshape to a vector
            hidden4 = tf.reshape(hidden4, [-1, np.prod([int(dim) for dim in hidden4.get_shape()[1:]])])
        else:
            hidden4 = tf.reduce_mean(hidden4, [1, 2])

        return tf.matmul(hidden4, weights['w5']) + weights['b5']

max_stars_repo_name: iguanaus/maml-auto

max_issues_repo_name: iguanaus/maml-auto

max_forks_repo_name: iguanaus/maml-auto


text:
# -*- coding: utf-8 -*-
"""
Created on Thu Dec 10 16:23:53 2020

@author: IfritR
"""
import numpy as np
import matplotlib.pyplot as plt
from sklearn.cluster import KMeans
from sklearn.metrics import davies_bouldin_score
from sklearn.decomposition import PCA
import pandas as pd
import seaborn as sns
from sklearn.preprocessing import StandardScaler

url="https://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-white.csv"

wine = pd.read_csv(url, sep = ';')
wine.mean().plot(kind='bar')
wine_dropped = wine.drop('quality', axis=1)
# Normalizing over the standard deviation
X =wine_dropped.values[:, 1:]
#X=StandardScaler().fit_transform(X)
#X = wine.drop('quality', axis=1)
x_cols=wine_dropped.columns
#X2=X.to_numpy(copy=True)
y=wine['quality']
#y2=y.to_numpy(copy=True)

#box plot for features
f2, ax = plt.subplots(2, 2, figsize=(16, 12))
sns.boxplot('quality', 'alcohol', data=wine, ax=ax[0, 0], palette='Blues')
sns.boxplot('quality', 'sulphates', data=wine, ax=ax[0, 1], palette='Blues')
sns.boxplot('quality', 'volatile acidity', data=wine, ax=ax[1, 0], palette='Blues')
sns.boxplot('quality', 'citric acid', data=wine, ax=ax[1, 1], palette='Blues')

#Plot elbow method figure
plt.figure(6);
distortions = []
for i in range (1,11):
    km = KMeans(n_clusters=i,
               max_iter=300,
               random_state=2020)
    km.fit(X)
    distortions.append(km.inertia_)

plt.plot(range(1,11), distortions, marker='o')
plt.xlabel('Number of clusters')
plt.ylabel('Distortion')
plt.show()

#Kmeans clustering
kmeans =KMeans(init='k-means++',n_clusters=2,random_state=2020,n_init=11)
kmeans.fit(X)
wine_labels = kmeans.labels_
wine_centers = kmeans.cluster_centers_
distX = kmeans.transform(X);
dist_center = kmeans.transform(wine_centers);
sse = kmeans.inertia_# Sum of Squared Errors(SSE)
score = kmeans.score(X)
wine_dropped['Cluster'] = wine_labels#add cluster column to data
cluster_details=wine_dropped.groupby('Cluster').mean()#means group by clusters
print(cluster_details)

# Visualizing of clustering in the distance space
fig = plt.figure(4);
plt.title('Wine data in the distance space');
plt.xlabel('Cluster 1');
plt.ylabel('Cluster 2');
plt.scatter(distX[:,0],distX[:,1],s=50,c=wine_labels);  # data
plt.scatter(dist_center[:,0],dist_center[:,1],s=10,c='red',marker='X');  # centroids
plt.show();

#Calculate davies boulding score
DB = davies_bouldin_score(X,wine_labels)

print(f'Within SSE: {sse}')
print(f'Davies-Bouldin index: {DB}')

"""
total sulfur dioxide:
amount of free and bound forms of S02; in low concentrations,
SO2 is mostly undetectable in wine, but at free SO2
"""
#plot
fig = plt.figure(3,figsize=(12,12));
plt.xlabel('alcohol', fontsize=18)
plt.ylabel('total sulfur dioxide', fontsize=16)
plt.scatter(X[:,9],X[:,5],c=wine_labels,label=wine_labels)  # data
plt.scatter(wine_centers[:,9],wine_centers[:,5],c='red',marker='X')  # centroids
plt.show()
plt.xlabel('alcohol', fontsize=18)
plt.ylabel('total sulfur dioxide', fontsize=16)
sns.scatterplot(x=X[:,9], y=X[:,5], hue="Cluster",
                data=wine_dropped, palette='Paired', s=20);
plt.legend(loc='lower right');

#PCA
pca = PCA(n_components=2);
pca.fit(X);
wine_pc = pca.transform(X);  #  data coordinates in the PC space
centers_pc = pca.transform(wine_centers);
score_pc=pca.score(X)
print("PCA score:",score_pc)

#plot pca clustering
fig = plt.figure(6,figsize=(12,12))
plt.title('Clustering of the Wine data after PCA')
plt.xlabel('PC1')
plt.ylabel('PC2')
plt.scatter(wine_pc[:,0],wine_pc[:,1],s=50,c=wine_labels)  # data
plt.scatter(centers_pc[:,0],centers_pc[:,1],s=200,c='red',marker='X')  # centroids
plt.show()
plt.title('Clustering of the Wine data after PCA')
plt.xlabel('PC1')
plt.ylabel('PC2')
sns.scatterplot(x=wine_pc[:,0], y=wine_pc[:,1], hue="Cluster",
                data=wine_dropped, palette='Paired', s=20)

#PCA2
X2=wine.drop('quality', axis=1).values
y2=wine['quality'].values
X2 = StandardScaler().fit_transform(X2)#Normalizing datas
pca2 = PCA(n_components=2)
principalComponents = pca2.fit_transform(X2)
principalDf = pd.DataFrame(data = principalComponents
             , columns = ['principal component 1', 'principal component 2'])
finalDF=pd.concat([principalDf,wine['quality']],axis=1)
score_pc2=pca2.score(X2)
print("PCA2 Score",score_pc2)

#plot pca2 clusters
fig = plt.figure(9,figsize = (8,8))
ax = fig.add_subplot(1,1,1)
ax.set_xlabel('Principal Component 1', fontsize = 15)
ax.set_ylabel('Principal Component 2', fontsize = 15)
targets = [5,7, 8]#Which original qualitys are the targets
colors = ['r', 'g','b']
for target, color in zip(targets,colors):
    indicesToKeep = finalDF['quality'] == target
    ax.scatter(finalDF.loc[indicesToKeep, 'principal component 1']
               , finalDF.loc[indicesToKeep, 'principal component 2']
               , c = color
               , s = 10)
ax.legend(targets)
ax.grid()

# Finding optimal cluster number
Max_K = 31  # maximum cluster number
SSE = np.zeros((Max_K-2))  #  array for sum of squares errors
DB = np.zeros((Max_K-2))  # array for Davies Bouldin indeces
for i in range(Max_K-2):
    n_c = i+2
    kmeans = KMeans(n_clusters=n_c, random_state=2020)
    kmeans.fit(X)
    wine_labels = kmeans.labels_
    SSE[i] = kmeans.inertia_
    DB[i] = davies_bouldin_score(X,wine_labels)

#Show optimal cluster number
fig = plt.figure(10)
plt.title('Sum of squares of error curve')
plt.xlabel('Number of clusters')
plt.ylabel('SSE')
plt.plot(np.arange(2,Max_K),SSE, color='red')
plt.show()

# Visualization of DB scores
fig = plt.figure(11)
plt.title('Davies-Bouldin score curve')
plt.xlabel('Number of clusters')
plt.ylabel('DB index')
plt.plot(np.arange(2,Max_K),DB, color='blue')
plt.show()


text:
# find conturs from openCV in out BB
import os
import cv2
import numpy as np
import tensorflow as tf
import sys
from datetime import datetime
import time
from utils import label_map_util
from utils import visualization_utils as vis_util
MODEL_NAME = 'inference_graph'
IMAGE_NAME = 'img'

CWD_PATH = ''

PATH_TO_CKPT = os.path.join(CWD_PATH,MODEL_NAME,'frozen_inference_graph.pb')

PATH_TO_LABELS = os.path.join(CWD_PATH,'data','object-detection.pbtxt')
PATH_TO_IMAGE = os.path.join(CWD_PATH,'img')
NUM_CLASSES = 2

label_map = label_map_util.load_labelmap(PATH_TO_LABELS)
categories = label_map_util.convert_label_map_to_categories(label_map, max_num_classes=NUM_CLASSES, use_display_name=True)
category_index = label_map_util.create_category_index(categories)
# Load the Tensorflow model into memory.
detection_graph = tf.Graph()
with detection_graph.as_default():
    od_graph_def = tf.GraphDef()
    with tf.gfile.GFile(PATH_TO_CKPT, 'rb') as fid:
        serialized_graph = fid.read()
        od_graph_def.ParseFromString(serialized_graph)
        tf.import_graph_def(od_graph_def, name='')

    sess = tf.Session(graph=detection_graph)
image_tensor = detection_graph.get_tensor_by_name('image_tensor:0')
detection_boxes = detection_graph.get_tensor_by_name('detection_boxes:0')
detection_scores = detection_graph.get_tensor_by_name('detection_scores:0')
detection_classes = detection_graph.get_tensor_by_name('detection_classes:0')
num_detections = detection_graph.get_tensor_by_name('num_detections:0')

start_time = datetime.now()
PATH_TO_SAVE='output'
with os.scandir(PATH_TO_IMAGE) as entries:
    for entry in entries:
        image = cv2.imread(os.path.join(PATH_TO_IMAGE,entry.name))
        start_time = datetime.now()
        image_expanded = np.expand_dims(image, axis=0)
        (boxes, scores, classes, num) = sess.run(
            [detection_boxes, detection_scores, detection_classes, num_detections],
            feed_dict={image_tensor: image_expanded})
        scores=np.squeeze(scores)
        boxes=np.squeeze(boxes)
        classes=np.squeeze(classes)
        max_boxes_to_draw = boxes.shape[0]
        height = image.shape[0]
        width = image.shape[1]
        channels = image.shape[2]

        for i in range(min(max_boxes_to_draw, boxes.shape[0])):
            if scores is None or scores[i] > 0.25:
                if classes[i]==2:
                    box = tuple(boxes[i].tolist())
                    ymin, xmin, ymax, xmax = box
                    start_point = (int(xmin*height), int(ymin*width))
                    end_point = (int(xmax*height), int(ymax*width))
                    color = (255, 0, 0)
                    image1=image[int(ymin*width):int(ymax*width),int(xmin*height):int(xmax*height)]
                    savename=str(i)+entry.name
                    cv2.imwrite(os.path.join(PATH_TO_SAVE,"class1","cropped",savename), image1)
                    img_grey = cv2.cvtColor(image[int(ymin*width):int(ymax*width),int(xmin*height):int(xmax*height)], cv2.COLOR_BGR2GRAY)
                    thresh  = cv2.adaptiveThreshold(img_grey,255,cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY,15,2)
                    cv2.imwrite(os.path.join(PATH_TO_SAVE,"class1","thresh",savename), thresh)
                    conturs,hierarchy = cv2.findContours( thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
                    for cnt in conturs:
                        peri = cv2.arcLength(cnt, True)
                        if peri>100 and peri<300:
                            for c in cnt:
                                c[0][0]=c[0][0]+int(xmin*height)
                                c[0][1]=c[0][1]+int(ymin*height)
                            cv2.drawContours(image, [cnt], -1, (255, 0, 0),2, cv2.LINE_AA)
            if scores is None or scores[i] > 0.25:
                if classes[i]==1:
                    box = tuple(boxes[i].tolist())
                    ymin, xmin, ymax, xmax = box
                    start_point = (int(xmin*height), int(ymin*width))
                    end_point = (int(xmax*height), int(ymax*width))
                    color = (255, 0, 0)
                    savename=str(i)+entry.name
                    image1=image[int(ymin*width):int(ymax*width),int(xmin*height):int(xmax*height)]
                    cv2.imwrite(os.path.join(PATH_TO_SAVE,"class2","cropped",savename), image1)
                    img_grey = cv2.cvtColor(image[int(ymin*width):int(ymax*width),int(xmin*height):int(xmax*height)], cv2.COLOR_BGR2GRAY)
                    thresh  = cv2.adaptiveThreshold(img_grey,255,cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY,15,2)
                    cv2.imwrite(os.path.join(PATH_TO_SAVE,"class2","thresh",savename), thresh)
                    conturs,hierarchy = cv2.findContours( thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
                    for cnt in conturs:
                        peri = cv2.arcLength(cnt, True)
                        if peri>100 and peri<200:
                            for c in cnt:
                                c[0][0]=c[0][0]+int(xmin*height)
                                c[0][1]=c[0][1]+int(ymin*height)
                            cv2.drawContours(image, [cnt], -1, (0, 255, 0),2, cv2.LINE_AA)
        vis_util.visualize_boxes_and_labels_on_image_array(
        image,
        np.squeeze(boxes),
        np.squeeze(classes).astype(np.int32),
        np.squeeze(scores),
        category_index,
        use_normalized_coordinates=True,
        line_thickness=3,
        min_score_thresh=0.20)
        cv2.imwrite(os.path.join(PATH_TO_SAVE,entry.name), image)