text:
#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import numpy as np
from math import floor, ceil

# Extension module
import quadrotorsim

NO_DISPLAY = False
try:
    from rlschool.quadrotor.render import RenderWindow
except Exception as e:
    NO_DISPLAY = True

class Quadrotor(object):
    """
    Quadrotor environment.

    Args:
        dt (float): duration of single step (in seconds).
        nt (int): number of steps of single episode if no collision
            occurs.
        seed (int): seed to generate target velocity trajectory.
        task (str): name of the task setting. Currently, support
            `no_collision` and `velocity_control`.
        map_file (None|str): path to txt map config file, default
            map is a 100x100 flatten floor.
        simulator_conf (None|str): path to simulator config xml file.
    """
    def __init__(self,
                 dt=0.01,
                 nt=1000,
                 seed=0,
                 task='no_collision',
                 map_file=None,
                 simulator_conf=None,
                 **kwargs):
        assert task in ['velocity_control', 'no_collision'], \
            'Invalid task setting'
        if simulator_conf is None:
            simulator_conf = os.path.join(os.path.dirname(__file__),
                                          'quadrotorsim', 'config.xml')
        assert os.path.exists(simulator_conf), \
            'Simulator config xml does not exist'

        self.dt = dt
        self.nt = nt
        self.ct = 0
        self.task = task
        self.simulator = quadrotorsim.Simulator()
        self.simulator.get_config(simulator_conf)
        self.state = {}
        self.viewer = None
        self.x_offset = self.y_offset = self.z_offset = 0

        if self.task == 'velocity_control':
            self.velocity_targets = \
                self.simulator.define_velocity_control_task(
                    dt, nt, seed)
        elif self.task == 'no_collision':
            self.map_matrix = Quadrotor.load_map(map_file)

            # Only for single quadrotor, also mark its start position
            y_offsets, x_offsets = np.where(self.map_matrix == -1)
            assert len(y_offsets) == 1
            self.y_offset = y_offsets[0]
            self.x_offset = x_offsets[0]
            self.z_offset = 5.  # TODO: setup a better init height
            self.map_matrix[self.y_offset, self.x_offset] = 0

    def reset(self):
        self.simulator.reset()
        sensor_dict = self.simulator.get_sensor()
        state_dict = self.simulator.get_state()
        self._update_state(sensor_dict, state_dict)

        return self.state

    def step(self, action):
        self.ct += 1
        cmd = np.asarray(action, np.float32)
        self.simulator.step(cmd.tolist(), self.dt)
        sensor_dict = self.simulator.get_sensor()
        state_dict = self.simulator.get_state()

        old_pos = [self.state['x'], self.state['y'], self.state['z']]
        self._update_state(sensor_dict, state_dict)
        new_pos = [self.state['x'], self.state['y'], self.state['z']]
        if self.task == 'no_collision':
            is_collision = self._check_collision(old_pos, new_pos)
            reward = self._get_reward(collision=is_collision)
            reset = False
            if is_collision:
                reset = True
                self.ct = 0
        elif self.task == 'velocity_control':
            reset = False
            velocity_target = self.velocity_targets[self.ct - 1]
            reward = self._get_reward(velocity_target=velocity_target)

        if self.ct == self.nt:
            reset = True
            self.ct = 0

        return self.state, reward, reset

    def render(self):
        if self.viewer is None:
            if NO_DISPLAY:
                raise RuntimeError('[Error] Cannot connect to display screen.')
            self.viewer = RenderWindow(task=self.task)

        if 'x' not in self.state:
            # It's null state
            raise Exception('You are trying to render before calling reset()')

        if self.task == 'velocity_control':
            self.viewer.view(
                self.state, self.dt,
                expected_velocity=self.velocity_targets[self.ct-1])
        else:
            self.viewer.view(self.state, self.dt)

    def close(self):
        del self.simulator

    def _get_reward(self, collision=False, velocity_target=(0.0, 0.0, 0.0)):
        """
        Reward function setting for different tasks.

        The default penalty is the cost of energy. In addition,
        for `no_collision` task, a strong penalty is added for
        collision, otherwise get +1 reward; for `velocity_control`
        task, an extra penalty for velocity difference is added.
        """
        reward = - self.dt * self.state['power']
        if self.task == 'no_collision':
            if collision:
                reward -= 10.0
            else:
                reward += 1.
        elif self.task == 'velocity_control':
            diff = self._get_velocity_diff(velocity_target)
            reward -= diff

        return reward

    def _check_collision(self, old_pos, new_pos):
        # TODO: update to consider the body size of the quadrotor
        min_max = lambda x, y, i: \
            (int(floor(min(x[i], y[i]))), int(ceil(max(x[i], y[i]))))
        x_min, x_max = min_max(old_pos, new_pos, 0)
        y_min, y_max = min_max(old_pos, new_pos, 1)
        z_min, z_max = min_max(old_pos, new_pos, 2)

        taken_pos = self.map_matrix[y_min:y_max+1, x_min:x_max+1]
        if z_min < np.any(taken_pos) or z_max < np.any(taken_pos):
            return True
        else:
            return False

    def _update_state(self, sensor, state):
        state['x'] = state['x'] + self.x_offset
        state['y'] = state['y'] + self.y_offset
        state['z'] = state['z'] + self.z_offset

        for k, v in sensor.items():
            self.state[k] = v

        for k, v in state.items():
            self.state[k] = v

        if self.task == 'velocity_control':
            t = min(self.ct, self.nt-1)
            next_velocity_target = self.velocity_targets[t]
            self.state['next_target_g_v_x'] = next_velocity_target[0]
            self.state['next_target_g_v_y'] = next_velocity_target[1]
            self.state['next_target_g_v_z'] = next_velocity_target[2]

    def _get_velocity_diff(self, velocity_target):
        vt_x, vt_y, vt_z = velocity_target
        diff = abs(vt_x - self.state['g_v_x']) + \
            abs(vt_y - self.state['g_v_y']) + \
            abs(vt_z - self.state['g_v_z'])
        return diff

    @staticmethod
    def load_map(map_file):
        if map_file is None:
            flatten_map = np.zeros([100, 100], dtype=np.int32)
            flatten_map[50, 50] = -1
            return flatten_map

        map_lists = []
        with open(map_file, 'r') as f:
            for line in f.readlines():
                map_lists.append([int(i) for i in line.split(' ')])

        return np.array(map_lists)

if __name__ == '__main__':
    import sys
    if len(sys.argv) == 1:
        task = 'no_collision'
    else:
        task = sys.argv[1]
    env = Quadrotor(task=task, nt=1000)
    env.reset()
    env.render()
    reset = False
    step = 1
    while not reset:
        action = np.array([2., 2., 1., 1.], dtype=np.float32)
        # action = np.array([1., 0., 0., 0.], dtype=np.float32)
        state, reward, reset = env.step(action)
        env.render()
        print('---------- step %s ----------' % step)
        print('state:', state)
        print('reward:', reward)
        step += 1
    env.close()

max_stars_repo_name: zhangjw-THU/RLSchool

max_issues_repo_name: zhangjw-THU/RLSchool

max_forks_repo_name: zhangjw-THU/RLSchool


text:
import functools

import numpy as np
import sympy as sp

from sympy.parsing.sympy_parser import parse_expr

from .quantity import Quantity, Dimension, DimensionError, dimensionify, quantify, make_quantity

def _parse_str_to_dic(exp_str):
    """
    Use sympy's parser to split a string "m/s**2"
    to a dict {"m":1, "s":-2}.
    """
    parsed = parse_expr(exp_str)
    exp_dic = {str(key):value for key,value in parsed.as_powers_dict().items()}
    return exp_dic

def _exp_dic_to_q(exp_dic, parsing_dict):
    """
    Helper to expand a dict to a quantity using :
        q = u1**v1 * u2**v2 ...
    where "u" are string of units, and "v" power values.
    """
    q = 1
    for key, value in exp_dic.items():
        # parse string to unit
        u = parsing_dict[key]
        # power up and multiply
        q *= u**value
    return q

def expr_to_q(exp_str, parsing_dict):
    """
    Parse a string expression to a quantity.
    """
    exp_dic = _parse_str_to_dic(exp_str)
    q = _exp_dic_to_q(exp_dic, parsing_dict)
    return q

def strunit_array_to_qunit_array(array_like_of_str, parsing_dict):
    """
    Converts an iterable of strings to an iterable of quantities

    Example :
    =========

    """
    qs = []
    for s in array_like_of_str:
        q = expr_to_q(s, parsing_dict)
        qs.append(q)
    return qs

def qarange(start_or_stop, stop=None, step=None, **kwargs):
    """Wrapper around np.arange"""
    # start_or_stop param
    final_start_or_stop = quantify(start_or_stop)
    in_dim = final_start_or_stop.dimension

    qwargs = dict()

    # stop param
    if stop is None:
        pass#final_stop = Quantity(1, in_dim)
    else:
        final_stop = quantify(stop)
        if not final_stop.dimension == final_start_or_stop.dimension:
            raise DimensionError(final_start_or_stop.dimension, final_stop.dimension)
        qwargs["stop"] = final_stop.value

    # step param
    if step is None:
        pass#final_step = Quantity(0.1, in_dim)
    else:
        final_step = quantify(step)
        if not final_step.dimension == final_start_or_stop.dimension:
            raise DimensionError(final_start_or_stop.dimension, final_step.dimension)
        qwargs["step"] = final_step.value

    # final call
    val = np.arange(final_start_or_stop.value, **qwargs, **kwargs)
    res = Quantity(val, in_dim)
    return res

def _iterify(x):
    """make x iterable"""
    return [x] if not isinstance(x, (list, tuple)) else x

def check_dimension(units_in=None, units_out=None):
    """Check dimensions of inputs and ouputs of function.

    Will check that all inputs and outputs have the same dimension
    than the passed units/quantities. Dimensions for inputs and
    outputs expects a tuple.

    Parameters
    ----------
    units_in : quantity_like or tuple of quantity_like
        quantity_like means an Quantity object or a
        numeric value (that will be treated as dimensionless Quantity).
        The inputs dimension will be checked with the units_in.
        Defaults to None to skip any check.
    units_out : quantity_like or tuple of quantity_like
        quantity_like means an Quantity object or a
        numeric value (that will be treated as dimensionless Quantity).
        The outputs dimension will be checked with the units_out.
        Default to None to skip any check.

    Returns
    -------
    func:
        decorated function with dimension-checked inputs and outputs.

    See Also
    --------
    Other decorators (TODO)

    Notes
    -----
    Notes about the implementation algorithm (if needed).

    This can have multiple paragraphs.

    You may include some math:

    .. math:: X(e^{j\omega } ) = x(n)e^{ - j\omega n}

    And even use a Greek symbol like :math:`\omega` inline.

    Examples (written in doctest format)
    --------
    >>> def add_meter(x): return x + 1*m
    >>> add_meter = check_dimension((m), (m))(add_meter)
    >>> add_meter(1*m)
    2 m
    >>> add_meter(1*s)
    raise DimensionError
    """

    # reading args and making them iterable
    if units_in:
        units_in = _iterify(units_in)
    if units_out:
        units_out = _iterify(units_out)

    # define the decorator
    def decorator(func):
        # create a decorated func
        @functools.wraps(func)
        def decorated_func(*args, **kwargs):
            # Checking dimension of inputs
            args = _iterify(args)
            if units_in:
                for arg, unit_in in zip(args, units_in):
                    # make everything dimensions
                    dim_check_in = dimensionify(unit_in)
                    dim_arg = dimensionify(arg)
                    # and checking dimensions
                    if not dim_arg == dim_check_in:
                        raise DimensionError(dim_arg, dim_check_in)

            # Compute outputs and iterify it
            ress = _iterify(func(*args, **kwargs))

            # Checking dimension of outputs
            if units_out:
                for res, unit_out in zip(ress, units_out):
                    # make everythin dimensions
                    dim_check_out = dimensionify(unit_out)
                    dim_res = dimensionify(res)
                    # and checking dimensions
                    if not dim_res == dim_check_out:
                        raise DimensionError(dim_res, dim_check_out)

            # still return funcntion outputs
            return tuple(ress) if len(ress) > 1 else ress[0]
        return decorated_func
    return decorator

def set_favunit(*favunits_out):
    """Sets favunit to outputs.

    Sets favunit for the function outputs.

    Parameters
    ----------
    favunits_out : quantity_like or tuple of quantity_like
        quantity_like means an Quantity object or a
        numeric value (that will be treated as dimensionless Quantity).
        `favunits_out` should be a list of Quantity object that have a defined symbol,
        hence making them suitables favunits.

    Returns
    -------
    func:
        decorated function with outputs with a favunit.

    See Also
    --------
    Other decorators (TODO)

    Examples (written in doctest format)
    --------
    >>> def add_meter(x): return x + 1*m
    >>> add_meter_favmm = set_favunit(mm)(add_meter)
    >>> add_meter(1*m)
    2000 mm
    """
    # make favunits iterable
    favunits_out = _iterify(favunits_out)
    # make decorator
    def decorator(func):
        # make decorated function
        @functools.wraps(func)
        def decorated_func(*args, **kwargs):
            # compute outputs and iterable it
            ress = _iterify(func(*args, **kwargs))
            # turn outputs to quantity with favunit
            ress_with_favunit = [make_quantity(res, favunit=favunit) for res, favunit in zip(ress, favunits_out)]
            return tuple(ress_with_favunit) if len(ress_with_favunit) > 1 else ress_with_favunit[0]
        return decorated_func
    return decorator

def dimension_and_favunit(inputs=[], outputs=[]):
    """Check dimensions of outputs and inputs, and add favunit to outputs.

    A wrapper of `check_dimension` and `set_favunit` decorators. The inputs will
    be dimension-checked, and the output will be both dimension-checked and have
    a favunit. The outputs favunit hence must have same dimension than the
    expected outputs.

    See Also
    --------
    set_favunit : Decorator to add favunit to outputs.
    check_dimension : Decorator to check dimension of inputs and outputs.
    """
    def decorator(func):
        return set_favunit(outputs)(check_dimension(inputs, outputs)(func))
    return decorator

def convert_to_unit(*unit_in, keep_dim=False):
    """Convert inputs to values in terms of specified units.

    Decorator to convert the function's inputs values in terms
    of specified units.

    Examples (written in doctest format)
    --------
    >>> @convert_to_unit(mm, mm)
        def add_one_mm(x_mm, y_mm):
            "Expects values as floats in mm"
            return x_mm + y_mm + 1
    >>> print(add_one_mm(1.2*m, 2*m))
    2201
    """
    unit_in = _iterify(unit_in)
    def decorator(func):
        @functools.wraps(func)
        def decorated(*args, **kwargs):
            arg_unitless = []
            for arg, unit in zip(args, unit_in):
                if not keep_dim:
                    arg_unitless.append(arg/unit)
                else:
                    arg_unitless.append(Quantity(arg/unit, unit.dimension))
            return func(*arg_unitless, **kwargs)
        return decorated
    return decorator

def drop_dimension(func):
    """Drop the inputs dimension and sends the SI-value.

    Decorator to drop inputs dimensions, quantity value is passed
    in terms of SI-value.

    Examples
    --------
    >>> @drop_dimension
        def sum_length_from_floats(x, y):
           "Expect dimensionless objects"
           return x + y
    >>> print(sum_length_from_floats(1.2*m, 2*m))
    2.2
    """
    @functools.wraps(func)
    def dimension_dropped(*args, **kwargs):
        args = _iterify(args)
        value_args = [quantify(arg).value for arg in args]
        return func(*value_args, **kwargs)
    return dimension_dropped

def add_back_unit_param(*unit_out):
    """Multiply outputs of function by the units_out

    @add_back_unit_param(m, s)
    def timed_sum(x_m, y_m):
        time_s = 10
        return x_m + y_m + 1, time_s
    print(timed_sum(1, 2))

    """
    unit_out = _iterify(unit_out)
    def decorator(func):
        @functools.wraps(func)
        def dimension_added_back_func(*args, **kwargs):
            ress = _iterify(func(*args, **kwargs))
            # multiply each output by the unit
            ress_q = [res * unit for res, unit in zip(ress, unit_out)]
            return tuple(ress_q) if len(ress_q) > 1 else ress_q[0]
        return dimension_added_back_func
    return decorator

def decorate_with_various_unit(inputs=[], ouputs=[]):
    """
    allow abitrary specification of dimension and unit:
    @decorate_with_various_unit(("A", "A"), "A")
    def func(x, y):
        return x+y

    It will do 2 things :
        - check that the inputs have coherent units vs each others
        - set the specified unit to the output

    TODO : get rid of eval with a expression parser"""
    inputs_str = _iterify(inputs)
    outputs_str = _iterify(ouputs)
    def decorator(func):
        @functools.wraps(func)
        def decorated(*args, **kwargs):
            dict_of_units = {}
            list_inputs_value = []
            # loop over function's inputs and decorator's inputs
            for arg, input_name in zip(args, inputs_str):
                if input_name == "pass":
                    pass
                #
                else:
                    # turn input into quantity
                    arg = quantify(arg)
                    si_unit = arg._SI_unitary_quantity
                    # store input value
                    list_inputs_value.append(arg.value)
                    # check if input name (=unit or expression) already exists
                    if input_name in dict_of_units and (not si_unit == dict_of_units[input_name]):
                        raise DimensionError((arg._SI_unitary_quantity).dimension,
                                             (dict_of_units[input_name]).dimension)
                    # if input_name is new, add it's unit to dict
                    else:
                        dict_of_units[input_name] = arg._SI_unitary_quantity
            # compute expression using decorator ouputs
            list_outputs_units = [eval(out_str, dict_of_units) for out_str in outputs_str]
            # compute function res on values
            res_brute = _iterify(func(*list_inputs_value, **kwargs))
            # turn back raw outputs into quantities
            res_q = [res * unit for res, unit in zip(res_brute, list_outputs_units)]
            return tuple(res_q) if len(res_q) > 1 else res_q[0]
        return decorated
    return decorator

def composed_decs(*decs):
    """A wrapper to combine multiple decorators"""
    def deco(f):
        for dec in reversed(decs):
            f = dec(f)
        return f
    return deco

def latex_parse_eq(eq):
    """Tests cases :
     - v = d/t
     - 2piRC
     - 1/(2piRC)
     - use of sqrt, quad
     - parse lmbda, nu, exp
    """
    #if "$" in eq:
    #    return eq
    #if "=" in eq:
    #    left, right = eq.split("=")
    #    res = "=".join([str(sp.latex(sp.sympify(left))), str(sp.latex(sp.sympify(right)))])
    #    return "$" + res + "$"
    #else:
    #    return "$" + str(sp.sympify(sp.latex(eq))) + "$"
    return eq

def latex_eq(eqn):
    """add a 'latex' attribute representation (a string most likely)
    to a function"""
    def wrapper(f):
        f.latex = latex_parse_eq(eqn)
        return f
    return wrapper

def name_eq(name):
    """add a 'name' attribute (a string most likely) to a function"""
    def wrapper(f):
        f.name = name
        return f
    return wrapper

def array_to_Q_array(x):
    raise ImplementedError
#    """Converts an array of Quantity to a Quantity with array value.
#
#    First aim to be used with the vectorize.
#
#    """
#    # aim to deal with np.ndarray
#    if type(x) == np.ndarray:
#        # if array is size 1
#        if x.size == 1:
#            return quantify(x.item(0))
#        # when size>1 and Quantity
#        elif isinstance(x[0], Quantity):
#            # extract values into an array
#            val_out = np.asarray([qu.value for qu in x])
#            # send in a new Quantity
#            return Quantity(val_out,
#                            x[0].dimension,
#                            favunit=x[0].favunit)
#        # otherwise create a dimless quantity
#        else:
#            return Quantity(x, Dimension(None))
#    else:
#        return quantify(x)

def list_of_Q_to_Q_array(Q_list):
    raise ImplementedError
#    """Convert list of Quantity's object to a Quantity with array value.
#    All Quantity must have the same dimension."""
#    first = quantify(Q_list[0])
#    dim = first.dimension
#    val_list = []
#    for q in Q_list:
#        q = quantify(q)
#        if q.dimension == dim:
#            val_list.append(q.value)
#        else:
#            raise ValueError
#    return Quantity(np.array(val_list), dim)

from collections.abc import Iterable

def _flatten(x):
    if isinstance(x, list):
        return [a for i in x for a in _flatten(i)]
    else:
        return [x]

def _is_nested(a):
    return any(isinstance(i, list) for i in a)

def _shape(lst):
    def ishape(lst):
        shapes = [ishape(x) if isinstance(x, list) else [] for x in lst]
        shape = shapes[0]
        if shapes.count(shape) != len(shapes):
            raise ValueError('Ragged list')
        shape.append(len(lst))
        return shape
    return tuple(reversed(ishape(lst)))

def _wrap(a):
    if _is_nested(a):
        shape = _shape(a)
        flatten = list(_flatten(a))
        return flatten, shape
    else:
        return a

def asqarray(array_like):
    """The value returned will always be a Quantity with array value"""

    # tuple or list
    if isinstance(array_like, list) or isinstance(array_like, tuple):
        # here should test if any is quantity, not the first one
        # also should test if list is flat or nested
        if any(isinstance(i, Quantity) for i in array_like):
            dim = quantify(array_like[0]).dimension
            val_list = []
            for q in array_like:
                q = quantify(q)
                if q.dimension == dim:
                    val_list.append(q.value)
                    res_val = np.array(val_list)
                else:
                    raise DimensionError(q.dimension, dim)
            return Quantity(res_val, dim)
        elif isinstance(array_like[0], list):
            flat_array, shape = _wrap(array_like)
            q = asqarray(flat_array)
            q.value = q.value.reshape(shape)
            return q
        # list/tuple of non-quantity value
        else:
            return quantify(array_like)
    # object is array
    elif isinstance(array_like, np.ndarray):
        # non mono-element
        if array_like.size > 1:
            # check all value for dim consistency
            if isinstance(array_like[0], Quantity):
                dim = array_like[0].dimension
                shape = array_like.shape
                val_list = []
                for q in array_like:
                    if q.dimension == dim:
                        val_list.append(q.value)
                        res_val = np.array(val_list)
                    else:
                        raise DimensionError(q.dim, dim)
                return Quantity(res_val.reshape(shape), dim)
            else:
                return quantify(array_like)
        # array is mono element
        else:
            # array has one d
            if len(array_like.shape)==1:
                return quantify(array_like[0])
            else:
                return quantify(array_like)
    # object is already a Quanity
    elif isinstance(array_like, Quantity):
        return array_like
    else:
        raise ValueError("Type {type(array_like)} not supported")

ext: py

max_stars_repo_name: mocquin/physipy

max_issues_repo_name: mocquin/physipy

max_forks_repo_name: mocquin/physipy

reason: import numpy,import sympy,from sympy


text:
import gym
import random
import numpy as np
import tflearn
from tflearn.layers.core import input_data, dropout, fully_connected
from tflearn.layers.estimator import regression
from statistics import mean, median
from collections import Counter

LR = 1e-3
env = gym.make("CartPole-v0")
env.reset()
goal_steps = 500
score_requirment = 50
initial_games = 10000

def some_random_games_first():
    for episode in range(5):
        env.reset()
        for t in range(200):
            env.render()
            action = env.action_space.sample()
            observation, reward, done, info = env.step(action)
            if done :
                break

#some_random_games_first()

def initial_population():
    training_data = []
    scores = []
    accepted_scores = []
    for _ in range(initial_games):
        score = 0
        game_memory =  []
        prev_observation = []
        for _ in range(goal_steps):
            action = random.randrange(0,2)
            observation, reward, done, info = env.step(action)

            if len(prev_observation) > 0:
                game_memory.append([prev_observation, action])
            prev_observation = observation
            score += reward
            if done:
                break
        if score >= score_requirment:
            accepted_scores.append(score)
            for data in game_memory:
                if data[1] ==  1:
                    output = [0 , 1]
                elif data[1] == 0 :
                    output = [1, 0]
                training_data.append([data[0] ,output])
        env.reset()
        scores.append(score)
    training_data_save =  np.array(training_data)
    np.save('saved.npy',training_data_save)
    print('Avg accepted score',mean(accepted_scores))
    print('Median accepted scores',median(accepted_scores))
    print(Counter(accepted_scores))

    return training_data

#initial_population()
def neural_network(input_size):
    network = input_data(shape = [None ,  input_size , 1] ,name='input' )

    network =  fully_connected(network , 128 , activation='relu')
    network = dropout(network , 0.8)

    network =  fully_connected(network , 256 , activation='relu')
    network = dropout(network , 0.8)

    network =  fully_connected(network , 512 , activation='relu')
    network = dropout(network , 0.8)
    #
    network =  fully_connected(network , 1024 , activation='relu')
    network = dropout(network , 0.8)

    network =  fully_connected(network , 512 , activation='relu')
    network = dropout(network , 0.8)

    network =  fully_connected(network , 256 , activation='relu')
    network = dropout(network , 0.8)

    network =  fully_connected(network , 128 , activation='relu')
    network = dropout(network , 0.8)

    network = fully_connected(network , 2 , activation ='softmax')
    network = regression(network,optimizer = 'adam' ,learning_rate = LR , loss=  'categorical_crossentropy',name='targets' )

    model = tflearn.DNN(network , tensorboard_dir ='log')

    return model

def train_model(training_data,model=False):
    X = np.array([i[0] for i in training_data] ).reshape(-1 , len(training_data[0][0] ) , 1 )
    y = [i[1] for i in training_data]

    if not model:
        model=  neural_network(input_size = len(X[0]) )

    model.fit({'input':X} , {'targets': y} , n_epoch = 3 , snapshot_step =500 ,  show_metric= True , run_id= 'openaistuff' )

    return model

training_data = initial_population()
model= train_model(training_data)

scores =[]
choices = []
for game in range(10):
    score = 0
    game_memory = []
    prev_obs = []
    env.reset()
    for _ in range(goal_steps):
        env.render()
        if len(prev_obs) == 0:
            action =  random.randrange(0,2)
        else:
            #print("obs matrix")
            #print(model.predict(prev_obs.reshape(-1 , len(prev_obs) , 1 )))
            #print("\n")
            action =  np.argmax(model.predict(prev_obs.reshape(-1 , len(prev_obs) , 1 ))[0] )
        choices.append(action)
        observation , reward, done , info = env.step(action)
        prev_obs = observation
        game_memory.append([observation ,  action])
        score+=reward

        if done:
            break
    scores.append(score)

print("Average score " ,  sum(scores) /  len(scores))
print("Choice 1:  {} , Choice 0 :  {} " .format(choices.count(1) /  len(choices) ,  choices.count(0) / len(choices) ) )

max_stars_repo_name: chetanchougle/DeepLcartpolegame

max_issues_repo_name: chetanchougle/DeepLcartpolegame

max_forks_repo_name: chetanchougle/DeepLcartpolegame

reason: import numpy


text:
import numpy as np
import tensorflow as tf
na = np.newaxis

class BatchNormalization:

    def __init__(self, layer, X):

        self.X = X
        self.W = layer.get_weights()[0]

    def _simple_lrp(self, R):
        '''
        LRP according to Eq(56) in DOI: 10.1371/journal.pone.0130140
        '''
        Rx = R * self.W

        return Rx


text:
import logging
import os
import shutil
import warnings

from modelforge.progress_bar import progress_bar
try:
    import tensorflow as tf
except ImportError:
    warnings.warn("Tensorflow is not installed, dependent functionality is unavailable.")
import numpy

from sourced.ml.algorithms.id_embedding import extract_coocc_matrix
from sourced.ml.models import Cooccurrences, DocumentFrequencies

def _int64s(xs):
    return tf.train.Feature(
        int64_list=tf.train.Int64List(value=list(xs)))

def _floats(xs):
    return tf.train.Feature(
        float_list=tf.train.FloatList(value=list(xs)))

def id2vec_preprocess(args):
    """
    Loads co-occurrence matrices for several repositories and generates the
    document frequencies and the Swivel protobuf dataset.

    :param args: :class:`argparse.Namespace` with "input", "vocabulary_size", \
                 "shard_size", "df" and "output".
    :return: None
    """
    log = logging.getLogger("preproc")
    log.info("Loading docfreq model from %s", args.docfreq_in)
    df_model = DocumentFrequencies(log_level=args.log_level).load(source=args.docfreq_in)
    coocc_model = Cooccurrences().load(args.input)
    if numpy.any(coocc_model.matrix.data < 0):
        raise ValueError(("Co-occurrence matrix %s contains negative elements. "
                          "Please check its correctness.") % args.input)
    if numpy.any(numpy.isnan(coocc_model.matrix.data)):
        raise ValueError(("Co-occurrence matrix %s contains nan elements. "
                          "Please check its correctness.") % args.input)

    try:
        df_meta = coocc_model.get_dep(DocumentFrequencies.NAME)
        if df_model.meta != df_meta:
            raise ValueError((
                "Document frequency model you provided does not match dependency inside "
                "Cooccurrences model:\nargs.docfreq.meta:\n%s\ncoocc_model.get_dep"
                "(\"docfreq\")\n%s\n") % (df_model.meta, df_meta))
    except KeyError:
        pass  # There is no docfreq dependency

    vs = args.vocabulary_size
    if len(df_model) < vs:
        vs = len(df_model)
    sz = args.shard_size
    if vs < sz:
        raise ValueError(
            "vocabulary_size=%s is less than shard_size=%s. You should specify a smaller "
            "shard_size (e.g. shard_size=%s)." % (vs, sz, vs))
    vs -= vs % sz
    log.info("Effective vocabulary size: %d", vs)
    df_model = df_model.greatest(vs)
    log.info("Sorting the vocabulary...")
    chosen_words = sorted(df_model.tokens())
    word_indices = {w: i for i, w in enumerate(chosen_words)}

    if not os.path.exists(args.output):
        os.makedirs(args.output)
    with open(os.path.join(args.output, "row_vocab.txt"), "w") as out:
        out.write("\n".join(chosen_words))
    log.info("Saved row_vocab.txt")
    shutil.copyfile(os.path.join(args.output, "row_vocab.txt"),
                    os.path.join(args.output, "col_vocab.txt"))
    log.info("Saved col_vocab.txt")
    del chosen_words

    ccmatrix = extract_coocc_matrix((vs, vs), word_indices, coocc_model)

    log.info("Planning the sharding...")
    bool_sums = ccmatrix.indptr[1:] - ccmatrix.indptr[:-1]
    reorder = numpy.argsort(-bool_sums)
    with open(os.path.join(args.output, "row_sums.txt"), "w") as out:
        out.write("\n".join(map(str, bool_sums.tolist())))
    log.info("Saved row_sums.txt")
    shutil.copyfile(os.path.join(args.output, "row_sums.txt"),
                    os.path.join(args.output, "col_sums.txt"))
    log.info("Saved col_sums.txt")

    log.info("Writing the shards...")
    os.makedirs(args.output, exist_ok=True)
    nshards = vs // args.shard_size
    for row in progress_bar(range(nshards), log, expected_size=nshards):
        for col in range(nshards):
            indices_row = reorder[row::nshards]
            indices_col = reorder[col::nshards]
            shard = ccmatrix[indices_row][:, indices_col].tocoo()

            example = tf.train.Example(features=tf.train.Features(feature={
                "global_row": _int64s(indices_row),
                "global_col": _int64s(indices_col),
                "sparse_local_row": _int64s(shard.row),
                "sparse_local_col": _int64s(shard.col),
                "sparse_value": _floats(shard.data)}))

            with open(os.path.join(args.output,
                                   "shard-%03d-%03d.pb" % (row, col)),
                      "wb") as out:
                out.write(example.SerializeToString())
    log.info("Success")

max_stars_repo_name: vmarkovtsev/ml

max_issues_repo_name: vmarkovtsev/ml

max_forks_repo_name: vmarkovtsev/ml


text:
import scipy as sci
import numpy as np
from DataCreator import *
from PODCreator import *
from dolfin import *
from tools import *
import csv

class rid(data_creator):
	def __init__(self,meshInput,lzInput,numberModeInputForRID,ssListInput):
		data_creator.__init__(self,meshInput,lzInput,ssListInput)
		#create dir and files to save data
		self.set_savedir()
		# create reduced basis to build the RID
		self.create_dataRID(meshInput,lzInput,numberModeInputForRID,ssListInput  )
		# create the list of coordinate to build the RID
		self.create_RID_coordinate(meshInput)
		self.create_RID_domain(meshInput,lzInput)
		self.create_RID_DOFs_list(meshInput)
		self.create_VPOD_over_RID(meshInput)
	# override load_data() of data_creator to load VPOD created by PODCreator
	def load_data(self):
		self.V_POD =np.load(self.cwd+"/"+self.podDirectory+'/V_POD.npy')
		self.S_POD=np.load(self.cwd+"/"+self.podDirectory+'/S_POD_out.npy')
		self.S1_POD=np.load(self.cwd+"/"+self.podDirectory+'/S1_POD_out.npy')
		self.S01_POD=np.load(self.cwd+"/"+self.podDirectory+'/S01_POD_out.npy')
		self.S10_POD=np.load(self.cwd+"/"+self.podDirectory+'/S10_POD_out.npy')
	def set_savedir(self):
		savedir = "RID"
		J0xml = File("J0domain.xml")
		self.file_subdomain = File(self.cwd+"/"+self.ridDirectory+"/subdomain.xml")
		self.RID = File(self.cwd+"/"+self.ridDirectory+"/RID.pvd")
		self.IZ_file = File(self.cwd+"/"+self.ridDirectory+"/IZ.pvd")
		self.file_submesh=File(self.cwd+"/"+self.ridDirectory+"/submesh.pvd")

	def create_dataRID(self,mesh,lz,numberModeInputForRID,ssListInput):
		dataRID = pod(mesh,lz,numberModeInputForRID,ssListInput)

		self.V_POD_RID =dataRID.V_POD
		self.V_POD_out_full_RID =dataRID.V_POD_iz_out_full
		self.S_POD_RID=dataRID.S_POD_iz_out_full
		self.S1_POD_RID=dataRID.S1_POD_iz_out_full
		self.S01_POD_RID=dataRID.S01_POD_iz_out_full
		self.S10_POD_RID=dataRID.S10_POD_iz_out_full
		print np.shape(self.V_POD_out_full_RID)
	def create_RID_coordinate(self,mesh1):
		dofinside0 = np.asarray(self.IZ_dofinside)
		print'deim V_POD'
		P_dof, self.F_dof_out = deim(self.V_POD_out_full_RID)
		#self.F_dof=np.append(dofinside0,self.F_dof_out)
		self.F_dof = self.F_dof_out

		print'deim S_POD'
		PS_dof, FS_dof = deim(self.S_POD_RID)

		print'deim S1_POD'
		PS1_dof, FS1_dof = deim(self.S1_POD_RID)

		print'deim S01_POD'
		PS01_dof, FS01_dof = deim(self.S01_POD_RID)

		print'deim S10_POD'
		PS10_dof, FS10_dof = deim(self.S10_POD_RID)

		FStress_dof = list(set(FS_dof) | set(FS1_dof) | set(FS01_dof) | set(FS10_dof))
		mesh1.init(self.tdim - 1, self.tdim)
		mesh1.init(self.tdim - 2, self.tdim)
		V = VectorFunctionSpace(mesh1, 'CG', 1)
		dofmap = V.dofmap()
		dofs = dofmap.dofs()
		# Get coordinates as len(dofs) x gdim array
		dofs_x = dofmap.tabulate_all_coordinates(mesh1).reshape((-1, self.gdim))
		self.rid_coor=[]
		for dof in self.F_dof:
			self.rid_coor.append(dofs_x[dof])

		for dof1 in FStress_dof:
			self.rid_coor.append(dofs_x[dof1*2])

		print "number of node used to build RID : ", np.shape(self.rid_coor)[0]

	def create_RID_domain(self,mesh1,lz):
		print "Loop over all cells to build RID ... "
		self.subdomains = CellFunction('size_t', mesh1, 0)
		for cell in cells(mesh1):
			for i in range(len(self.rid_coor)):
				if (cell.contains(Point(self.rid_coor[i]))):
					self.subdomains[cell] = 1
					if cell.midpoint()[0]>=0.9*lz:
						for vectex_cell in vertices(cell):
							for cell1 in cells(vectex_cell) :
								self.subdomains[cell1] = 1

		class IZO(SubDomain):
			def inside(self, x, on_boundary):
				return True  if (x[0]<1.05*lz) and (x[0]>=0.) else False

		izon = IZO()
		izon.mark(self.subdomains, 1)
		self.file_subdomain<<self.subdomains
		self.RID<<self.subdomains

	def create_RID_DOFs_list(self,mesh1):
		V1 = VectorFunctionSpace(mesh1, 'CG', 1)
		dx =  Measure('dx', domain=mesh1, subdomain_data=self.subdomains)
		du, du_t = TrialFunction(V1), TestFunction(V1)
		u = Function(V1)
		A_test= assemble(inner(du, du_t)*dx(0))
		A_test1= assemble(inner(du, du_t)*dx(1))

		Amatrx_sp = petsc_csr(A_test)
		Amatrx = Amatrx_sp.diagonal()
		Amatrx1_sp = petsc_csr(A_test1)
		Amatrx1 = Amatrx1_sp.diagonal()

		self.F_dofinside=[]
		self.F_dofoutside=[]
		for i in range (np.shape(Amatrx)[0]):
			if Amatrx[i] == 0 :
				self.F_dofinside.append(i)
		for i in range (np.shape(Amatrx1)[0]):
			if Amatrx1[i] != 0 :
				self.F_dofoutside.append(i)
		print len(self.F_dofinside)
		print len(self.F_dofoutside)

	def create_VPOD_over_RID(self,mesh1):
		self.Nrdof=np.shape(self.F_dofoutside)[0]
		self.Ndof = np.shape(self.V_POD)[0]
		self.VR0_POD = np.zeros(shape=(self.Nrdof,np.shape(self.V_POD)[1]))
		self.ZtZ0=np.zeros(shape=(self.Nrdof,self.Nrdof))
		for i in range(self.Nrdof):
			self.VR0_POD[i,:]=self.V_POD[self.F_dofoutside[i],:]
			if self.F_dofoutside[i] in self.F_dofinside:
				self.ZtZ0[i,i]=1

		self.submesh1 = SubMesh(mesh1, self.subdomains, 1)
		print 'node', self.submesh1.num_vertices()
		print 'cells', self.submesh1.num_cells()
		self.file_submesh<<self.submesh1

		self.mydomains = CellFunction('size_t', self.submesh1)
		self.mydomains.set_all(0)
		dx_subdomain = Measure('dx', domain=mesh1, subdomain_data=self.mydomains)
		Vt = VectorFunctionSpace(self.submesh1, "Lagrange", 1)
		V = VectorFunctionSpace(mesh1, 'CG', 1)
		gsub_dim = self.submesh1.geometry().dim()
		submesh1_dof_coordinates = Vt.dofmap().tabulate_all_coordinates(self.submesh1).reshape(-1, gsub_dim)
		mesh1_dof_coordinates = V.dofmap().tabulate_all_coordinates(mesh1).reshape(-1, gsub_dim)

		mesh1_dof_index_coordinates0={}
		for index,coor in enumerate(mesh1_dof_coordinates):
			mesh1_dof_index_coordinates0.setdefault(coor[0],[]).append(index)

		mesh1_dof_index_coordinates1={}
		for index,coor in enumerate(mesh1_dof_coordinates):
			mesh1_dof_index_coordinates1.setdefault(coor[1],[]).append(index)

		sub_to_glob_map = {}
		for bnd_dof_nr, bnd_dof_coords in enumerate(submesh1_dof_coordinates):
			corresponding_dofs = np.intersect1d(mesh1_dof_index_coordinates0[bnd_dof_coords[0]], mesh1_dof_index_coordinates1[bnd_dof_coords[1]])
			if corresponding_dofs[0] not in sub_to_glob_map.values():
				sub_to_glob_map[bnd_dof_nr] = corresponding_dofs[0]
			else:
				sub_to_glob_map[bnd_dof_nr] = corresponding_dofs[1]
		#print sub_to_glob_map
		glob_to_sub_map = dict((v,k) for k,v in sub_to_glob_map.items())
		#print glob_to_sub_map
		self.VR_POD=np.zeros(shape=(np.shape(self.VR0_POD)))
		self.ZtZ=np.zeros(shape=(self.Nrdof,self.Nrdof))
		for i in range(self.Nrdof):
			ai=glob_to_sub_map[self.F_dofoutside[i]]
			self.VR_POD[ai]=self.VR0_POD[i]
			self.ZtZ[ai,ai]=self.ZtZ0[i,i]

		np.save(self.cwd+"/"+self.rpodDirectory+'/ZtZ', self.ZtZ)
		np.save(self.cwd+"/"+self.rpodDirectory+'/VR_POD', self.VR_POD)
		np.save(self.cwd+"/"+self.rpodDirectory+'/F_inside', self.F_dofinside)
		np.save(self.cwd+"/"+self.rpodDirectory+'/F_outside', self.F_dofoutside)

		w = csv.writer(open("glob_to_sub_map.csv", "w"))
		for key, val in glob_to_sub_map.items():
			w.writerow([key, val])

reason: import numpy,import scipy


text:
from PIL import Image
import numpy as np
from libtiff import TIFF
img = Image.open('/media/allen/orange/mm_dataset/mydataset_Potsdam_600_normal/ndsm_dir/train/top_potsdam_02_10_RGB_000.png')
print(img.mode)
k = np.array(img)
h,w = k.shape
pad = k[:,-1]
pad = pad[:,np.newaxis];
c = np.concatenate((k,pad),axis=1)
tif = TIFF.open('/media/allen/orange/mm_dataset/potsdam_normal/dsm_test/top_potsdam_03_13_dsm_new.tif', mode='w')
tif.write_image(c, compression=None)
tif.close()
a = 1
# tif = TIFF.open('/home/allen/img1.tif', mode='w')
# tif.write_image(k, compression=None)
# tif.close()


text:
"""
# Copyright Xiang Wang, Inc. All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0

Author: Xiang Wang, xiangking1995@163.com
Status: Active
"""

import unicodedata
import transformers
import numpy as np

from typing import List
from copy import deepcopy
from ark_nlp.processor.tokenizer._tokenizer import BaseTokenizer

class TransfomerTokenizer(BaseTokenizer):
    """
    Transfomer文本编码器，用于对文本进行分词、ID化、填充等操作

    Args:
        vocab: transformers词典类对象、词典地址或词典名，用于实现文本分词和ID化
        max_seq_len (:obj:`int`): 预设的文本最大长度
    """  # noqa: ignore flake8"

    def __init__(
        self,
        vocab,
        max_seq_len
    ):
        if isinstance(vocab, str):
            # TODO: 改成由自定义的字典所决定
            vocab = transformers.AutoTokenizer.from_pretrained(vocab)

        self.vocab = vocab
        self.max_seq_len = max_seq_len
        self.additional_special_tokens = set()
        self.tokenizer_type = 'transfomer'

    @staticmethod
    def _is_control(ch):
        """控制类字符判断
        """
        return unicodedata.category(ch) in ('Cc', 'Cf')

    @staticmethod
    def _is_special(ch):
        """判断是不是有特殊含义的符号
        """
        return bool(ch) and (ch[0] == '[') and (ch[-1] == ']')

    @staticmethod
    def recover_bert_token(token):
        """获取token的“词干”（如果是##开头，则自动去掉##）
        """
        if token[:2] == '##':
            return token[2:]
        else:
            return token

    def get_token_mapping(self, text, tokens, is_mapping_index=True):
        """给出原始的text和tokenize后的tokens的映射关系"""
        raw_text = deepcopy(text)
        text = text.lower()

        normalized_text, char_mapping = '', []
        for i, ch in enumerate(text):
            ch = unicodedata.normalize('NFD', ch)
            ch = ''.join([c for c in ch if unicodedata.category(c) != 'Mn'])
            ch = ''.join([
                c for c in ch
                if not (ord(c) == 0 or ord(c) == 0xfffd or self._is_control(c))
            ])
            normalized_text += ch
            char_mapping.extend([i] * len(ch))

        text, token_mapping, offset = normalized_text, [], 0
        for token in tokens:
            token = token.lower()
            if token == '[unk]' or token in self.additional_special_tokens:
                if is_mapping_index:
                    token_mapping.append(char_mapping[offset:offset+1])
                else:
                    token_mapping.append(raw_text[offset:offset+1])
                offset = offset + 1
            elif self._is_special(token):
                # 如果是[CLS]或者是[SEP]之类的词，则没有对应的映射
                token_mapping.append([])
            else:
                token = self.recover_bert_token(token)
                start = text[offset:].index(token) + offset
                end = start + len(token)
                if is_mapping_index:
                    token_mapping.append(char_mapping[start:end])
                else:
                    token_mapping.append(raw_text[start:end])
                offset = end

        return token_mapping

    def sequence_to_ids(self, sequence_a, sequence_b=None):
        if sequence_b is None:
            return self.sentence_to_ids(sequence_a)
        else:
            return self.pair_to_ids(sequence_a, sequence_b)

    def sentence_to_ids(self, sequence, return_sequence_length=False):
        if type(sequence) == str:
            sequence = self.tokenize(sequence)

        if return_sequence_length:
            sequence_length = len(sequence)

        # 对超长序列进行截断
        if len(sequence) > self.max_seq_len - 2:
            sequence = sequence[0:(self.max_seq_len - 2)]
        # 分别在首尾拼接特殊符号
        sequence = ['[CLS]'] + sequence + ['[SEP]']
        segment_ids = [0] * len(sequence)
        # ID化
        sequence = self.vocab.convert_tokens_to_ids(sequence)

        # 根据max_seq_len与seq的长度产生填充序列
        padding = [0] * (self.max_seq_len - len(sequence))
        # 创建seq_mask
        sequence_mask = [1] * len(sequence) + padding
        # 创建seq_segment
        segment_ids = segment_ids + padding
        # 对seq拼接填充序列
        sequence += padding

        sequence = np.asarray(sequence, dtype='int64')
        sequence_mask = np.asarray(sequence_mask, dtype='int64')
        segment_ids = np.asarray(segment_ids, dtype='int64')

        if return_sequence_length:
            return (sequence, sequence_mask, segment_ids, sequence_length)

        return (sequence, sequence_mask, segment_ids)

    def pair_to_ids(self, sequence_a, sequence_b, return_sequence_length=False):
        if type(sequence_a) == str:
            sequence_a = self.tokenize(sequence_a)

        if type(sequence_b) == str:
            sequence_b = self.tokenize(sequence_b)

        if return_sequence_length:
            sequence_length = (len(sequence_a), len(sequence_b))

        # 对超长序列进行截断
        if len(sequence_a) > ((self.max_seq_len - 3)//2):
            sequence_a = sequence_a[0:(self.max_seq_len - 3)//2]
        if len(sequence_b) > ((self.max_seq_len - 3)//2):
            sequence_b = sequence_b[0:(self.max_seq_len - 3)//2]

        # 分别在首尾拼接特殊符号
        sequence = ['[CLS]'] + sequence_a + ['[SEP]'] + sequence_b + ['[SEP]']
        segment_ids = [0] * (len(sequence_a) + 2) + [1] * (len(sequence_b) + 1)

        # ID化
        sequence = self.vocab.convert_tokens_to_ids(sequence)

        # 根据max_seq_len与seq的长度产生填充序列
        padding = [0] * (self.max_seq_len - len(sequence))
        # 创建seq_mask
        sequence_mask = [1] * len(sequence) + padding
        # 创建seq_segment
        segment_ids = segment_ids + padding
        # 对seq拼接填充序列
        sequence += padding

        sequence = np.asarray(sequence, dtype='int64')
        sequence_mask = np.asarray(sequence_mask, dtype='int64')
        segment_ids = np.asarray(segment_ids, dtype='int64')

        if return_sequence_length:
            return (sequence, sequence_mask, segment_ids, sequence_length)

        return (sequence, sequence_mask, segment_ids)

class SentenceTokenizer(TransfomerTokenizer):
    """
    Transfomer文本编码器，用于单句子进行分词、ID化、填充等操作

    Args:
        vocab: transformers词典类对象、词典地址或词典名，用于实现文本分词和ID化
        max_seq_len (:obj:`int`): 预设的文本最大长度
    """  # noqa: ignore flake8"

    def sequence_to_ids(self, sequence, **kwargs):
        return self.sentence_to_ids(sequence, **kwargs)

class PairTokenizer(TransfomerTokenizer):
    """
    Transfomer文本编码器，用于句子对拼接进行分词、ID化、填充等操作

    Args:
        vocab: transformers词典类对象、词典地址或词典名，用于实现文本分词和ID化
        max_seq_len (:obj:`int`): 预设的文本最大长度
    """  # noqa: ignore flake8"

    def sequence_to_ids(self, sequence_a, sequence_b, **kwargs):
        return self.pair_to_ids(sequence_a, sequence_b, **kwargs)

class TokenTokenizer(TransfomerTokenizer):
    """
    Transfomer文本编码器，用于按字符进行分词、ID化、填充等操作

    Args:
        vocab: transformers词典类对象、词典地址或词典名，用于实现文本分词和ID化
        max_seq_len (:obj:`int`): 预设的文本最大长度
    """  # noqa: ignore flake8"

    def tokenize(self, text, **kwargs):
        tokens = []
        text = ' '.join([token_ for token_ in text])
        tokens = self.vocab.tokenize(text)
        return tokens

    def sequence_to_ids(self, sequence, **kwargs):
        return self.sentence_to_ids(sequence, **kwargs)

class SpanTokenizer(TransfomerTokenizer):
    """
    Transfomer文本编码器，用于对文本（基于分隔符分割维度）进行分词、ID化、填充等操作

    Args:
        vocab: transformers词典类对象、词典地址或词典名，用于实现文本分词和ID化
        max_seq_len (:obj:`int`):
            预设的文本最大长度
        split_token (:obj:`string`):
            分隔符
        additional_special_token (:obj:`list`):
            额外添加的特殊字符
    """  # noqa: ignore flake8"

    def __init__(
        self,
        vocab,
        max_seq_len,
        split_token=' ',
        additional_special_token='[blank]'
    ):
        super(SpanTokenizer, self).__init__(vocab, max_seq_len)
        self.split_token = split_token
        self.additional_special_token = self.split_token
        if self.additional_special_token:
            self.additional_special_token = additional_special_token
            self.vocab.add_special_tokens({'additional_special_tokens': [self.additional_special_token]})
            self.additional_special_tokens = set([self.additional_special_token])

    def tokenize(self, text, **kwargs) -> List[str]:
        tokens = []
        for span_ in text.split(self.split_token):
            tokens += self.vocab.tokenize(span_)
            tokens += [self.additional_special_token]
        return tokens[:-1]

    def sequence_to_ids(self, sequence, **kwargs):
        return self.sentence_to_ids(sequence, **kwargs)

max_stars_repo_name: Zrealshadow/ark-nlp

max_issues_repo_name: Zrealshadow/ark-nlp

max_forks_repo_name: Zrealshadow/ark-nlp


text:
import analytics_api as ga
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
from google.cloud import bigquery
import json
import requests
from functools import cache
# from IPython.core.display import HTML
import re

def authenticate_ga():
	ga.authenticate();

# def display_link(url, text):
# 	display(HTML('{}'.format(url, text)))

@cache
def get_project_name(id):
	info = requests.get("https://service.azul.data.humancellatlas.org/index/projects/" + id).json()
	if "projects" in info:
		return info["projects"][0]["projectTitle"]
	else:
		return id

# def display_project_link(id):
# 	display_link("https://data.humancellatlas.org/explore/projects/" + id, get_project_name(id))

def percent_change(valfrom, valto):
	return (valto - valfrom)/valfrom * 100

def format_export_url_info(type, secondary_type, filter):
	result = type.replace("-", " ")
	if secondary_type:
		result += ", " + secondary_type.replace("-", " ")
	for facet in json.loads(filter):
		if facet["facetName"] == "projectId":
			result += "\r\nProject: " + ", ".join([get_project_name(id) for id in facet["terms"]])
		else:
			result += "\r\n" + facet["facetName"] + ": " + ", ".join(facet["terms"])
	return result

def adjust_table_index_key(val):
	if isinstance(val, str):
		match = re.search("\\/explore\\/(?:projects\\/([^\\/#?]+)|export\\/(export-to-terra|get-curl-command|download-manifest)(?:\\/(select-species))?\\?filter=(.+))", val)
		if match:
			if match.group(1):
				return get_project_name(match.group(1)) + "\r\n" + val
			else:
				return format_export_url_info(match.group(2), match.group(3), match.group(4))
	return val

def format_pc_change_table(df, include_plus=False, cell_classes=None, hide_index=False, hide_columns=False):
	# Expects pairs of columns in a 2D multi-index where the second column is named "% Change"

	change_cols = [name for name in df.columns if name[1] == '% Change']

	change_format = '({:+.2f}%)' if include_plus else '({:.2f}%)'

	s = df.style
	if hide_index:
		s = s.hide(axis="index")
	else:
		s = s.format_index(adjust_table_index_key)
	if hide_columns:
		s = s.hide(axis="columns")
	s = s.format(na_rep='', formatter=lambda v: '' if v == np.inf else change_format.format(v), subset=change_cols)
	s = s.applymap(lambda v: 'color: red' if v < 0 else 'color: green' if v > 0 else None, subset=change_cols)
	if not cell_classes is None:
		s = s.set_td_classes(cell_classes)
	s = s.set_table_styles([
		{'selector': '', 'props': 'width: 100%; table-layout: auto; display: table'},
		{'selector': 'th.col_heading', 'props': 'text-align: center'},
		{'selector': 'thead > tr:nth-child(2)', 'props': 'display: none'},
		{'selector': 'th.index_name', 'props': 'text-align: left'},
		{'selector': 'th.row_heading', 'props': 'text-align: left; white-space: pre-wrap; line-break: anywhere; font-weight: normal'},
		{'selector': ', '.join(["td.col%i" % (i * 2 + 1) for i in range(len(change_cols))]), 'props': 'text-align: left; padding-left: 0'},
		{'selector': '.up::before', 'props': 'content: "↑\\00a0"; color: gray'},
		{'selector': '.down::before', 'props': 'content: "↓\\00a0"; color: gray'},
		{'selector': '.new::before', 'props': 'content: "+\\00a0"; color: gray'}
	], overwrite=False)

	return s

def format_change_over_time_table(df):
	df2 = df.copy(deep=True)

	data_cols = [c for c in df2.columns]

	df2['Quarter'] = df2.index.quarter
	df2['Year'] = df2.index.year

	df2 = df2.groupby(['Year','Quarter']).sum()

	df2.columns = pd.MultiIndex.from_tuples([(name, 'Value') for name in data_cols])

	for name in data_cols:
		df2[(name, '% Change')] = df2[name].pct_change().mul(100)

	df2 = df2[[(a, b) for a in data_cols for b in ['Value', '% Change']]]

	return format_pc_change_table(df2, include_plus=True)

def format_table_with_change(df, df_prev, hide_index=False, show_direction=True, hide_columns=False):
	# The data frames must have the same column names but may have some different rows

	df_joined = df.join(df_prev, rsuffix="_prev")
	df_change = pd.concat([v for name in df.columns for v in (df_joined[name], percent_change(df_joined[name + "_prev"], df_joined[name]))], axis=1)
	df_change.columns = pd.MultiIndex.from_tuples([(df_change.columns[i - i%2], "Value" if i%2 == 0 else "% Change") for (i, name) in enumerate(df_change.columns)])

	indices = pd.DataFrame(index=df.index)
	indices["index"] = range(indices.shape[0])
	indices_prev = pd.DataFrame(index=df_prev.index)
	indices_prev["index"] = range(indices_prev.shape[0])
	indices_combined = indices.join(indices_prev, rsuffix="_prev")

	indices_diff = indices_combined["index"] - indices_combined["index_prev"]
	classes_series = indices_diff.map(lambda v: "new" if pd.isna(v) else None if not show_direction or v == 0 else "up" if v < 0 else "down") # lower index = higher in chart

	classes = pd.DataFrame(index=df_change.index, columns=df_change.columns)
	classes[:] = None
	classes.iloc[:, [0]] = classes_series

	return format_pc_change_table(df_change, include_plus=True, cell_classes=classes, hide_index=hide_index, hide_columns=hide_columns)

def plot_users_over_time(ga_property, start_date, end_date):

	metrics = 'ga:1dayUsers, ga:uniquePageviews'
	dimensions = 'ga:date'
	df = ga.get_metrics_by_dimensions(ga_property, metrics,dimensions, start_date, end_date)

	# Convert date to datetime object
	df["ga:date"] = pd.to_datetime(df['ga:date'])
	df.set_index('ga:date', inplace=True)

	# Convert strings returned by API to integers. Can we do this earlier!
	# If not numeric data the series won't graph
	df["ga:1dayUsers"] = df['ga:1dayUsers'].astype(str).astype(int)
	df["ga:uniquePageviews"] = df['ga:uniquePageviews'].astype(str).astype(int)

	# Rename for display
	df.rename(columns={'ga:1dayUsers':'Total Daily Users', 'ga:uniquePageviews':'Total Unique Pageviews'}, inplace=True)

	#Smooth (coiuld we not just use 7 day users then?)
	# df = df.rolling(window=1).mean()

	# Notes: Linking Mandas and Matplotlib
	# https://stackoverflow.com/questions/29568110/how-to-use-ax-with-pandas-and-matplotlib

	fontsize=16
	fig, ax = plt.subplots(figsize=(16, 9))
	df.plot(ax=ax) #Link the df with the axis

	ax.set_xlabel('Time', fontsize=fontsize)
	ax.set_ylabel('Count', fontsize=fontsize)

	for label in (ax.get_xticklabels() + ax.get_yticklabels()):
		label.set_fontsize(fontsize)

	plt.rcParams['font.size'] = fontsize
	fig.suptitle('Daily Activity Overview')
	plt.show()

	# Average per qurter

	dfmean = df.rolling(window=30).mean()
	fig, ax = plt.subplots(figsize=(16, 9))
	dfmean.plot(ax=ax) #Link the df with the axis

	ax.set_xlabel('Time', fontsize=fontsize)
	ax.set_ylabel('Count', fontsize=fontsize)

	for label in (ax.get_xticklabels() + ax.get_yticklabels()):
		label.set_fontsize(fontsize)

	plt.rcParams['font.size'] = fontsize
	fig.suptitle('Daily Activity Overview 30 Day average')
	plt.show()

	return format_change_over_time_table(df)

def get_top_ga_df(ga_property, metrics, dimensions, start_date, end_date, ascending=True, limit=20, filters=None, num_keep_dimensions=1):
	df = ga.get_metrics_by_dimensions(ga_property, ",".join(metrics), dimensions and ",".join(dimensions), start_date, end_date, filters=filters)

	if dimensions:
		if len(dimensions) > 1:
			df.drop(columns=dimensions[num_keep_dimensions:], inplace=True);
		df.set_index(dimensions[:num_keep_dimensions], inplace=True)
	for metric in metrics:
		df[metric] = df[metric].astype(str).astype(int)
	if ascending != None:
		df = df.sort_values(by=metrics, ascending=ascending)

	if not limit is None:
		df = df.tail(limit) if ascending else df.head(limit)

	return df

def plot_hbar(ga_property, title, xlabel, ylabel, metric, dimension, start_date, end_date):

	df = get_top_ga_df(ga_property, [metric], dimension and [dimension], start_date, end_date)

	fontsize=16
	# position = list(range(1, 20))

	fig, ax = plt.subplots(figsize=(16, 9))
	plt.barh(df.index, df[metric]) #Link the df with the axis

	fig.suptitle(title)
	ax.set_ylabel(ylabel)
	ax.set_xlabel(xlabel)

	for index, value in enumerate(df[metric]):
		plt.text(value, index,
			' ('+ str(value)+')')

	plt.rcParams['font.size'] = fontsize
	plt.show()

def show_difference_table(ga_property, xlabels, ylabels, metrics, dimensions, period, prev_period, filters=None, ordered=True):
	if isinstance(xlabels, str):
		xlabels = [xlabels]
	if isinstance(metrics, str):
		metrics = [metrics]
	if isinstance(dimensions, str):
		dimensions = [dimensions]

	period = pd.Period(period)
	prev_period = pd.Period(prev_period)

	shared_params = {
		"ga_property": ga_property,
		"metrics": metrics,
		"dimensions": dimensions,
		"filters": filters,
		"ascending": False if ordered else None,
		"num_keep_dimensions": len(ylabels) if isinstance(ylabels, list) else 1
	}
	df = get_top_ga_df(start_date=period.start_time.isoformat()[:10], end_date=period.end_time.isoformat()[:10], **shared_params)
	df_prev = get_top_ga_df(start_date=prev_period.start_time.isoformat()[:10], end_date=prev_period.end_time.isoformat()[:10], **shared_params)

	is_single_cell = df.shape[0] == 1 and df.shape[1] == 1 and df_prev.shape[0] == 1 and df_prev.shape[1] == 1

	if is_single_cell and not dimensions:
		df.index = pd.Index(xlabels)
		df_prev.index = pd.Index(xlabels)
	else:
		xlabels_dict = {metric: xlabel for metric, xlabel in zip(metrics, xlabels)}
		df.rename(columns=xlabels_dict, inplace=True)
		df_prev.rename(columns=xlabels_dict, inplace=True)
		if dimensions:
			df.index.rename(ylabels, inplace=True)
			df_prev.index.rename(ylabels, inplace=True)

	display(format_table_with_change(df, df_prev, show_direction=ordered, hide_index=not dimensions and not is_single_cell, hide_columns=is_single_cell))

def plot_downloads():
	QUERY = """
		SELECT
			day,
			COUNT(*) AS totalSessionCount,
			COUNTIF("fastq" IN UNNEST(fileTypes)) AS fastqSessionCount,
			COUNTIF("bam" IN UNNEST(fileTypes)) AS bamSessionCount,
			COUNTIF("matrix" IN UNNEST(fileTypes)) AS matrixSessionCount,
			COUNTIF("other" IN UNNEST(fileTypes)) AS otherSessionCount,
			COUNTIF(".loom" IN UNNEST(matrixTypes)) AS loomMatrixSessionCount,
			COUNTIF(ARRAY_LENGTH(matrixTypes) > 1 OR (ARRAY_LENGTH(matrixTypes) = 1 AND matrixTypes[OFFSET(0)] != ".loom")) AS otherMatrixSessionCount
		FROM (
			SELECT
				DATE_TRUNC(timestamp, DAY) AS day,
				protopayload_auditlog.requestMetadata.callerIp,
				ARRAY_AGG(
					DISTINCT
					IF(
						REGEXP_CONTAINS(protopayload_auditlog.resourceName, "(?i)\\\\.fastq(?:\\\\.|$)"),
						"fastq",
						IF(
							REGEXP_CONTAINS(protopayload_auditlog.resourceName, "(?i)\\\\.bam(?:\\\\.|$)"),
							"bam",
							IF(
								REGEXP_CONTAINS(protopayload_auditlog.resourceName, "(?i)\\\\.(?:h5ad|loom|mtx)(?:\\\\.|$)"),
								"matrix",
								"other"
							)
						)
					)
				) AS fileTypes,
				ARRAY_AGG(
					DISTINCT
					REGEXP_EXTRACT(protopayload_auditlog.resourceName, "(?i)\\\\.(?:(?:h5ad|mtx)(?:\\\\.|$)|loom$|loom\\\\.)")
					IGNORE NULLS
				) AS matrixTypes
			FROM `broad-datarepo-terra-prod-hca2.accesslogs.cloudaudit_googleapis_com_data_access_*`
			GROUP BY day, callerIp
		)
		GROUP BY day ORDER BY day
	"""

	client = bigquery.Client()

	df = client.query(QUERY).result().to_dataframe(create_bqstorage_client=False)
	df.set_index("day", inplace=True)

	def plot_total():
		plt.rc("font", size=16)

		ax_total = df["totalSessionCount"].plot(title='Number of sessions per day downloading files', legend=False, xlabel='', figsize=(16, 9))
		plt.show()

	def plot_a():
		plt.rc("font", size=16)

		df_a = df[["fastqSessionCount", "bamSessionCount", "matrixSessionCount", "otherSessionCount"]].rolling("7d").mean()

		axs_a = df_a.plot(subplots=True, ylim=(0, df_a.max().max() + 5), title="Number of sessions per day downloading given file types", xlabel="", figsize=(16, 9))
		axs_a[0].legend(["FASTQ"])
		axs_a[1].legend(["BAM"])
		axs_a[2].legend(["Matrix"])
		axs_a[3].legend(["Other"])
		plt.tight_layout()
		plt.show()

	def plot_b():
		plt.rc("font", size=16)

		df_b = pd.concat([
			(df["matrixSessionCount"] - df["loomMatrixSessionCount"]).rename("CGMs only"),
			(df["loomMatrixSessionCount"] + df["otherMatrixSessionCount"] - df["matrixSessionCount"]).rename("CGMs and DCP matrices"),
			(df["matrixSessionCount"] - df["otherMatrixSessionCount"]).rename("DCP matrices only")
		], axis=1).rolling("7d").mean()
		ax_b = df_b.plot(kind="area", stacked=True, title="Number of sessions per day downloading matrices", xlabel="", figsize=(16, 9))
		plt.tight_layout()
		plt.show()

	return plot_total, plot_a, plot_b

def plot_ethnicity():
	catalog = "dcp15"

	def get_sources_expr():
		sources = requests.get("https://service.azul.data.humancellatlas.org/repository/sources?catalog=" + catalog).json()["sources"]
		return "(" + " UNION ALL ".join(["SELECT * FROM `" + get_table_name(source["sourceSpec"]) + "`" for source in sources]) + ")"

	def get_table_name(spec):
		parts = [p.split(":") for p in spec.split("/")]
		return parts[0][1] + "." + parts[1][0] + ".donor_organism"

	QUERY = """
		select
			IFNULL(ethnicity, "[Missing]") as Ethnicity,
			Code,
			count(*) as Count
		from (
			select
				datarepo_row_id,
				json_extract_scalar(gs, "$.ontology") as species,
				json_extract_scalar(eth, "$.ontology_label") as ethnicity,
				json_extract_scalar(eth, "$.ontology") as Code,
				content,
				json_extract_scalar(content, "$.provenance.document_id") AS did
			from
				%s
				LEFT JOIN UNNEST(JSON_EXTRACT_ARRAY(content, "$.genus_species")) as gs
				LEFT JOIN UNNEST(JSON_EXTRACT_ARRAY(content, "$.human_specific.ethnicity")) AS eth
		) as T
		where species = "NCBITaxon:9606"
		GROUP BY T.ethnicity, T.Code
		ORDER BY Count DESC
		""" % get_sources_expr()

	client = bigquery.Client()

	df = client.query(QUERY).result().to_dataframe(create_bqstorage_client=False)
	df.set_index("Ethnicity", inplace=True)

	series_a = df["Count"].groupby(lambda eth: "Unspecified" if eth == "[Missing]" else "Specified").sum()
	series_b = df.loc[df.index != "[Missing]"]["Count"]

	series_a.sort_values(inplace=True, ascending=False)

	def plot_a():
		plt.rc("font", size=16)

		ax_a = series_a.plot(kind="barh", title="Donor ethnicity (" + catalog.upper() + ") - Specified vs. Unspecified", xlabel="", figsize=(16, 9))
		ax_a.set_xlabel("Donors")
		plt.tight_layout()
		plt.show()

	def plot_b():
		plt.rc("font", size=16)

		ax_b = series_b.plot(kind="barh", title="Donor ethnicity (" + catalog.upper() + ")", xlabel="", figsize=(16, 9))
		ax_b.set_xlabel("Donors")
		plt.tight_layout()
		plt.show()

	return plot_a, plot_b

max_stars_repo_name: DataBiosphere/data-browser

max_issues_repo_name: DataBiosphere/data-browser

max_forks_repo_name: DataBiosphere/data-browser


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Geoscience I/O functions.

Single file module for now.

:copyright: 2020 Agile Geoscience
:license: Apache 2.0
"""
from io import StringIO
import os
import gzip
import re

import numpy as np
import pandas as pd

def read_opendtect_points(filename):
    """
    Read an OpendTect points file. Returns array with columns UTMx, UTMy, TWT.

    TODO:
        Do something with metadata in file.
    """
    data = np.loadtxt(filename, skiprows=6, comments='!', usecols=[0,1,2])
    return data

def read_petrel_points(filename):
    """
    Read a Petrel points file. Return an array.

    TODO:
        Do something with Comments and Fields.
    """
    with open(filename) as f:

        comments = []
        fields = []
        in_header = False

        while True:
            line = f.readline().strip()

            if line.startswith('#'):
                comments.append(line.strip('# '))
            elif line.startswith('VERSION'):
                # version = line.split()[-1]
                pass
            elif line.startswith('BEGIN'):
                in_header = True
            elif line.startswith('END'):
                in_header = False
                break
            elif in_header:
                fields.append(line.strip())
            else:
                break

        d = f.read()
        s = StringIO(d)

        return np.loadtxt(s)

def read_odt_seismic(filename, start_time=None, sample_interval=None):
    """
    Read an ascii seismic volume from OdT.  Assumes we have inline and
    xline numbers, not x, y.

    Returns a tuple of timebase, arrays representing inline and xline numbers,
    and an array of data.

    TODO:
        Is it better to return an array of inline, xline coordinates?
        Is that np.meshgrid...?
    """
    # Sniff file compression.
    file_extension = os.path.splitext(filename)[1]

    if file_extension[-2:] == 'gz':
        with gzip.open(filename, 'rb') as f:
            raw = f.readlines()
    else:
        with open(filename, 'r') as f:
            raw = f.readlines()

    # Sniff for a header.
    if len(raw[0].split()) == 3:
        temp = [float(x) for x in raw.pop(0).split()]
        start_time, sample_interval, number_samples = temp

    # Gather the data.
    inlines, xlines, traces = [], [], []

    for line in raw:
        l = line.split()
        inlines.append(int(l.pop(0)))
        xlines.append(int(l.pop(0)))
        trace = np.loadtxt(l)
        traces.append(trace)

    # Prepare everything for return.
    number_inlines = max(inlines) - min(inlines) + 1
    number_xlines = max(xlines) - min(xlines) + 1
    number_samples = len(traces[0])  # Nevermind what the header said

    if start_time and sample_interval:
        end_time = start_time + (number_samples-1)*sample_interval
        time_basis = np.linspace(start_time, end_time, number_samples)
    else:
        time_basis = None

    data = np.array(traces)
    data = np.reshape(data, (number_inlines, number_xlines, number_samples))

    return time_basis, np.array(inlines), np.array(xlines), data

def read_spwla(fname, null=-999.25):
    """
    Read SPWLA SPT core analysis / mudlog format.
    This has only been tested on one file! Use at your own risk.
    Reads fields 30, 36 and 40, using column names from field 15.

    Args
        fname (str): An SPWLA / SPT file.
        null (float): The number representing nulls; will be cast to NaNs.

    Returns
        pandas.DataFrame
    """
    # Not using this one.
    # rx_fields = re.compile(r'''
    #     ^
    #     15\s+?\d+?\s+?\d+?\n
    #     (?P<fields>[\s\S]+?)
    #     (?=^[^1]|\Z)
    # ''', re.MULTILINE | re.VERBOSE)

    rx_fields = re.compile(r'''
        ^
        \s+?\d+?\s+?\d+?\s+?\d+?\s+?0\S+?\s+?(?P<field>[\w\d][- ,.\w\d]+?)\n
    ''', re.MULTILINE | re.VERBOSE)

    rx_depth = re.compile(r'''
        ^
        30\s+?1\n
        \s+?(?P<depth>[.\d]+?)[ \t]+?[.\d]+?[ \t]+?(?P<seq>[.\d]+?)\n
        (?P<record>[\s\S]+?)
        (?=^30|\Z)
    ''', re.MULTILINE | re.VERBOSE)

    rx_data = re.compile(r'''
        ^
        (?:36\s+?1\s+?1\n
        \s+?(?P<descr>.+?)\n)?
        40\s+?1\s+?\d+?\n
        \s+?(?P<data>[- .\d]+?)\n
    ''', re.MULTILINE | re.VERBOSE)

    records = (field.group('field') for field in rx_fields.finditer(s))

    result = (
              (
                float(record.group('depth')),
                record.group('seq'),
                data.group('descr'),
                *[float(x) for x in data.group('data').split()]
              )
               for record in rx_depth.finditer(s)
               for data in rx_data.finditer(record.group('record'))
    )

    columns = ['depth', 'seq', 'descr'] + list(records)

    df = pd.DataFrame(result, columns=columns)
    df = df.replace(null, np.nan)

    return df


text:
#!/usr/bin/env python3
# coding: utf-8

#=====
# Subscript e means Emitter
# Subscript g means Gate (Extraction electrode)
# Subscript c means Collector (Faraday cup)
#
# Rs is resistivity of Shunt resistor using in the colloector (Faraday Cup).

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib import gridspec
import datetime, time
import h5py

# %matplotlib inline
Rs = 100e3 #ohm
Rprotect = 10e+6 #ohm

### Modified 161107
def Ve_correct(Ve, Ig, Ic, Rprotect):
    """Because we use a resistor connected in a serries on the emitter circuit
    in order to stabilize emission current, we need to correct voltage dropping
    at the resistor, which is called as "Protection Resistor" or somthing like that.

    Inputs -- Ve, Ig, Ic, Rprotect
    Ve: Emiter voltage
    Ig: Gate current
    Ic: Collector current
    Rprotect: Resistivity in ohm

    Output -- Vext
    Vext: Extraction voltage that is actually applied between the emitter and the ground

    """
    Vext = Ve - (np.abs(Ig + Ic))*Rprotect
    return Vext

def mydate(str_date):
    """Convert from datetime str with original format into seconds
    """
    fmt_date = datetime.datetime.strptime(str_date, "%y%m%d-%H:%M:%S")
    sec = time.mktime(fmt_date.timetuple())
    return sec

def get_data_old(datafile):
    # For emitter no.6 and befor
    data = pd.read_csv(datafile, delimiter='\t', comment='#',names=['date','Ve','Ig','Ic', 'P', 'IVno'],           dtype={'Ve':'float64','Ig':'float64','Ic':'float64','P':'float64'})

    ### convert date to laspe time in sec
    tmpdate = data['date'].apply(lambda x: x.split('.')[0])
    t0 = mydate(tmpdate[0])
    lapse_time = tmpdate.apply(lambda x: mydate(x)-t0 )
    data['time'] = lapse_time
    cols = data.columns.tolist()
    cols = cols[0:1]+cols[-1:]+cols[1:-1]
    data = data[cols]
    return data

def date2sec(date):
    """Convert from datetime str to lapse time and delta t for my pandas dataframe
    Input  -- date in pandas dataframe or series
    Output -- Pandas dataframe with columns of 'lapse', 'dt'
    lapse: lapse time in sec
    dt: defferential time in sec
    """
    t0 = date.apply(lambda x: x.split('.')[0])[0] # 0th element in the 'date' pandas series
    tmp_sec = date.apply(lambda x: x.split('.')[0]) # into sec converted only from integer
    tmp_ms  = date.apply(lambda x: "0." + x.split('.')[1]) # into sec the rest after "."
    lapse_time = tmp_sec.apply(lambda x: mydate(x)) + tmp_ms.apply(lambda x: float(x)) - mydate(t0)
    dt      = lapse_time.diff()
    time = pd.concat([lapse_time, dt], axis=1)
    time.columns = ['lapse','dt']
    return time

def data4dt(data):
    """Filter function for 'data' dataframe to apply date2sec
    """
#     return pd.concat([data, date2sec(data['date'])], axis=1)
    data['time']=date2sec(data['date'])['lapse']
    return pd.concat([data, date2sec(data['date'])['dt']], axis=1)

def save_dfFile(df, filename):
    df.to_csv(filename, sep='\t')

# def generate_plot(datafile, oldtype=False, comment=None, exc_iv=False, Rprotect=10e6, topdf=False, pdffile=None, tosvg=False, svgfile=None, ymax=0, ymin=0):
def generate_plot(datafile, oldtype=False, comment=None, exc_iv=False, Rprotect=10e6, topdf=False, tosvg=False,ymax=0, ymin=0, tolog=False):
    ext = datafile.rsplit('.')[-1]
    base = datafile.rsplit('.')[0]
    pdffile = base+'.pdf'
    svgfile = base+'.svgz'
    # if pdffile == 'None':
        # pdffile = base+'.pdf'
        # print(pdffile)
    # if svgfile == None:
    #     svgfile = base+'.svg'

    if ext == 'dat':
        if oldtype == False:
            data = pd.read_csv(datafile, delimiter='\t', comment='#',
                               names=['date','time','Ve','Ig','Ic', 'P', 'IVno'],
                               dtype={'Ve':'float64','Ig':'float64','Ic':'float64','P':'float64'})
        else:
            data = get_data_old(datafile)
        if comment == True:
            ### Read first line as comment
            with open(datafile, 'r') as f:
                cmt = f.readline()

        else:
            cmt = None

    elif (ext == 'hdf5') | (ext == 'h5'):
        data = pd.read_hdf(datafile, key='data')
        with h5py.File(datafile, 'r') as hf:
            # print(hf.keys())
            cmt = hf.get('comment').value.decode('utf-8')
            # print(cmt.value.decode('utf-8'))

    ### Omit Abnormal data
    ignore1 = data['Ig'].abs() > 10e+0
    ignore2 = data['Ic'].abs() > 10e+0

    if exc_iv == False:
        data = data[(ignore1 | ignore2) == False]
    else:
        ignore3 = data['IVno'] != 0
        data = data[(ignore1 | ignore2 | ignore3) == False]
        pdffile = base+'_noiv.pdf'
        # print(data)

    ### Calc lapse time and dt from 'date' column but not from 'time' column.
    ### This is because to make sure no matter what the 'time' column is presice.
    data = data4dt(data)

    # fig = plt.figure()
    gs = gridspec.GridSpec(3, 1, height_ratios=[1, 1, 3])
    ax1 = plt.subplot(gs[0])
    axp = plt.subplot(gs[1])
    ax2 = plt.subplot(gs[2])
    time_h = data['time']/3600

    if cmt:
        ax1.set_title(datafile+'\n'+cmt, fontsize=10)
    else:
        ax1.set_title(datafile)
    ax1.set_ylabel('Ve (kV)')
    # ax1.set_ylim(0,10)
    ax1.set_xticklabels('')
    ax1.grid('on')
    axp.set_ylabel('P (Pa)')
    axp.set_ylim(1e-6,1e-3)
    axp.grid('on')
    axp.set_xticklabels('')
    axp.set_yscale('log')

    ### Current - Time graph
    ax2.set_ylabel('Ig, Ic (nA)')
    ax2.set_xlabel('Time (h)')
    if (ymax !=0) or (ymin !=0):
        ax2.set_ylim(ymax=ymax, ymin=ymin)
    ax2.ticklabel_format(style = 'sci', axis='y', useOffset=False)
    ax2.grid('on')

    Vext = Ve_correct(data['Ve'], data['Ig']/Rs, data['Ic']/Rs, Rprotect) ### Added 161107

    ax1.plot(time_h, data['Ve']/1e3, 'k-')
    ax1.plot(time_h, Vext/1e3, 'r-') ### Added 161107
    axp.plot(time_h, data['P'], 'm-')

    if tolog == True:
        ax2.set_yscale('log')
        data['Ig'] = np.abs(data['Ig'])
        data['Ic'] = np.abs(data['Ic'])
    ax2.plot(time_h, data['Ig']/Rs*1e9, 'g-', label='Ig')
    ax2.plot(time_h, data['Ic']/Rs*1e9, 'b-', label='Ic')
    I = (data['Ic']+data['Ig'])/Rs
    # tot_fluence = I.sum()

    tot_fluence = np.sum((data['Ic']+data['Ig'])/Rs * data['dt']) # For the case of dt != 1

    ax2.set_title('Total '+ "{0:.2e}".format(tot_fluence) + ' (C)')
    ax2.plot(time_h, I*1e9, 'r-', label='Ig+Ic')
    ax2.legend(loc='best')

    if (topdf == False) and (tosvg == False):
    # if topdf == False:
        plt.show(block=False)

        input("<Hit Enter To Close>")
        return tot_fluence
    else:
        if topdf == True:
            plt.savefig(pdffile)
            print("{0} is created.".format(pdffile))
        if tosvg == True:
            plt.savefig(svgfile, format="svgz")
            print("{0} is created.".format(svgfile))
    return tot_fluence

    # return data

# generate_plot(datafile, oldtype=True)

    # {f} [-o | --oldtype] [-c | --comment] [-e | --exclude-iv]  [-r | --protect-resistor=<num>] [--ymax=<num>] [--ymin=<num>] [-p | --pdf=PDFFILE] [-s | --svg=SVGFILE] DATFILE
    # -s --svg=SVGFILE               Export graph to svg file [default: None]
__doc__ = """{f}
Usage:
    {f} [-o | --oldtype] [-c | --comment] [-e | --exclude-iv] [-l | --log]  [-p | --pdf] [-s | --svg] [-r | --protect-resistor=<num>] [--ymax=<num>] [--ymin=<num>] DATFILE
    {f} -h | --help

Options:
    -h --help                      Show this screen and exit.
    -o --oldtype                   Spesify dat file is formated with old type
    -c --comment                   Read comment from dat file and label on graph
    -l --log                       Specify y axis to be log for Ion current
    -p --pdf                       Export graph to pdf file
    -s --svg                       Export graph to svg file
    -r --protect-resistor=<num>    Set Rprotect in float [ohm]
    -e --exclude-iv                Exclude I-V measurement step
    --ymax=<num>                   Set ymax
    --ymin=<num>                   Set ymin
""".format(f=__file__)

def main():
    from docopt import docopt
    args = docopt(__doc__)

    # print(args["--oldtype"])
    oldtype = args["--oldtype"]
    comment = args["--comment"]
    datafile = args["DATFILE"]

    # if args['--pdf']==[]:
    #     topdf, pdfname = False, None
    # else:
    #     topdf, pdfname = True, args['--pdf']
    tolog = args['--log']
    topdf = args['--pdf']
    tosvg = args['--svg']
    # print(args['--svg'])
    # if args['--svg']==[]:
    #     tosvg, svgname = False, None
    # else:
    #     tosvg, svgname = True, args['--svg'][0]

    exc_iv = args['--exclude-iv']
    Rprotect = 10e6 if args["--protect-resistor"] == [] else float(args["--protect-resistor"][0])
    # print(args["--ymax"], args["--ymin"])
    ymax = 0 if args["--ymax"] == None else float(args["--ymax"])
    ymin = 0 if args["--ymin"] == None else float(args["--ymin"])
    tf = generate_plot(datafile, oldtype, comment, exc_iv, Rprotect, topdf=topdf, tosvg=tosvg, ymax=ymax, ymin=ymin, tolog=tolog)
    # tf = generate_plot(datafile, oldtype, comment, exc_iv, Rprotect, topdf=False, pdffile=None, tosvg=tosvg, svgfile=svgname, ymax=ymax, ymin=ymin)
    print("Total charge (C): {0:.3e}".format(tf))

if __name__ == '__main__':
    main()


text:
# From the course: Bayesin Machine Learning in Python: A/B Testing
# https://deeplearningcourses.com/c/bayesian-machine-learning-in-python-ab-testing
# https://www.udemy.com/bayesian-machine-learning-in-python-ab-testing
from __future__ import print_function, division
from builtins import range
# Note: you may need to update your version of future
# sudo pip install -U future

import numpy as np
from flask import Flask, jsonify, request
from scipy.stats import beta

# create an app
app = Flask(__name__)

# define bandit
# there's no "pull arm" here
# since that's technically now the user/client
class Bandit:
  def __init__(self, name):
    self.clks = 0
    self.views = 0
    self.name = name

  def sample(self):
    # Beta(1, 1) is the prior
    a = 1 + self.clks
    b = 1 + self.views - self.clks
    return np.random.beta(a, b)

  def add_click(self):
    self.clks += 1

  def add_view(self):
    self.views += 1

    # print some helpful stats
    if self.views % 50 == 0:
      print("%s: clks=%s, views=%s" % (self.name, self.clks, self.views))

# initialize bandits
banditA = Bandit('A')
banditB = Bandit('B')

@app.route('/get_ad')
def get_ad():
  if banditA.sample() > banditB.sample():
    ad = 'A'
    banditA.add_view()
  else:
    ad = 'B'
    banditB.add_view()
  return jsonify({'advertisement_id': ad})

@app.route('/click_ad', methods=['POST'])
def click_ad():
  result = 'OK'
  if request.form['advertisement_id'] == 'A':
    banditA.add_click()
  elif request.form['advertisement_id'] == 'B':
    banditB.add_click()
  else:
    result = 'Invalid Input.'

  # nothing to return really
  return jsonify({'result': result})

if __name__ == '__main__':
  app.run(host='127.0.0.1', port='8888')

max_stars_repo_name: JouniVatanen/NLP-and-Deep-Learning

reason: import numpy,from scipy


text:
# imports -------------------------------------------------------------------------#
import sys
import os
import argparse
import numpy as np
import math
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torchvision import datasets, transforms
from torchsummary import summary
from PIL import Image
from ema import EMA
from datasets import MnistDataset
from transforms import RandomRotation
from models.modelM3 import ModelM3
from models.modelM5 import ModelM5
from models.modelM7 import ModelM7

def run(p_seed=0, p_epochs=150, p_kernel_size=5, p_logdir="temp"):
    # random number generator seed ------------------------------------------------#
    SEED = p_seed
    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.benchmark = False
    torch.manual_seed(SEED)
    torch.cuda.manual_seed_all(SEED)
    np.random.seed(SEED)

    # kernel size of model --------------------------------------------------------#
    KERNEL_SIZE = p_kernel_size

    # number of epochs ------------------------------------------------------------#
    NUM_EPOCHS = p_epochs

    # file names ------------------------------------------------------------------#
    if not os.path.exists("../logs/%s"%p_logdir):
        os.makedirs("../logs/%s"%p_logdir)
    OUTPUT_FILE = str("../logs/%s/log%03d.out"%(p_logdir,SEED))
    MODEL_FILE = str("../logs/%s/model%03d.pth"%(p_logdir,SEED))

    # enable GPU usage ------------------------------------------------------------#
    use_cuda = torch.cuda.is_available()
    device = torch.device("cuda" if use_cuda else "cpu")
    if use_cuda == False:
        print("WARNING: CPU will be used for training.")
        exit(0)

    # data augmentation methods ---------------------------------------------------#
    transform = transforms.Compose([
        RandomRotation(20, seed=SEED),
        transforms.RandomAffine(0, translate=(0.2, 0.2)),
        ])

    # data loader -----------------------------------------------------------------#
    train_dataset = MnistDataset(training=True, transform=transform)
    test_dataset = MnistDataset(training=False, transform=None)
    train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=120, shuffle=True)
    test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=100, shuffle=False)

    # model selection -------------------------------------------------------------#
    if(KERNEL_SIZE == 3):
        model = ModelM3().to(device)
    elif(KERNEL_SIZE == 5):
        model = ModelM5().to(device)
    elif(KERNEL_SIZE == 7):
        model = ModelM7().to(device)

    summary(model, (1, 28, 28))

    # hyperparameter selection ----------------------------------------------------#
    ema = EMA(model, decay=0.999)
    optimizer = optim.Adam(model.parameters(), lr=0.001)
    lr_scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.98)

    # delete result file ----------------------------------------------------------#
    f = open(OUTPUT_FILE, 'w')
    f.close()

    # global variables ------------------------------------------------------------#
    g_step = 0
    max_correct = 0

    # training and evaluation loop ------------------------------------------------#
    for epoch in range(NUM_EPOCHS):
        #--------------------------------------------------------------------------#
        # train process                                                            #
        #--------------------------------------------------------------------------#
        model.train()
        train_loss = 0
        train_corr = 0
        for batch_idx, (data, target) in enumerate(train_loader):
            data, target = data.to(device), target.to(device)
            optimizer.zero_grad()
            output = model(data)
            loss = F.nll_loss(output, target)
            train_pred = output.argmax(dim=1, keepdim=True)
            train_corr += train_pred.eq(target.view_as(train_pred)).sum().item()
            train_loss += F.nll_loss(output, target, reduction='sum').item()
            loss.backward()
            optimizer.step()
            g_step += 1
            ema(model, g_step)
            if batch_idx % 100 == 0:
                print('Train Epoch: {} [{:05d}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
                    epoch, batch_idx * len(data), len(train_loader.dataset),
                    100. * batch_idx / len(train_loader), loss.item()))
        train_loss /= len(train_loader.dataset)
        train_accuracy = 100 * train_corr / len(train_loader.dataset)

        #--------------------------------------------------------------------------#
        # test process                                                             #
        #--------------------------------------------------------------------------#
        model.eval()
        ema.assign(model)
        test_loss = 0
        correct = 0
        total_pred = np.zeros(0)
        total_target = np.zeros(0)
        with torch.no_grad():
            for data, target in test_loader:
                data, target = data.to(device), target.to(device)
                output = model(data)
                test_loss += F.nll_loss(output, target, reduction='sum').item()
                pred = output.argmax(dim=1, keepdim=True)
                total_pred = np.append(total_pred, pred.cpu().numpy())
                total_target = np.append(total_target, target.cpu().numpy())
                correct += pred.eq(target.view_as(pred)).sum().item()
            if(max_correct < correct):
                torch.save(model.state_dict(), MODEL_FILE)
                max_correct = correct
                print("Best accuracy! correct images: %5d"%correct)
        ema.resume(model)

        #--------------------------------------------------------------------------#
        # output                                                                   #
        #--------------------------------------------------------------------------#
        test_loss /= len(test_loader.dataset)
        test_accuracy = 100 * correct / len(test_loader.dataset)
        best_test_accuracy = 100 * max_correct / len(test_loader.dataset)
        print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.2f}%) (best: {:.2f}%)\n'.format(
            test_loss, correct, len(test_loader.dataset), test_accuracy, best_test_accuracy))

        f = open(OUTPUT_FILE, 'a')
        f.write(" %3d %12.6f %9.3f %12.6f %9.3f %9.3f\n"%(epoch, train_loss, train_accuracy, test_loss, test_accuracy, best_test_accuracy))
        f.close()

        #--------------------------------------------------------------------------#
        # update learning rate scheduler                                           #
        #--------------------------------------------------------------------------#
        lr_scheduler.step()

if __name__ == "__main__":
    p = argparse.ArgumentParser()
    p.add_argument("--seed", default=0, type=int)
    p.add_argument("--trials", default=15, type=int)
    p.add_argument("--epochs", default=150, type=int)
    p.add_argument("--kernel_size", default=5, type=int)
    p.add_argument("--gpu", default=0, type=int)
    p.add_argument("--logdir", default="temp")
    args = p.parse_args()
    os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"
    os.environ["CUDA_VISIBLE_DEVICES"]=str(args.gpu)
    for i in range(args.trials):
        run(p_seed = args.seed + i,
            p_epochs = args.epochs,
            p_kernel_size = args.kernel_size,
            p_logdir = args.logdir)


text:
# coding: utf-8

# ## Import libraries

# In[1]:

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

from sklearn import linear_model
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression

# ## Import dataset

# In[2]:

df = pd.read_csv('housing.csv', index_col=[0])

# In[3]:

df.head()

# ## Get price and lotsize

# In[5]:

df = df[['price', 'lotsize']]
X = df[['lotsize']]
y = df[['price']]

# ## Spilt into train and test set

# In[6]:

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0)

# ## Fitting the model

# In[7]:

regressor = LinearRegression()
regressor.fit(X_train, y_train)

# ## Predict on test set

# In[12]:

y_pred = regressor.predict(X_test)
y_pred[:10]

# ## House Lotsize vs Price

# In[9]:

plt.figure(figsize=(20,10))
plt.scatter(X_train, y_train, color='red')
plt.plot(X_train, regressor.predict(X_train), color='blue', linewidth=3)
plt.title('Housing Prices Dataset (Training set)')
plt.xlabel('House Lotsize')
plt.ylabel('House Price')
plt.show()

# In[10]:

plt.figure(figsize=(20,10))
plt.scatter(X_test, y_test, color='red')
plt.plot(X_test, regressor.predict(X_test), color='blue', linewidth=3)
plt.title('Housing Prices Dataset (Test set)')
plt.xlabel('House Lotsize')
plt.ylabel('House Price')
plt.show()

max_stars_repo_name: nsudhanva/mca-code

max_issues_repo_name: nsudhanva/mca-code

max_forks_repo_name: nsudhanva/mca-code


text:
"""Module containing tools that are useful when benchmarking algorithms
"""
from math import hypot, sqrt
from functools import wraps
from itertools import repeat
try:
    import numpy
except ImportError:
    numpy = False

try:
    # try importing the C version
    from ._hypervolume import hv
except ImportError:
    # fallback on python version
    from ._hypervolume import pyhv as hv

def diversity(first_front, first, last):
    """Given a Pareto front `first_front` and the two extreme points of the
    optimal Pareto front, this function returns a metric of the diversity
    of the front as explained in the original NSGA-II article by K. Deb.
    The smaller the value is, the better the front is.
    """
    df = hypot(first_front[0].fitness.values[0] - first[0],
               first_front[0].fitness.values[1] - first[1])
    dl = hypot(first_front[-1].fitness.values[0] - last[0],
               first_front[-1].fitness.values[1] - last[1])
    dt = [hypot(first.fitness.values[0] - second.fitness.values[0],
                first.fitness.values[1] - second.fitness.values[1])
          for first, second in zip(first_front[:-1], first_front[1:])]

    if len(first_front) == 1:
        return df + dl

    dm = sum(dt)/len(dt)
    di = sum(abs(d_i - dm) for d_i in dt)
    delta = (df + dl + di)/(df + dl + len(dt) * dm )
    return delta

def convergence(first_front, optimal_front):
    """Given a Pareto front `first_front` and the optimal Pareto front,
    this function returns a metric of convergence
    of the front as explained in the original NSGA-II article by K. Deb.
    The smaller the value is, the closer the front is to the optimal one.
    """
    distances = []

    for ind in first_front:
        distances.append(float("inf"))
        for opt_ind in optimal_front:
            dist = 0.
            for i in range(len(opt_ind)):
                dist += (ind.fitness.values[i] - opt_ind[i])**2
            if dist < distances[-1]:
                distances[-1] = dist
        distances[-1] = sqrt(distances[-1])

    return sum(distances) / len(distances)

def hypervolume(front, ref=None):
    """Return the hypervolume of a *front*. If the *ref* point is not
    given, the worst value for each objective +1 is used.

    :param front: The population (usually a list of undominated individuals)
                  on which to compute the hypervolume.
    :param ref: A point of the same dimensionality as the individuals in *front*.
    """
    # Must use wvalues * -1 since hypervolume use implicit minimization
    wobj = numpy.array([ind.fitness.wvalues for ind in front]) * -1
    if ref is None:
        ref = numpy.max(wobj, axis=0) + 1
    return hv.hypervolume(wobj, ref)

max_stars_repo_name: joaofracasso/semanticGP

max_issues_repo_name: joaofracasso/semanticGP

max_forks_repo_name: joaofracasso/semanticGP