text:
#!/usr/bin/env python3
"""Contains `KappaComplex`, a class to represents a list of agents chained into a larger entity, and the `embed_and_map`
function."""

import re
import networkx as nx
import numpy as np
from collections import deque
from typing import Deque, Dict, List, FrozenSet, Set, Tuple, Union

from .KappaMultiAgentGraph import KappaMultiAgentGraph
from .KappaAgent import KappaAgent
from .KappaBond import KappaBond
from .KappaError import ComplexParseError, AgentParseError
from .KappaSite import KappaPort

class KappaComplex(KappaMultiAgentGraph):
    """Class for representing Kappa complexes. E.g. `A(b[1] s{u}[.]), B(a[1] c[2]), C(b[2] a[3]), A(c[3] s[.]{x})`."""

    # define agent pattern
    _agent_idnt_pat = r'(?:x\d+:)?'
    _agent_name_pat = r'(?:[_~][a-zA-Z0-9_~+-]+|[a-zA-Z][a-zA-Z0-9_~+-]*)'
    _agent_sign_pat = r'\([^()]*\)'
    _agent_pat = _agent_idnt_pat + _agent_name_pat + _agent_sign_pat
    _agent_pat_re = re.compile(_agent_pat)

    def __init__(self, expression: str):
        self._raw_expression: str
        self._agents: List[KappaAgent]
        self._agent_identifiers: List[int]
        self._agent_types: Set[KappaAgent]
        self._kappa_expression: str
        self._composition: Dict[KappaAgent, int]

        self._raw_expression = expression
        # get the set of agents making up this complex
        matches = self._agent_pat_re.findall(expression.strip())
        if len(matches) == 0:
            raise ComplexParseError('Complex <' + self._raw_expression + '> appears to have zero agents.')
        try:
            agent_list = []
            agent_idents = []
            agent_types = set()
            composition = {}
            for item in matches:
                agent = KappaAgent(item)
                agent_list.append(agent)
                if agent.get_agent_identifier() is not None:
                    agent_idents.append(agent.get_agent_identifier())
                # update type set, composition structures
                agent_type = KappaAgent(agent.get_agent_name() + '()')
                agent_types.update([agent_type])
                if agent_type in composition:
                    composition[agent_type] += 1
                else:
                    composition[agent_type] = 1
        except AgentParseError as a:
            raise ComplexParseError('Could not parse agents in complex <' + expression + '>.') from a
        self._agents = sorted(agent_list)
        self._agent_identifiers = agent_idents
        self._agent_types = agent_types
        self._composition = dict(sorted(composition.items(), key=lambda item: item[1]))
        # canonicalize the kappa expression
        self._kappa_expression = ', '.join([str(agent) for agent in self._agents])

    def get_number_of_bonds(self) -> int:
        """Returns the number of bonds in the complex."""
        bonds = set()
        for agent in self._agents:
            bonds.update(agent.get_bond_identifiers())
        return len(bonds)

    def get_size_of_complex(self) -> int:
        """Returns the size, in agents, of this complex."""
        return len(self._agents)

    def get_agent_types(self) -> Set[KappaAgent]:
        """Returns the set of agent names (or agent types) that make up the complex."""
        return self._agent_types

    def get_all_agents(self) -> List[KappaAgent]:
        """Returns a list of KappaAgents, filled with agents plus their signatures, present in this complex."""
        # replace commas with spaces, then split string into a list at closing parenthesis
        return self._agents

    def get_complex_composition(self) -> Dict[KappaAgent, int]:
        """Returns a dictionary where the key is an agent (fully qualified, not just a name), and the value the number
        of times that agent appears in this complex."""
        return self._composition

    def get_number_of_embeddings_of_agent(self, query) -> int:
        """Returns the number of embeddings the query agent has on the KappaComplex. For the 'truth table' of site
        nomenclature, see `KappaPort`."""
        # type the query into an Agent, if it's not one already
        if not type(query) is KappaAgent:
            q_agent = KappaAgent(query)
        else:
            q_agent = query
        # iterate over agents, checking if query is in each agent
        match_number = 0
        for s_agent in self._agents:
            if q_agent in s_agent:
                match_number += 1
        return match_number

    def get_number_of_embeddings_of_complex(self, query, symmetry_adjust: bool = True) -> int:
        """Returns the number of embeddings the query complex has on the KappaComplex. Optional parameter to not perform
        the symmetry adjustment and report number of raw embeddings. See the `embed_and_map` function for examples and
        advanced usage."""
        if not type(query) is KappaComplex:
            q_complex = KappaComplex(query)
        else:
            q_complex = query
        total_maps, unique_maps = embed_and_map(q_complex, self)
        if symmetry_adjust:
            return len(unique_maps)
        else:
            return len(total_maps)

    def get_number_of_embeddings(self, query, symmetry_adjust: bool = True) -> int:
        """Wrapper for the two specialized functions, for agent and complex. Optional parameter to not perform
        the symmetry adjustment and report the number of raw embeddings. See the `embed_and_map` function for examples
        and advanced usage."""
        if type(query) is KappaAgent:
            return self.get_number_of_embeddings_of_agent(query)
        elif type(query) is KappaComplex:
            return self.get_number_of_embeddings_of_complex(query, symmetry_adjust)
        else:
            try:
                try:
                    return self.get_number_of_embeddings_of_agent(KappaAgent(query))
                except AgentParseError:
                    return self.get_number_of_embeddings_of_complex(KappaComplex(query), symmetry_adjust)
            except ComplexParseError:
                raise ValueError('Could not parse <{}> as a KappaAgent nor as a KappaComplex.'.format(query))

    def get_agent_identifiers(self) -> List[int]:
        """Returns a list with the numeric agent identifiers, if any."""
        return self._agent_identifiers

    def to_networkx(self, identifier_offset: int = 0) -> nx.MultiGraph:
        """Returns a Multigraph representation of the complex, abstracting away binding site data. Nodes represent
        agents, edges their bonds. Nodes have an attribute dictionary where the key `kappa` holds the KappaAgent.
        Edges have an attribute dictionary where the key `bond id` holds the bond identifier from the Kappa expression.
        Node identifiers are integers, using the order of agent declaration. For a graph `g`, `g.nodes.data()` displays
        the node identifiers and their corresponding `KappaAgents`, and `g.edges.data()` displays the edges, using the
        node identifiers as well as the kappa identifiers.
        The optional parameter `identifier_offset` will offset all numeric identifiers reported; used in unlabeled
        snapshots, or when combining graphs."""
        kappa_complex_multigraph = nx.MultiGraph()
        dangle_bond_dict = {}                       # store unpaired bonds he
        paired_bond_list = []                       # store tuples of (agent index 1, agent index 2, bond identifier)
        agent_counter = 0                           # if using un-labeled kappa, default to this
        for agent in self.get_all_agents():
            if agent.get_agent_identifier():
                agent_global_id = agent.get_agent_identifier() + identifier_offset
            else:
                agent_global_id = agent_counter + identifier_offset
            kappa_complex_multigraph.add_node(agent_global_id, kappa=agent)
            for bond in agent.get_bond_identifiers():
                # if we've already seen this edge and it is in the dangling list, it's partner has already been matched,
                # so we can add this terminus to the bond database and delete from the dangle list
                if bond in dangle_bond_dict:
                    # special case for self-bonds: the first pass already got the alphabetically lower terminus,
                    # so this pass should get the second terminus of the bond
                    if len(agent.get_terminii_of_bond(bond)) > 1:
                        second_terminus = agent.get_terminii_of_bond(bond)[1]
                    else:
                        second_terminus = agent.get_terminii_of_bond(bond)[0]
                    paired_bond_list.append((dangle_bond_dict[bond]['agent id'], agent_global_id,
                                             {'bond id': bond, 'bond type':
                                             KappaBond(agent_one=dangle_bond_dict[bond]['agent name'],
                                                       site_one=dangle_bond_dict[bond]['site name'],
                                                       agent_two=agent.get_agent_name(),
                                                       site_two=second_terminus
                                                       )}))
                    del dangle_bond_dict[bond]
                else:
                    dangle_bond_dict[bond] = {'agent id': agent_global_id,
                                              'agent name': agent.get_agent_name(),
                                              'site name': agent.get_terminii_of_bond(bond)[0]}
            agent_counter += 1
        # if anything remains in the dangling bond list, it means we failed to pair at least one bond terminus
        if dangle_bond_dict:
            raise ValueError('Dangling bonds <' + ','.join(dangle_bond_dict.keys()) +
                             '> found in complex: ' + self._raw_expression)
        kappa_complex_multigraph.add_edges_from(paired_bond_list)
        return kappa_complex_multigraph

    def to_cytoscape_cx(self) -> List[Dict]:
        """
        Export to a structure that, via some json encoding and dumping, can be read by Cytoscape as a CX file. Usage:
        >>> my_cx = my_complex.to_cytoscape_cx()
        >>> with open('my_cx.cx', 'w') as out_file:
        json.dump(my_cx, out_file)
        """
        cx_data = self._kappa_to_cytoscape_cx()
        cx_network_attributes = [{'n': 'name', 'v': 'network'}]
        cx_data.insert(2, {'networkAttributes': cx_network_attributes})
        return cx_data

def embed_and_map(ka_query: KappaComplex, ka_target: KappaComplex) -> \
        Tuple[List[List[Tuple[int, int]]], List[List[Tuple[int, int]]]]:
    """
    Calculates all the embeddings of `ka_query` into `ka_target`, returning both the map of all embeddings, as well as
    the map of embeddings corrected for the number of preserved automorphisms. For a rotational symmetry:
    >>> from KaSaAn.core.KappaComplex import embed_and_map, KappaComplex
    >>> my_comp = KappaComplex('Bob(h[10], t[11]), Bob(h[11], t[12]), Bob(h[12], t[10])')
    >>> maps_all, maps_unique = embed_and_map(my_comp, my_comp)
    >>> maps_all
    [[(0, 0), (2, 2), (1, 1)], [(0, 1), (2, 0), (1, 2)], [(0, 2), (2, 1), (1, 0)]]
    >>> maps_unique
    [[(0, 0), (2, 2), (1, 1)]]

    There are three ways of satisfying the query in the target, and these rotations inflate the number of "embeddings".
    However, the set of identifiers making up the image of the query in the target is the same for these three:
    `(0, 2, 1)`, `(1, 0, 2)`, and `(2, 1, 0)` are equivalent, and so the target contains only one copy of the query.
    These dual-purpose interpretation of the "embedding" concept yields a function that returns both.
    """
    # litany of short circuits
    if ka_query.get_size_of_complex() > ka_target.get_size_of_complex():    # not enough agents
        return ([], [])
    if ka_query.get_number_of_bonds() > ka_target.get_number_of_bonds():    # not enough bonds
        return ([], [])
    query_comp = ka_query.get_complex_composition()
    target_comp = ka_target.get_complex_composition()
    if not set(query_comp) <= set(target_comp):                             # query agent(s) not present in target
        return ([], [])
    for agent_type, query_abundance in query_comp.items():
        if agent_type not in target_comp:                                   # query agent missing in target
            return ([], [])
        if target_comp[agent_type] < query_abundance:                       # target sum formula too small
            return ([], [])
    # start from the least abundant type, get their node indexes in query network and target network;
    # from the <<query is improper subset of target>> check above, all of query's are in target by type, so query's
    # minimum must be a type in common; moreover all query's type abundances are equal or greater in target, so if
    # a type is the minimum in query, its abundance will also be either *the*, or just *a*, minimum in target
    common_min: KappaAgent = next(iter(query_comp))
    query_network = ka_query.to_networkx()
    target_network = ka_target.to_networkx()
    if not nx.is_connected(query_network):
        raise ValueError('Error: query is not a connected graph.')
    if not nx.is_connected(target_network):
        raise ValueError('Error: target is not a connected graph.')
    common_min_q = []
    for node_id in query_network.nodes:
        if common_min in query_network.nodes[node_id]['kappa']:
            common_min_q.append(node_id)
    common_min_t = []
    for node_id in target_network.nodes:
        if common_min in target_network.nodes[node_id]['kappa']:
            common_min_t.append(node_id)
    # embark on systematic traversal
    query_start_node = common_min_q[0]
    maps_all: List[List[Tuple[int, int]]] = []
    maps_distinct:  List[List[Tuple[int, int]]] = []
    map_set: Set[FrozenSet[int]] = set()
    for target_start_node in common_min_t:
        map_found = _traverse_from(query_network, target_network, query_start_node, target_start_node)
        if map_found:
            maps_all.append(map_found)
            _, t_nodes = zip(*map_found)
            t_nodes = frozenset(t_nodes)
            if t_nodes not in map_set:
                map_set.update([t_nodes])
                maps_distinct.append(map_found)
    return maps_all, maps_distinct

def _traverse_from(query_net: nx.MultiGraph, target_net: nx.MultiGraph, query_start: int, target_start: int) -> \
        List[Tuple[int, int]]:
    """Attempt a traversal of `target_net`, starting at `target_start`, matched to `query_start`, following
    `query_net`'s topology. If successful, returns the network mapping: a list of `(q,t)`, the indexes in the query and
    target networks respectively."""
    # stack of node identifiers, integers from the networkx representation
    node_stack: Deque[Tuple[int, Union[int, None], int]] = deque()
    node_stack.append((query_start, None, target_start))
    nodes_visited: Set[int] = set()
    network_map = []
    while node_stack:
        q_node, q_prev, t_node = node_stack.pop()
        if q_node not in nodes_visited:
            # update t_node according to what got us from q_prev to q_node
            if q_prev is not None:
                # uniqueness of site name per agent, plus orientation of bond types,
                # guarantee a bond type can appear at most once in an agent's
                # bond set when read with the right orientation; this yields "rigidity" to the matcher
                transforming_bond_type: KappaBond = query_net.get_edge_data(q_prev, q_node)[0]['bond type']
                transform_index_source = np.where([node == q_prev for node in query_net.nodes])[0][0]
                transform_index_destin = np.where([node == q_node for node in query_net.nodes])[0][0]
                if transform_index_destin < transform_index_source:
                    transforming_bond_type = transforming_bond_type.reverse()
                target_bonds: List[KappaBond] = []
                # get oriented bond list, incident on target node
                for a, b in nx.edges(target_net, t_node):
                    bond_type: KappaBond = target_net.get_edge_data(a, b)[0]['bond type']
                    if a == t_node:
                        source_index = np.where([node == a for node in target_net.nodes])[0][0]
                        destin_index = np.where([node == b for node in target_net.nodes])[0][0]
                    else:
                        source_index = np.where([node == b for node in target_net.nodes])[0][0]
                        destin_index = np.where([node == a for node in target_net.nodes])[0][0]
                    if source_index < destin_index:
                        target_bonds.append(bond_type)
                    else:
                        target_bonds.append(bond_type.reverse())
                target_transforming_bond_index: int = \
                    np.where([transforming_bond_type == item for item in target_bonds])[0][0]
                t_node = list(nx.edges(target_net, t_node))[target_transforming_bond_index][1]
            # now at parity, match by node & edge data
            node_matched = _node_match(query_net, target_net, q_node, t_node)
            edge_matched = _edge_match(query_net, target_net, q_node, t_node)
            if node_matched and edge_matched:
                network_map.append((q_node, t_node))
                nodes_visited.add(q_node)
                for q_neighbor in nx.neighbors(query_net, q_node):
                    node_stack.append((q_neighbor, q_node, t_node))
            else:
                return []
    return network_map

def _node_match(query_net: nx.MultiGraph, target_net: nx.MultiGraph, query_node: int, target_node: int) -> bool:
    """Special purpose matcher that ignores bond types, considering only if sites are bound. Internal states are
    matched normally."""
    match: bool = False
    query = query_net.nodes[query_node]['kappa']
    target = target_net.nodes[target_node]['kappa']
    for site in query.get_agent_ports():
        s_name = site.get_port_name()
        s_stat = '{' + site.get_port_int_state() + '}'
        s_bond = '[' + site.get_port_bond_state() + ']' if site.get_port_bond_state() in ['.', '_', '#'] else '[_]'
        relaxed_port = KappaPort(s_name + s_bond + s_stat)
        if not any([relaxed_port in t_site for t_site in target.get_agent_ports()]):
            return match
    match = True
    return match

def _edge_match(query_net: nx.MultiGraph, target_net: nx.MultiGraph, query_node: int, target_node: int) -> bool:
    """
Special purpose matcher that only compares bond types. Returns a tuple, where the first value is a boolean
holding whether the whole thing matched, and the second the list of neighbors in target for which there was a
bond-type match.

Rigidity in bonds in addition to site ordering!
Since there is at maximum one of any bond type per node in the network, finding it means finding the only path
like it, reducing the search space. This relies on the bonds being oriented, in this case I chose "outgoing" from
the current node. Since agent identifiers can be written in arbitrary order, I can't rely on just a<b comparison
at the identifier level to know if the bond's string was written in that same direction; to resolve this one must
compare the order of node introduction, which tracks the order of edge / node declaration.
    """
    match: bool = False
    query_edges = nx.edges(query_net, query_node)
    query_bond_types: List[KappaBond] = []
    # orient bonds so as to be read outgoing
    for a, b in query_edges:
        this_bond: KappaBond = query_net.get_edge_data(a, b)[0]['bond type']
        # NetworkX appears to return the edges in incident format (a,b), where the 'a' identifier
        # is the same as the requested node, so for all, a==target_node (just to be safe, I have
        # a manual check...)
        if a == query_node:
            source_index = np.where([node == a for node in query_net.nodes])[0][0]
            destin_index = np.where([node == b for node in query_net.nodes])[0][0]
        else:
            source_index = np.where([node == b for node in query_net.nodes])[0][0]
            destin_index = np.where([node == a for node in query_net.nodes])[0][0]
        if source_index < destin_index:
            query_bond_types.append(this_bond)
        else:
            query_bond_types.append(this_bond.reverse())
    target_edges = nx.edges(target_net, target_node)
    target_bond_types: List[KappaBond] = []
    for a, b in target_edges:
        this_bond: KappaBond = target_net.get_edge_data(a, b)[0]['bond type']
        if a == target_node:
            source_index = np.where([node == a for node in target_net.nodes])[0][0]
            destin_index = np.where([node == b for node in target_net.nodes])[0][0]
        else:
            source_index = np.where([node == b for node in target_net.nodes])[0][0]
            destin_index = np.where([node == a for node in target_net.nodes])[0][0]
        if source_index < destin_index:
            target_bond_types.append(this_bond)
        else:
            target_bond_types.append(this_bond.reverse())
    # check now-oriented bonds
    for query_bond_type in query_bond_types:
        if not any([query_bond_type == target_bond_type for target_bond_type in target_bond_types]):
            return match
    match = True
    return match

max_stars_repo_name: hmedina/KaSaAn

max_issues_repo_name: hmedina/KaSaAn

max_forks_repo_name: hmedina/KaSaAn

reason: import numpy,import networkx


text:
# coding=utf-8
# Copyright 2018 The Tensor2Tensor Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Utilities for openai gym."""

from collections import deque

# Dependency imports

import gym

import numpy as np

import six

from tensor2tensor.data_generators import image_utils

class WarmupWrapper(gym.Wrapper):
  """Warmup wrapper."""

  def __init__(self, env, warm_up_examples=0, warmup_action=0):
    gym.Wrapper.__init__(self, env)
    self.warm_up_examples = warm_up_examples
    self.warm_up_action = warmup_action
    self.observation_space = gym.spaces.Box(
        low=0, high=255, shape=(210, 160, 3), dtype=np.uint8)

  def get_starting_data(self, num_frames):
    self.reset()
    starting_observations, starting_actions, starting_rewards = [], [], []
    for _ in range(num_frames):
      observation, rew, _, _ = self.env.step(self.warm_up_action)
      starting_observations.append(observation)
      starting_rewards.append(rew)
      starting_actions.append(self.warm_up_action)

    return starting_observations, starting_actions, starting_rewards

  def step(self, ac):
    action = ac
    return self.env.step(action)

  def reset(self, **kwargs):
    self.env.reset()
    observation = None
    for _ in range(self.warm_up_examples):
      observation, _, _, _ = self.env.step(self.warm_up_action)

    return observation

class PongWrapper(WarmupWrapper):
  """Pong Wrapper."""

  def __init__(self, env, warm_up_examples=0,
               action_space_reduction=False,
               reward_skip_steps=0,
               big_ball=False):
    super(PongWrapper, self).__init__(env, warm_up_examples=warm_up_examples)
    self.action_space_reduction = action_space_reduction
    if self.action_space_reduction:
      self.action_space = gym.spaces.Discrete(2)
    self.warm_up_examples = warm_up_examples
    self.observation_space = gym.spaces.Box(
        low=0, high=255, shape=(210, 160, 3), dtype=np.uint8)
    self.reward_skip_steps = reward_skip_steps
    self.big_ball = big_ball

  def step(self, ac):
    action = ac
    if self.action_space_reduction:
      action = 2 if int(ac) == 0 else 5
    ob, rew, done, info = self.env.step(action)
    ob = self.process_observation(ob)
    if rew != 0 and self.reward_skip_steps != 0:
      for _ in range(self.reward_skip_steps):
        self.env.step(0)
    return ob, rew, done, info

  def reset(self, **kwargs):
    observation = super(PongWrapper, self).reset(**kwargs)
    observation = self.process_observation(observation)
    return observation

  def process_observation(self, obs):
    if self.big_ball:
      pos = PongWrapper.find_ball(obs)
      if pos is not None:
        x, y = pos
        obs[x-5:x+5, y-5:y+5, :] = 255

    return obs

  @staticmethod
  def find_ball(obs, default=None):
    ball_area = obs[37:193, :, 0]
    res = np.argwhere(ball_area == 236)
    if not res:
      return default
    else:
      x, y = res[0]
      x += 37
      return x, y

def wrapped_pong_factory(warm_up_examples=0, action_space_reduction=False,
                         reward_skip_steps=0, big_ball=False):
  """Wrapped pong games."""
  env = gym.make("PongDeterministic-v4")
  env = env.env  # Remove time_limit wrapper.
  env = PongWrapper(env, warm_up_examples=warm_up_examples,
                    action_space_reduction=action_space_reduction,
                    reward_skip_steps=reward_skip_steps,
                    big_ball=big_ball)
  return env

gym.envs.register(id="T2TPongWarmUp20RewSkip200Steps-v1",
                  entry_point=lambda: wrapped_pong_factory(  # pylint: disable=g-long-lambda
                      warm_up_examples=20, reward_skip_steps=15),
                  max_episode_steps=200)

gym.envs.register(id="T2TPongWarmUp20RewSkip2000Steps-v1",
                  entry_point=lambda: wrapped_pong_factory(  # pylint: disable=g-long-lambda
                      warm_up_examples=20, reward_skip_steps=15),
                  max_episode_steps=2000)

class BreakoutWrapper(WarmupWrapper):
  """Breakout Wrapper."""

  FIRE_ACTION = 1

  def __init__(self, env, warm_up_examples=0,
               ball_down_skip=0,
               big_ball=False,
               include_direction_info=False,
               reward_clipping=True):
    super(BreakoutWrapper, self).__init__(
        env, warm_up_examples=warm_up_examples,
        warmup_action=BreakoutWrapper.FIRE_ACTION)
    self.warm_up_examples = warm_up_examples
    self.observation_space = gym.spaces.Box(low=0, high=255,
                                            shape=(210, 160, 3),
                                            dtype=np.uint8)
    self.ball_down_skip = ball_down_skip
    self.big_ball = big_ball
    self.reward_clipping = reward_clipping
    self.include_direction_info = include_direction_info
    self.direction_info = deque([], maxlen=2)
    self.points_gained = False
    msg = ("ball_down_skip should be bigger equal 9 for "
           "include_direction_info to work correctly")
    assert not self.include_direction_info or ball_down_skip >= 9, msg

  def step(self, ac):
    ob, rew, done, info = self.env.step(ac)

    if BreakoutWrapper.find_ball(ob) is None and self.ball_down_skip != 0:
      for _ in range(self.ball_down_skip):
        # We assume that nothing interesting happens during ball_down_skip
        # and discard all information.
        # We fire all the time to start new game
        ob, _, _, _ = self.env.step(BreakoutWrapper.FIRE_ACTION)
        self.direction_info.append(BreakoutWrapper.find_ball(ob))

    ob = self.process_observation(ob)

    self.points_gained = self.points_gained or rew > 0

    if self.reward_clipping:
      rew = np.sign(rew)

    return ob, rew, done, info

  def reset(self, **kwargs):
    observation = super(BreakoutWrapper, self).reset(**kwargs)
    self.env.step(BreakoutWrapper.FIRE_ACTION)
    self.direction_info = deque([], maxlen=2)
    observation = self.process_observation(observation)
    return observation

  @staticmethod
  def find_ball(ob, default=None):
    off_x = 63
    clipped_ob = ob[off_x:-21, :, 0]
    pos = np.argwhere(clipped_ob == 200)

    if not pos.size:
      return default

    x = off_x + pos[0][0]
    y = 0 + pos[0][1]
    return x, y

  def process_observation(self, obs):
    if self.big_ball:
      pos = BreakoutWrapper.find_ball(obs)
      if pos is not None:
        x, y = pos
        obs[x-5:x+5, y-5:y+5, :] = 255

    if self.include_direction_info:
      for point in list(self.direction_info):
        if point is not None:
          x, y = point
          obs[x-2:x+2, y-2:y+2, 1] = 255

    return obs

def wrapped_breakout_factory(warm_up_examples=0,
                             ball_down_skip=0,
                             big_ball=False,
                             include_direction_info=False,
                             reward_clipping=True):
  """Wrapped breakout games."""
  env = gym.make("BreakoutDeterministic-v4")
  env = env.env  # Remove time_limit wrapper.
  env = BreakoutWrapper(env, warm_up_examples=warm_up_examples,
                        ball_down_skip=ball_down_skip,
                        big_ball=big_ball,
                        include_direction_info=include_direction_info,
                        reward_clipping=reward_clipping)
  return env

gym.envs.register(id="T2TBreakoutWarmUp20RewSkip500Steps-v1",
                  entry_point=lambda: wrapped_breakout_factory(  # pylint: disable=g-long-lambda
                      warm_up_examples=1,
                      ball_down_skip=9,
                      big_ball=False,
                      include_direction_info=True,
                      reward_clipping=True
                  ),
                  max_episode_steps=500)

class FreewayWrapper(WarmupWrapper):
  """Wrapper for Freeway."""

  def __init__(self, env,
               warm_up_examples=0,
               reward_clipping=True,
               easy_freeway=False):
    super(FreewayWrapper, self).__init__(env, warm_up_examples)
    self.easy_freeway = easy_freeway
    self.half_way_reward = 1.0

    # this is probably not needed, just in case
    self.reward_clipping = reward_clipping

  def chicken_height(self, image):
    raise NotImplementedError()

  def step(self, ac):
    ob, rew, done, info = self.env.step(ac)

    if self.easy_freeway:
      if rew > 0:
        self.half_way_reward = 1
      chicken_height = self.chicken_height(ob)
      if chicken_height < 105:
        rew += self.half_way_reward
        self.half_way_reward = 0

    if self.reward_clipping:
      rew = np.sign(rew)

    return ob, rew, done, info

  def reset(self, **kwargs):
    self.half_way_reward = 1.0
    observation = super(FreewayWrapper, self).reset(**kwargs)
    return observation

def wrapped_freeway_factory(warm_up_examples=0,
                            reward_clipping=True,
                            easy_freeway=False):
  """Wrapped freeway games."""
  env = gym.make("FreewayDeterministic-v4")
  env = env.env  # Remove time_limit wrapper.
  env = FreewayWrapper(env, warm_up_examples=warm_up_examples,
                       reward_clipping=reward_clipping,
                       easy_freeway=easy_freeway)

  return env

gym.envs.register(id="T2TFreewayWarmUp20RewSkip500Steps-v1",
                  entry_point=lambda: wrapped_freeway_factory(  # pylint: disable=g-long-lambda
                      warm_up_examples=1,
                      reward_clipping=True,
                      easy_freeway=False
                  ),
                  max_episode_steps=500)

def encode_image_to_png(image):
  encoded = six.next(
      image_utils.encode_images_as_png([image]))
  return encoded

ext: py

reason: import numpy


text:
#!/usr/bin/env python
# coding: utf-8

# # Tarea 3
#
# _Tarea 3_ de _Benjamín Rivera_ para el curso de __Métodos Numéricos__ impartido por _Joaquín Peña Acevedo_. Fecha limite de entrega __20 de Septiembre de 2020__.

# ### Como ejecutar

# ##### Requerimientos
#
# Este programa se ejecuto en mi computadora con la version de __Python 3.8.2__ y con estos
# [requerimientos](https://github.com/BenchHPZ/UG-Compu/blob/master/MN/requerimientos.txt)
#
# #### Jupyter
#
# En caso de tener acceso a un _servidor jupyter_ ,con los requerimientos antes mencionados, unicamente basta con ejecutar todas las celdas de este _notebook_. Probablemente no todas las celdas de _markdown_ produzcan el mismo resultado por las
# [_Nbextensions_](jupyter-contrib-nbextensions.readthedocs.io).
#
# #### Consola
#
# Habrá archivos e instrucciones para poder ejecutar cada uno de los ejercicios desde la consola.
#
# #### Si todo sale mal
#
#
#
#
#
# En caso de que todo salga mal, tratare de dejar una copia disponible en __GoogleColab__ que se pueda ejecutar con la versión de __Python__ de _GoogleColab_

# In[9]:

# Init cell
import sys
import numpy as np
import matplotlib.pyplot as plt

notebook = False

if notebook:
    print("Todo bien con la importacion xD")

# ## Ejercicio 1

# ### .
#
# Muestre que la inversa de $L_k$ es $L^{-1}_k = I + l_ke_k^T$. Asi, la inversa de una matriz triangular inferior elemental es otra matriz triangular inferior elemental.

# Sabemos que, si la matriz es invertible, entonces $L_kL_k^{-1} = I$.
#
# Procedemos por contradiccion. Supongamos que $L^{-1}_k \neq I + l_ke_k^T$, de manera que $L_k(I + l_ke_k^T) \neq I$; si desarrolamos lo anterior resulta que
#
# \begin{eqnarray*}
#     L_k(I + l_ke_k^T) &\neq& I \\
#     (I - l_ke_k^T)(I + l_ke_k^T) &\neq& I \\
#     I^2 - (l_ke_k^T)^2 &\neq& I \\
#     I - (l_ke_k^T)^2 &\neq& I \\
# \end{eqnarray*}
#
# lo cual es cierto siempre que $(l_ke_k^T)^2 \neq \bar{0}$. Sin embargo, por la manera en que estan definidas $l_ke_k^T$, $(l_ke_k^T)^2 = 0$. Por lo que llegamos auna contradicion y $L^{-1}_k = I + l_ke_k^T$.
#

# ### .
#
# Muestre que $L^{-1}_{k-1}L^{-1}_k = I + l_{k-1}e^T_k + l_ke^t_k$ y con esto demostrar que
#
# \begin{equation*} L = L^{-1}_1 L^{-1}_2 \dots L^{-1}_{n-1} = I + \sum_{k=1}^{n-1}l_ke_k^T
# \end{equation*}
#
# por lo que $L$ es una matriz triangular inferior con 1's en la diagonal
#

# \textbf{Parte 1} Por definicion y el punto anterior
#
# \begin{eqnarray*}
#     L_{k-1}^{-1}L_{k}^{-1} &=& (I + l_{k-1}e_{k-1}^T)(I + l_ke_k^T) \\
#         &=& I^2 + l_{k-1}e_{k-1}^T(l_ke_k^T) + l_{k-1}e_{k-1}^T + l_ke_k^T \\
#         &&\text{y como sabemos que la multiplicacion de las submatrices $l,e$ es 0} \\
#         &=& I + \bar{0} + l_{k-1}e_{k-1}^T + l_ke_k^T \\
#         &=& I + l_{k-1}e_{k-1}^T + l_ke_k^T
# \end{eqnarray*}
#
# \textbf{Parte 2} ...

# ## Ejercicio 2

# In[2]:

def show1D( x,sz,/, max_sz=8, prnt=True):
    """ Funcion que recibe un arreglo 1D y, en caso de
    que no sea muy grande, lo imprime en pantalla. En
    caso de que lo sea, imprime datos representativos.
        Input:
            x := Apuntador al arreglo 1D para imprimir
            sz := Tamanio del arreglo. (para la imple_
                mentacion en python es inecesario, pe_
                ro lo solicita el ejercicio)
            max_sz := Maximo de elementos a imprimir
            prnt := Indica si el string obtenido se
                debe imprimir en pantalla
        Output:
            Esta funcion regresa

        _Doctest:
            >>> show1D([1,2,3,8,5,6,7,8,9,0], 10, prnt=False)
            '1, 2, 3, 8, ... , 7, 8, 9, 0'

            >>> show1D([1,2,3,8], 4, prnt=False)
            '1, 2, 3, 8'
    """
    ret = '=('

    if sz <= max_sz:
        ret = str(x)[1:-1]
    else:
        mid = max_sz//2

        ret = str(x[ :mid])[1:-1]
        ret += ', ... , '
        ret += str(x[-mid:])[1:-1]

    if prnt: print(ret)
    return ret

# In[3]:

def forwardSubstitution( L, n, b, t,/,dtype=np.float64):
    """ Funcion que implementa el metodo desustitucion pa_
    ra adelante dada una matriz triangular inferior.
        Input:
            L := apuntador a la matriz L
            n := tamanio de la matriz L
            b := apuntador al vector b
            t := tolerancia de 0's
        Otput:
            ret := vector x de respuestas si se encontro, o
                None en caso de que hay habido algun error

        _Doctest:
            >>> forwardSubstitution(np.matrix([[1,0],[0,1]]), 1, np.matrix([[3,4]]), 0.1)
            matrix([[3.],
                    [4.]])
    """

    if len(L) == b.size:
        n = len(L)
        # verificamos que la matriz tenga solucion
        for i in range(n):
            if abs(L[i,i]) <= t:
                print("Err: 0 o casi cero en diagonal")
                return None

        x = np.zeros((n), dtype=dtype)
        for i in range(n):
            x[i] = (b[0,i] - np.sum([L[i,j]*x[j] for j in range(i)],
                                    dtype=dtype)
                   )/L[i,i]

        return np.asmatrix(x, dtype=dtype).transpose()

    else:
        print("Err: dimensiones")
        return None

# In[4]:

def readFile(file,/, path='datos/npy/', ext='.npy', dtype=np.float64):
    """ Funcion para cargar los archivos en memoria en el
    formato que se meustra en el punto 4 del ejercicio 2.
    No lleva el path, ni la extension, solo el nombre del
    archivo. Por default trata de leer los archivos npy
        Input:
            file := nombre del archivo sin extension
            path := directorio para buscar el archivo
            ext := extension del archivo a buscar
            dtype := tipo de dato para guardar los valores
    """
    try:
        return np.asmatrix(np.load(file= str(path+file+ext),
                                   allow_pickle=False),
                          dtype=dtype)
    except:
        raise Exception("Error al encontrar el archivo solicitado.")

def Ejercicio2(f_L, f_b):
    """ Programa para ejecutar el ejercicio 2. espera el nombre
    de los archivos de la matriz y el vector soluciones para
    funcionar.
    """
    L = readFile(f_L)
    b = readFile(f_b)
    dtype = np.float64
    t = (2.2e-16)**(2/3) # De la tarea 1 y como estamos usando flaot64

    print(len(L), b.size)

    x = forwardSubstitution(L, len(L), b, t, dtype=dtype)

    if not x is None:
        show1D(x, len(x))
        error = np.linalg.norm(L*x-b.transpose()) #Norma de numpy
        print(f'Error =\n {error}')
    else:
        print("El sistema no tiene solucion.\nY es singular.")

if notebook:
    for sz in [5, 50, 500]:
        Ejercicio2('matL'+str(sz), 'vecb'+str(sz))
        print('\n')

# #### Como ejecutar
# Para ejecutar en consola se necesita estar en el mismo directorio que la carpeta de la tarea `datos`. Como nombre de los archivos no se espera la direccion completa ni la extension del archivo. Se infiere automatincamente que los datos sera obtenidos del subdirectorio `datos/npy` y con la extension `.npy`. De manera qu la ejecucion debe parecerse a lo siguiente
# ```console
# $ python3 Tarea3.py matL5 vecb5
# ```
#
# #### Otros
# [Norma de Numpy](https://numpy.org/doc/stable/reference/generated/numpy.linalg.norm.html)

# ## Ejercicio 3

# In[5]:

def backwardSubstitution( U, n, b, t,/,dtype=np.float64):
    """ Funcion que implementa el metodo desustitucion pa_
    ra atras dada una matriz triangular inferior.
        Input:
            U := apuntador a la matriz U
            n := tamanio de la matriz U
            b := apuntador al vector b
            t := tolerancia de 0's
        Otput:
            ret := vector x de respuestas si se encontro, o
                None en caso de que hay habido algun error

    """

    if len(U) == b.size:
        n = len(U)

        # verificamos que la matriz tenga solucion
        for i in range(n):
            if abs(U[i,i]) <= t:
                print("Err: 0 o casi cero en diagonal")
                return None

        x = np.zeros((n), dtype=dtype)
        for i in range(n-1, -1, -1):
            x[i] = (b[0,i] - np.sum([U[i,j]*x[j] for j in range(i+1-1, n)],
                                    dtype=dtype)
                   )/U[i,i]

        return np.asmatrix(x, dtype=dtype).transpose()

    else:
        print("Err: dimensiones")
        return None

# In[8]:

def Ejercicio3(f_U, f_b):
    """ Programa para ejecutar el ejercicio 3. espera el nombre
    de los archivos de la matriz y el vector soluciones para
    funcionar.
    """
    U = readFile(f_U)
    b = readFile(f_b)
    dtype = np.float64
    t = (2.2e-16)**(2/3) # De la tarea 1 y como estamos usando flaot64

    print(len(U), b.size)

    x = backwardSubstitution(U, len(U), b, t, dtype=dtype)

    if not x is None:
        show1D(x, len(x))
        error = np.linalg.norm(U*x-b.transpose()) #Norma de numpy
        print(f'Error =\n {error}')
    else:
        print("El sistema no tiene solucion.\nY es singular.")

if notebook:
    for sz in [5, 50, 500]:
        Ejercicio3('matU'+str(sz), 'vecb'+str(sz))
        print('\n')

if __name__ == "__main__" and not notebook:
    import doctest
    doctest.testmod()

    if len(sys.argv) >= 2:
        ejercicio = sys.argv[1].lower()

        if ejercicio == 'ejercicio2':
            Ejercicio2( sys.argv[2], sys.argv[3])
        elif ejercicio == 'ejercicio3':
            Ejercicio3( sys.argv[2], sys.argv[3])
        else:
            print(')= ... =(')

max_stars_repo_name: BenchHPZ/UG-Compu

max_issues_repo_name: BenchHPZ/UG-Compu

max_forks_repo_name: BenchHPZ/UG-Compu


text:
### Right now we only suppost csv.
import io
import os
import glob
import numpy as np
import pandas as pd
import json
import model
from flask import Flask, request, jsonify, render_template, send_file, Response,session
import flask
import csv
import json
from flask_session import Session

app = Flask(__name__)
app.config['subdomain_matching'] = 'covidseverity.com/hospitalization'
app.config['SEVER_NAME'] = 'covidseverity.com/hospitalization'
app.secret_key = os.urandom(24)
### Get format of input file.
def format(filename):
	if '.' not in filename:
		return '',''
	else:
		return filename.rsplit('.',1)[1].lower(),filename.rsplit('.',1)[0]

@app.route('/')
def home():
    return render_template('index.html')

@app.route('/predict',methods=['POST'])
def predict():
    def writename(name):
        text_file = open("name.txt","w")
        n = text_file.write(name)
        text_file.close()
    file = request.files['file']
    ### Check if there's input before processing.
    if not(file.filename):
    	flash('No selected file.')
    	return redirect(request.url)
    f,name = format(file.filename)
    if not(f):
    	flash('Wrong file format.')
    	return redirect(request.url)
    data = pd.read_csv(file).dropna()
    var = data.columns[1]
    ## Get prediction results
    fig, prediction, prediction_interval = model.predict(data, 14, var)
    ## Prepare dataframe to write
    prediction.append(round(sum(prediction),1))
    prediction_interval.append((round(sum([x[0] for x in prediction_interval],1)),
                                round(sum([x[1] for x in prediction_interval],1))))
    dic = {'Date': [i+1 for i in range(14)] +['sum'],
            data.columns[1]+' prediction': prediction,
            'prediction interval':prediction_interval}
    df = pd.DataFrame(dic, columns = ['Date',data.columns[1]+' prediction','prediction interval'])
    ## Write dataframe to file and save the file name in name.txt
    name = name + '_prediction.csv'
    df.round(1).to_csv(name,index=False)
    text_file = open("name.txt", "w")
    n = text_file.write(name)
    return render_template("predict.html", plotcode = fig)

@app.route('/download',methods=['POST'])
def download():
    f = open("name.txt", "r")
    return send_file(f.readline(), as_attachment = True)

if __name__ == "__main__":
    app.secret_key = 'A0Zr98j/3yX R~XHH!jmN]LWX/,?RT'
    app.run(debug=True, host='0.0.0.0')

max_stars_repo_name: Inklingdq/hospital-website

max_issues_repo_name: Inklingdq/hospital-website

max_forks_repo_name: Inklingdq/hospital-website


text:
from sklearn import metrics
from matplotlib.pylab import rcParams
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

# Data Collection and reading
data = pd.read_csv("OnlineNewsPopularity.csv")
data.head()

# Analysing data & finding dependent and independent variables
X = data.iloc[:, 1:60]
X.head()

corr = X.corr()
sns.heatmap(corr)
rcParams['figure.figsize'] = 20, 20

Y = data["shares"]
Y.head()

# Splitting data into training and testing
train_x, test_x, train_y, test_y = train_test_split(
    X, Y, random_state=5, test_size=0.20)

# Initializing model and training model
lin_model = LinearRegression()
lin_model.fit(train_x, train_y)

# Predict data
predicted = lin_model.predict(test_x)

# Mean Squared errors
metrics.mean_squared_error(predicted, test_y)

# Plotting predicted and test data
plt.scatter(predicted, test_y)
plt.show()

max_stars_repo_path: Final Project Assignments/case study 1/questions.py


text:
# Machine Learning model 3 (Test w/ some different MAC address + K.Arunan's Echo)

import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import itertools
from sklearn.preprocessing import LabelEncoder,OneHotEncoder
from sklearn.feature_extraction.text import CountVectorizer
from catboost import CatBoostClassifier,Pool
import os
import time
from sklearn.metrics import make_scorer, accuracy_score
from sklearn.model_selection import train_test_split,GridSearchCV

TRAIN_DATASET = os.path.join(os.path.dirname(os.path.abspath(__file__)), "model3_dataset", "model3_train_paper.csv")
TEST_DATASET = os.path.join(os.path.dirname(os.path.abspath(__file__)), "experiment_dns", "experiment_dns.csv")
MODEL_NAME = os.path.join(os.path.dirname(os.path.abspath(__file__)), "experiment_dns", "experiment_dns.cbm")

def plot_confusion_matrix(cm, classes,
                          normalize=False,
                          title='Confusion matrix',
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)
    fmt = '.4f' if normalize else 'd'
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]),   range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt),horizontalalignment="center",
            color="white" if cm[i, j] > thresh else "black")
    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')

# Train dataset
headernames = ['ipAddr', 'mac', 'portSrc', 'portDst', 'pktLength', 'deviceName', 'protocol', 'detail' ]
dataset = pd.read_csv(TRAIN_DATASET, names = headernames)
dataset.head()
X_train = dataset.drop(['deviceName'],axis=1)
X_train = X_train.drop(['detail'],axis=1)
y_train = dataset['deviceName']
#########################################################################
# Test dataset
testDataset = pd.read_csv(TEST_DATASET, names = headernames)
testDataset.head()
X_test = testDataset.drop(['deviceName'],axis=1)
X_test = X_test.drop(['detail'],axis=1)
y_test = testDataset['deviceName']

categorical_features_indices = np.where(X_train.dtypes != np.float)[0]
# IMPORTANT! : cannot use two CatBoost pool.
pool = Pool(X_train, y_train,cat_features=categorical_features_indices)

params = {
    'iterations': 100,
    'learning_rate': 0.59,
    'eval_metric': 'Accuracy',
    'l2_leaf_reg':3,
    'use_best_model': False
    }

classifier = CatBoostClassifier(**params)
start = time.time()
classifier.fit(pool)
stop = time.time()

start2 = time.time()
y_pred = classifier.predict(X_test)
stop2 = time.time()

from sklearn.metrics import classification_report, confusion_matrix, accuracy_score
cm = confusion_matrix(y_test, y_pred)
print("Confusion Matrix:")
print(cm)
report = classification_report(y_test, y_pred)
print("Classification Report:",)
print (report)
accuracy = accuracy_score(y_test,y_pred)
print("Accuracy:",accuracy)
traningTime = str(stop - start)
print("Training time: " + traningTime)
testingTime = str(stop2 - start2)
print("Testing time: " + testingTime)

feature_importances = classifier.get_feature_importance(pool)
feature_names = X_train.columns
for score, name in sorted(zip(feature_importances, feature_names), reverse=True):
    print('{}: {}'.format(name, score))

className = ["Device 0","Device 1","Device 2","Device 3","Device 4","Device 5","Device 6","Device 7"]
plt.figure()
plot_confusion_matrix(cm, classes=className,normalize=True,
                      title="DNS packets dataset's confusion matrix")
plt.show()

classifier.save_model(MODEL_NAME,format="cbm",export_parameters=None,pool=None)

max_stars_repo_name: donkyduck/ScottishFold-on-Edge

max_issues_repo_name: donkyduck/ScottishFold-on-Edge

max_forks_repo_name: donkyduck/ScottishFold-on-Edge


text:
# -*- coding: utf-8 -*-
from __future__ import division, absolute_import, print_function
from numpy.testing import assert_array_almost_equal
import numpy as np

from kpfm.lockin import LockIn, FIRStateLock

def test_lockin_even_fir():
    xb = np.array([-1, -np.sqrt(2)/2, 0, np.sqrt(2)/2,
                    1, np.sqrt(2)/2, 0, -np.sqrt(2)/2])
    repeats = 128
    N = xb.size * repeats
    x = np.tile(xb, repeats)
    t = np.arange(N)/16.0
    li = LockIn(t, x, fs=16.0)
    fir = np.ones(16)/16.0
    li.run(f0=2.0, fir=fir)
    li.manual_phase(0.0)
    assert_array_almost_equal(-np.ones_like(li("z"), dtype=np.complex128),
                              li("z"))

def test_lockin_odd_fir():
    xb = np.sin(2*np.pi/9 * np.arange(9))
    repeats = 96
    N = xb.size * repeats
    x = np.tile(xb, repeats)
    t = np.arange(N)/27.0
    li = LockIn(t, x, fs=27.0)
    fir = np.ones(27)/27.0
    li.run(f0=3.0, fir=fir)
    li.manual_phase(0.0)
    assert_array_almost_equal(-1j*np.ones_like(li("z"), dtype=np.complex128),
                              li("z"))

def test_FIRStateLock():
    t1 = np.arange(128, dtype=np.float64)
    t2 = np.arange(128, 256, dtype=np.float64)
    x1 = np.sin(np.pi/2*t1) * np.exp(-t1/200000.)
    x2 = np.sin(np.pi/2*t2) * np.exp(-t2/200000.)
    f0 = 0.25
    phi0 = np.pi/2
    fir = np.array([ -2.24576405e-21,   7.38375885e-07,   4.47985274e-06,
         1.04499910e-05,   9.53493909e-06,  -2.11536301e-05,
        -1.21463857e-04,  -3.45121941e-04,  -7.45930068e-04,
        -1.35036096e-03,  -2.11774624e-03,  -2.89530921e-03,
        -3.38156003e-03,  -3.11543482e-03,  -1.50742755e-03,
         2.07858743e-03,   8.19742309e-03,   1.71690331e-02,
         2.89335254e-02,   4.29551745e-02,   5.82110859e-02,
         7.32838205e-02,   8.65526725e-02,   9.64518343e-02,
         1.01743149e-01,   1.01743149e-01,   9.64518343e-02,
         8.65526725e-02,   7.32838205e-02,   5.82110859e-02,
         4.29551745e-02,   2.89335254e-02,   1.71690331e-02,
         8.19742309e-03,   2.07858743e-03,  -1.50742755e-03,
        -3.11543482e-03,  -3.38156003e-03,  -2.89530921e-03,
        -2.11774624e-03,  -1.35036096e-03,  -7.45930068e-04,
        -3.45121941e-04,  -1.21463857e-04,  -2.11536301e-05,
         9.53493909e-06,   1.04499910e-05,   4.47985274e-06,
         7.38375885e-07,  -2.24576405e-21])
    firstate = FIRStateLock(fir, 2, f0, phi0)
    firstate.filt(x1)
    firstate.filt(x2)
    t_out = firstate.get_t()
    A_out = abs(firstate.z_out)
    A_expected = np.exp(-t_out / 200000.)
    assert_array_almost_equal(A_out, A_expected, decimal=5)

def test_FIRStateLock_indices():
    x = np.array([0., 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15])
    h = np.array([0.25, 0.25, 0.25, 0.25])/2.
    dec = 2
    f0 = 0.
    fs = 1.
    firstate = FIRStateLock(h, 2, f0, 0., fs=1.)
    firstate.filt(x)
    exp_z = np.array([1.5, 3.5, 5.5, 7.5, 9.5, 11.5, 13.5])
    assert_array_almost_equal(firstate.z_out, exp_z)

max_stars_repo_name: marohngroup/kpfm

max_issues_repo_name: marohngroup/kpfm

max_forks_repo_name: marohngroup/kpfm

reason: import numpy,from numpy


text:
import os
import numpy as np
import shutil
import torch
from sklearn.metrics import average_precision_score
from datasets.cub import CUBDataset
from datasets.imagenet import ImageNetDataset
from datasets.pascal_voc import VOCDataset

def creat_data_loader(cfg, root_dir):
    print('==> Preparing data...')
    if cfg.DATA.DATASET == 'CUB':
        train_loader = torch.utils.data.DataLoader(
            CUBDataset(root=root_dir, cfg=cfg, is_train=True),
            batch_size=cfg.TRAIN.BATCH_SIZE, shuffle=True, num_workers=cfg.BASIC.NUM_WORKERS, pin_memory=True)
        val_loader = torch.utils.data.DataLoader(
            CUBDataset(root=root_dir, cfg=cfg, is_train=False),
            batch_size=cfg.TEST.BATCH_SIZE, shuffle=False, num_workers=cfg.BASIC.NUM_WORKERS, pin_memory=True)
    elif cfg.DATA.DATASET == 'ImageNet':
        train_loader = torch.utils.data.DataLoader(
            ImageNetDataset(root=root_dir, cfg=cfg, is_train=True),
            batch_size=cfg.TRAIN.BATCH_SIZE, shuffle=True, num_workers=cfg.BASIC.NUM_WORKERS, pin_memory=True)
        val_loader = torch.utils.data.DataLoader(
            ImageNetDataset(root=root_dir, cfg=cfg, is_train=False),
            batch_size=cfg.TEST.BATCH_SIZE, shuffle=False, num_workers=cfg.BASIC.NUM_WORKERS, pin_memory=True)
    elif cfg.DATA.DATASET == 'PascalVoc':
        train_loader = torch.utils.data.DataLoader(
            VOCDataset(root=root_dir, cfg=cfg, is_train=True),
            batch_size=cfg.TRAIN.BATCH_SIZE, shuffle=True, num_workers=cfg.BASIC.NUM_WORKERS, pin_memory=True)
        val_loader = torch.utils.data.DataLoader(
            VOCDataset(root=root_dir, cfg=cfg, is_train=False),
            batch_size=cfg.TEST.BATCH_SIZE, shuffle=False, num_workers=cfg.BASIC.NUM_WORKERS, pin_memory=True)

    print('done!')
    return train_loader, val_loader

def str_gpus(ids):
    str_ids = ''
    for id in ids:
        str_ids =  str_ids + str(id)
        str_ids =  str_ids + ','

    return str_ids

def map_sklearn(labels, results):
    map = average_precision_score(labels, results, average="micro")
    return map

def adjust_learning_rate(optimizer, epoch, cfg):
    """"Sets the learning rate to the initial LR decayed by lr_factor"""
    lr_decay = cfg.SOLVER.LR_FACTOR**(sum(epoch > np.array(cfg.SOLVER.LR_STEPS)))
    lr = cfg.SOLVER.START_LR * lr_decay
    for param_group in optimizer.param_groups:
        param_group['lr'] = lr * param_group['lr_mult']

def adjust_learning_rate_normal(optimizer, epoch, cfg):
    """Sets the learning rate to the initial LR decayed by 0.5 every 20 epochs"""
    e = False
    for step in cfg.SOLVER.LR_STEPS[::-1]:
        if epoch % step == 0:
            e = True
            break
    lr_decay = cfg.SOLVER.LR_FACTOR
    #lr_decay = cfg.SOLVER.LR_FACTOR ** (sum(epoch > np.array(cfg.SOLVER.LR_STEPS)))
    if e:
        for param_group in optimizer.param_groups:
            param_group['lr'] = lr_decay * param_group['lr']

def save_checkpoint(state, save_dir, epoch, is_best):
    filename = os.path.join(save_dir, 'ckpt_'+str(epoch)+'.pth.tar')
    torch.save(state, filename)
    if is_best:
        best_name = os.path.join(save_dir, 'model_best.pth.tar')
        shutil.copyfile(filename, best_name)

class AverageMeter(object):
    """Computes and stores the average and current value"""
    def __init__(self):
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count

def accuracy(output, target, topk=(1,)):
    """Computes the precision@k for the specified values of k"""
    maxk = max(topk)
    batch_size = target.size(0)

    _, pred = output.topk(maxk, 1, True, True)
    pred = pred.t()
    correct = pred.eq(target.view(1, -1).expand_as(pred))

    res = []
    for k in topk:
        correct_k = correct[:k].reshape(-1).float().sum(0)
        res.append(correct_k.mul_(100.0 / batch_size))
    return res

def list2acc(results_list):
    """
    :param results_list: list contains 0 and 1
    :return: accuarcy
    """
    accuarcy = results_list.count(1)/len(results_list)
    return accuarcy

max_stars_repo_name: vasgaowei/ts-cam-voc

max_issues_repo_name: vasgaowei/ts-cam-voc

max_forks_repo_name: vasgaowei/ts-cam-voc


text:
import os.path

import cv2
import matplotlib.pyplot as plt
import numpy as np
from keras import Input, regularizers
from keras.applications import vgg16
from keras.engine import Model
from keras.layers import Flatten, Dense, Lambda, Cropping2D, Dropout, ELU, Activation, MaxPooling2D, Conv2D, \
    BatchNormalization
from keras.models import Sequential, load_model
from keras.utils import plot_model

from data_loader import load_training_data_as_generator, generator, data_resampling, preprocessing_pipe

def test_model_on_images(model_file):
    model = load_model(model_file)
    center = cv2.imread("test_images/center.jpg")
    left = cv2.imread("test_images/left.jpg")
    right = cv2.imread("test_images/right.jpg")

    print("Expected Steering : 0.0")

    images = [left, center, right]
    label = ["Left", "Center", "Right"]
    for pos, image in enumerate(images):
        this_image = preprocessing_pipe(image=center, img_rescale=100)
        image_array = np.asarray(this_image)
        steering_angle = float(model.predict(image_array[None, :, :, :], batch_size=1))
        print(label[pos] + " : " + str(steering_angle))

def vgg_net(input_shape):
    """
    Vgg net references
    - [Very Deep Convolutional Networks for Large-Scale Image Recognition](https://arxiv.org/abs/1409.1556)
    """
    model = Sequential()

    # Use batch normalization to speed up process
    model.add(BatchNormalization(input_shape=input_shape))

    # Smaller VGG Net (Blocks from VGG 16 with smaller filter depth)
    # 2 Conv, 3 Conv and 2 Conv block
    # Filter depth 8 -> 16 -> 32 (vgg16 64 --> 128 --> 256 --> 512 --> 512)
    # Dense 2048 -> 100 -> 10 --> 1 (vgg16 4096 --> 4096 --> classes)

    # Block 1
    model.add(Conv2D(8, (3, 3), padding='valid', strides=(1, 1), activation='relu', name='Conv 1-1'))
    model.add(Conv2D(16, (3, 3), padding='valid', strides=(1, 1), activation='relu', name='Conv 1-2'))
    model.add(MaxPooling2D(pool_size=(2, 2)))

    # Block 2
    # Dropout 0.2 (0.5 worse results)
    model.add(Conv2D(16, (3, 3), padding='valid', strides=(1, 1), activation='relu', name='Conv 2-1'))
    model.add(Dropout(0.2))
    model.add(Conv2D(32, (3, 3), padding='valid', strides=(1, 1), activation='relu', name='Conv 2-2'))
    model.add(Dropout(0.2))
    model.add(Conv2D(32, (3, 3), padding='valid', strides=(1, 1), activation='relu', name='Conv 2-3'))
    model.add(Dropout(0.2))
    model.add(MaxPooling2D(pool_size=(2, 2)))

    # Block 3
    model.add(Conv2D(64, (3, 3), padding='valid', strides=(1, 1), activation='relu', name='Conv 3-1'))
    model.add(Dropout(0.2))
    model.add(Conv2D(64, (3, 3), padding='valid', strides=(1, 1), activation='relu', name='Conv 3-2'))
    model.add(Dropout(0.2))
    model.add(MaxPooling2D(pool_size=(2, 2)))

    # Dense Block
    # Relu activation (ELU with no better results) --> nvidia_net_not_working()
    model.add(Flatten())
    model.add(Dense(2048))
    model.add(Activation(activation='relu'))
    model.add(Dense(100))
    model.add(Activation(activation='relu'))
    model.add(Dense(10))
    model.add(Activation(activation='relu'))
    model.add(Dense(1))

    print(model.summary())
    plot_model(model, to_file='model_plot.png', show_shapes=True, show_layer_names=True)
    return model

def nvidia_net_not_working(row, col, ch):
    """
    Try to use keras vgg net with own fc layers, but results (steering from prediction) were the same for different input images
    """
    input_tensor = Input(shape=(row, col, ch))
    model_vgg16 = vgg16.VGG16(include_top=False,
                              input_tensor=input_tensor)
    x = Flatten()(model_vgg16.output)
    x = Dense(1164)(x)
    x = ELU()(x)
    x = Dropout(0.5)(x)
    x = Dense(100)(x)
    x = ELU()(x)
    x = Dropout(0.5)(x)
    x = Dense(50)(x)
    x = ELU()(x)
    x = Dense(10)(x)
    x = ELU()(x)
    x = Dropout(0.5)(x)
    x = Dense(1)(x)
    # create graph of your new model
    head_model = Model(input=model_vgg16.input, output=x)
    print(head_model.summary())
    plot_model(head_model, to_file='model_plot.png', show_shapes=True, show_layer_names=True)
    return head_model

def test_model(ch, row, col):
    """
    Test the model on three images (center/left/right) and predict the steering
    """
    model = Sequential()
    model.add(Lambda(lambda x: x / 127.5 - 1.,
                     input_shape=(row, col, ch),
                     output_shape=(row, col, ch)))
    model.add(Cropping2D(cropping=((70, 25), (0, 0))))
    model.add(Flatten())
    model.add(Dense(1))
    return model

def print_summary(history_object):
    # plot the training and validation loss for each epoch
    plt.plot(history_object.history['loss'])
    plt.plot(history_object.history['val_loss'])
    plt.title('model mean squared error loss')
    plt.ylabel('mean squared error loss')
    plt.xlabel('epoch')
    plt.legend(['training set', 'validation set'], loc='upper right')
    plt.savefig("model_summary.png")
    plt.show()

def main_loop(data_path, img_folder="IMG"):
    image_rescale = 100
    data_path = data_path.replace("/", os.sep).replace("\\", os.sep)
    data_img_dir = os.path.join(data_path.rsplit(os.sep, 1)[0], img_folder)
    train_samples, validation_samples = load_training_data_as_generator(data_path)
    print("Number of training data {}".format(len(train_samples)))
    print("Number of validation data {}".format(len(validation_samples)))

    batch_size = 64

    # compile and train the model using the generator function
    train_generator = generator(train_samples, batch_size=batch_size,
                                image_folder=data_img_dir, img_rescale=image_rescale)
    validation_generator = generator(validation_samples, batch_size=batch_size,
                                     image_folder=data_img_dir, img_rescale=image_rescale)

    ch = 3
    row = int(50 * (image_rescale / 100))
    col = int(320 * (image_rescale / 100))

    model = vgg_net((row, col, ch))

    model.compile(loss='mse', optimizer='adam')
    history_object = model.fit_generator(train_generator,
                                         steps_per_epoch=int(len(train_samples) / batch_size),
                                         validation_data=validation_generator,
                                         validation_steps=int(len(validation_samples) / batch_size),
                                         epochs=10,
                                         verbose=1)
    print("Save model")
    model.save("model.h5")
    print("Done, print summary to file")
    print_summary(history_object)
    print("Finished")

if __name__ == "__main__":
    test = False
    datagen = False
    train = True

    if test:
        test_model_on_images("model_old.h5")
    if datagen:
        data_resampling("./data/track_1/driving_log.csv")
    if train:
        main_loop("./data/track_1/driving_log_to_center_only.csv")


text:
import pycuda.autoinit
import pycuda.driver as drv
import numpy as np
from pycuda.compiler import SourceModule
import time

src = """
#if %(use_double)d
#define FLOAT double
#define IFLOAT unsigned long long int
#define F2I __double_as_longlong
#define I2F __longlong_as_double
#else
#define FLOAT float
#define IFLOAT unsigned int
#define F2I __float_as_int
#define I2F __int_as_float
#endif

// Problem size
#define N %(N)d
// Block size
#define Nb %(Nb)d
// Flattened block size
#define Nb2 %(Nb2)d
// Block data size
#define Nbd %(Nbd)d
// Number of memory writes per thread
#define Nwrite %(Nwrite)d

__device__ FLOAT atomicAddF(FLOAT* address, FLOAT val)
{
    IFLOAT* address_as_ull = (IFLOAT*)address;
    IFLOAT old = *address_as_ull, assumed;
    do {
        assumed = old;
        old = atomicCAS(
            address_as_ull, assumed,
            F2I(val + I2F(assumed))
        );
    } while (assumed != old);
    return I2F(old);
}

__device__ __constant__ FLOAT wsr[N];
__device__ __constant__ FLOAT wsi[N];

__global__ void kernel(FLOAT *global_out_data, FLOAT *global_in_data) {
    const int nb = threadIdx.x;            // X position in block
    const int n = nb + Nb*blockIdx.x;      // X position in full grid
    const int k = blockIdx.y;
    const int j = nb;
    const int m_max = min(n+1, N-n);
    int m, p;
    FLOAT Wnkr, Wnki;
    FLOAT dWnpmkr, dWnpmki;
    FLOAT dWnmmkr, dWnmmki;
    FLOAT Wnk_angle_r, Wnk_angle_i;
    FLOAT wkmr, wkmi;
    FLOAT znpmr, znpmi, zcnmmr, zcnmmi;
    FLOAT Knmr, Knmi;
    FLOAT cnk, bd;

    __shared__ FLOAT block_out_data[Nbd];
    FLOAT *block_cost = block_out_data;
    FLOAT *block_grad_r = block_out_data + 1;
    FLOAT *block_grad_i = block_out_data + (N+1);

    FLOAT *zr = global_in_data;
    FLOAT *zi = global_in_data + N;

    for (m=0; m<Nwrite; m++) {
        p = j + m*Nb2;
        if (p < Nbd)
            block_out_data[p] = 0;
    }
    __syncthreads();

    if (n < N && k < N) {
        Wnkr = 0;
        Wnki = 0;
        for (m=0; m<m_max; m++) {
            wkmr = wsr[k*m %% N];
            wkmi = wsi[k*m %% N];
            znpmr = zr[n+m];
            znpmi = zi[n+m];
            zcnmmr = zr[n-m];
            zcnmmi = -zi[n-m];
            Knmr = znpmr*zcnmmr - znpmi*zcnmmi;
            Knmi = znpmi*zcnmmr + znpmr*zcnmmi;
            Wnkr += wkmr*Knmr - wkmi*Knmi;
            Wnki += wkmi*Knmr + wkmr*Knmi;
        }

        cnk = sqrt(Wnkr*Wnkr + Wnki*Wnki);
        atomicAddF(block_cost, cnk);

        if (cnk > 1e-7) {
            Wnk_angle_r = Wnkr / cnk;
            Wnk_angle_i = -Wnki / cnk;
            for (m=0; m<m_max; m++) {
                wkmr = wsr[k*m %% N];
                wkmi = wsi[k*m %% N];
                znpmr = zr[n+m];
                znpmi = zi[n+m];
                zcnmmr = zr[n-m];
                zcnmmi = -zi[n-m];
                dWnpmkr = wkmr*zcnmmr - wkmi*zcnmmi;
                dWnpmki = wkmr*zcnmmi + wkmi*zcnmmr;
                dWnmmkr = wkmr*znpmr - wkmi*znpmi;
                dWnmmki = wkmr*znpmi + wkmi*znpmr;
                atomicAddF(block_grad_r + (n+m), Wnk_angle_r*dWnpmkr - Wnk_angle_i*dWnpmki);
                atomicAddF(block_grad_i + (n+m), -(Wnk_angle_r*dWnpmki + Wnk_angle_i*dWnpmkr));
                atomicAddF(block_grad_r + (n-m), Wnk_angle_r*dWnmmkr - Wnk_angle_i*dWnmmki);
                atomicAddF(block_grad_i + (n-m), Wnk_angle_r*dWnmmki + Wnk_angle_i*dWnmmkr);
            }
        }
    }

    __syncthreads();

    for (m=0; m<Nwrite; m++) {
        p = j + m*Nb2;
        if (p < Nbd) {
            bd = block_out_data[p];
            if (bd != 0)
                atomicAddF(global_out_data + p, bd);
        }
    }

}
"""

def l1_wvd(N, block_size=16, use_double=False):
    Nb = block_size
    Ng = int(np.ceil(float(N) / Nb))
    Nb2 = Nb
    Nbd = 2*N + 1
    Nwrite = int(np.ceil(float(Nbd) / Nb2))
    cfg = dict(N=N, Ng=Ng, Nb=Nb, Nb2=Nb2, Nbd=Nbd, Nwrite=Nwrite, use_double=use_double)
    mod = SourceModule(src % cfg)
    kernel = mod.get_function('kernel')
    dtype = np.float64 if use_double else np.float32
    w0 = np.exp(-2j * np.pi / N)
    ws = np.array([w0**k for k in range(N)])
    wsr_cmem = mod.get_global('wsr')[0]
    wsi_cmem = mod.get_global('wsi')[0]
    drv.memcpy_htod(wsr_cmem, np.ascontiguousarray(ws.real.astype(dtype)))
    drv.memcpy_htod(wsi_cmem, np.ascontiguousarray(ws.imag.astype(dtype)))

    def run(z, zi=None):
        if zi is None:
            indata = np.concatenate([z.real, z.imag])
        else:
            indata = np.concatenate([z, zi])
        indata = drv.In(np.ascontiguousarray(indata.astype(dtype)))
        outdata = drv.InOut(np.zeros(Nbd, dtype=dtype))
        kernel(outdata, indata, block=(Nb, 1, 1), grid=(Ng, N))
        cost = outdata.array[0]
        grad_r = outdata.array[1:N+1]
        grad_i = outdata.array[N+1:]
        return cost, grad_r, grad_i

    return run

if __name__ == '__main__':
    import sys
    N = int(sys.argv[2])
    Nb = int(sys.argv[1])
    i = int(sys.argv[3])
    eps = float(sys.argv[4])
    runner = l1_wvd(N, Nb)
    np.random.seed(5432)
    z = np.random.randn(N) + 1j*np.random.randn(N)
    print('warming up?', end=' ')
    for _ in range(3):
        c, dcr, dci = runner(z)
        print('.', end=' ')
    print('result =', c)
    print('calling...', end=' ')
    t0 = time.time()
    c1, dcr, dci = runner(z)
    t1 = time.time() - t0
    print('finished:', t1)
    dz = np.eye(N)[i]
    c2, _, _ = runner(z + eps*dz)
    c3, _, _ = runner(z + 1j*eps*dz)
    dcr2 = (c2 - c1) / eps
    dci2 = (c3 - c1) / eps
    print('grad re: calc =', dcr[i], 'appx =', dcr2)
    print('grad im: calc =', dci[i], 'appx =', dci2)
    drv.stop_profiler()

max_stars_repo_name: rayonde/yarn

max_issues_repo_name: rayonde/yarn

max_forks_repo_name: rayonde/yarn

reason: import numpy,import pycuda,from pycuda


text:
# -*- coding: utf-8 -*-
"""
Various utility functions
"""
import numpy as np
from collections import OrderedDict

def match_score(pi, pj, prefix_weight=0.1, middle_weight=0.8, suffix_weight=0.1):
    """ Compute match between phrases using a dot product of vectors
    :param pi Phrase or pattern
    :param pj phrase or pattern
    # add weights to dot products to put more emphasis on matching the middles
    """
    assert(pi.keys() == pj.keys())

    number_of_middles = len([i for i in pi.keys() if i.startswith('middle')])

    prefix_i = pi['prefix']

    middles_i = []
    for m in range(0, number_of_middles):
        middles_i.append(pi['middle_' + str(m+1)])

    suffix_i = pi['suffix']

    prefix_j = pj['prefix']

    middles_j = []
    for m in range(0, number_of_middles):
        middles_j.append(pj['middle_' + str(m+1)])

    suffix_j = pj['suffix']

    if len(prefix_i) == 0 and len(prefix_j) == 0:
        prefix_dot_prod = 1.0
    else:
        prefix_dot_prod = np.dot(prefix_i, prefix_j)

    middle_dot_prod = []
    for i in range(0, len(middles_i)):
        if len(middles_i[i]) == 0 and len(middles_j[i]) == 0:
            middle_dot_prod.append(1.0)
        elif len(middles_i[i]) != len(middles_j[i]):
            middle_dot_prod.append(0)
        else:
            m_dot = np.dot(middles_i[i], middles_j[i])
            middle_dot_prod.append(m_dot)

    if len(suffix_i) == 0 and len(suffix_j) == 0:
        suffix_dot_prod = 1.0
    else:
        suffix_dot_prod = np.dot(suffix_i, suffix_j)

    sim = (prefix_weight * prefix_dot_prod) + ((middle_weight/number_of_middles)*sum(middle_dot_prod)) + (suffix_weight * suffix_dot_prod)
    return sim

def vectorise(phrase, cluster):
    """Vectorise a phrase object against a given cluster

    Arguments:
        phrase {[type]} -- [description]
        cluster {[type]} -- [description]
    """
    # print("Vectorising phrase ", phrase)
    # print("Pattern", cluster.pattern)
    phrase_element_vectors = {}
    pattern_element_vectors = {}
    pattern = cluster.pattern

    for element in cluster.dictionaries.keys():  # prefix, middles, suffix
        # print("element", element, '\n')
        local_dictionary = OrderedDict()

        # fill the local dictionary with the cluster tokens
        for token in cluster.dictionaries[element]['token dict']:
            if token in local_dictionary.keys():
                local_dictionary[token] += 1
            else:
                local_dictionary[token] = cluster.dictionaries[element]['token dict'][token][0]

        # Same for the phrase tokens
        for token in phrase.elements[element]['tokens']:
            if token in local_dictionary.keys():
                local_dictionary[token] += 1
            else:
                local_dictionary[token] = 1
        # print(local_dictionary, '\n')

        phrase_element_vector = np.zeros(len(local_dictionary.keys()))
        pattern_element_vector = np.zeros(len(local_dictionary.keys()))
        # Now vectorise the phrase and pattern
        for i, token in enumerate(local_dictionary.keys()):
            if token in phrase.elements[element]['tokens']:
                phrase_element_vector[i] += 1
            if token in pattern.elements[element]['tokens']:
                pattern_element_vector[i] += 1

        # normalise the vectors
        phrase_element_vectors[element] = phrase_element_vector / np.linalg.norm(phrase_element_vector)
        pattern_element_vectors[element] = pattern_element_vector / np.linalg.norm(pattern_element_vector)

    # print(phrase_element_vectors)
    # print(pattern_element_vectors)
    return phrase_element_vectors, pattern_element_vectors

def match(phrase, cluster, prefix_weight, middles_weight, suffix_weight):
    """Vectorise the phrase against this cluster to determine the match score

    Arguments:
        phrase {[type]} -- [description]
        cluster {[type]} -- [description]
    """
    if phrase.order != cluster.order:
        return 0

    phrase_vectors, pattern_vectors = vectorise(phrase, cluster)
    score = match_score(phrase_vectors, pattern_vectors, prefix_weight, middles_weight, suffix_weight)
    # print(score)
    return score

def mode_rows(a):
    """
    Find the modal row of a 2d array
    :param a: The 2d array to process
    :type a: np.array()
    :return: The most frequent row
    """
    a = np.ascontiguousarray(a)
    void_dt = np.dtype((np.void, a.dtype.itemsize * np.prod(a.shape[1:])))
    _, ids, count = np.unique(a.view(void_dt).ravel(),
                              return_index=True,
                              return_counts=True)
    largest_count_id = ids[count.argmax()]
    most_frequent_row = a[largest_count_id]
    return most_frequent_row

def KnuthMorrisPratt(text, pattern):
    """Yields all starting positions of copies of the pattern in the text.
        Calling conventions are similar to string.find, but its arguments can be
        lists or iterators, not just strings, it returns all matches, not just
        the first one, and it does not need the whole text in memory at once.
        Whenever it yields, it will have read the text exactly up to and including
        the match that caused the yield.

    Source: http://code.activestate.com/recipes/117214/"""

    # allow indexing into pattern and protect against change during yield
    pattern = list(pattern)

    # build table of shift amounts
    shifts = [1] * (len(pattern) + 1)
    shift = 1
    for pos in range(len(pattern)):
        while shift <= pos and pattern[pos] != pattern[pos-shift]:
            shift += shifts[pos-shift]
        shifts[pos+1] = shift

    # do the actual search
    startPos = 0
    matchLen = 0
    for c in text:
        while matchLen == len(pattern) or \
              matchLen >= 0 and pattern[matchLen] != c:
            startPos += shifts[matchLen]
            matchLen -= shifts[matchLen]
        matchLen += 1
        if matchLen == len(pattern):
            yield startPos

def subfinder(mylist, pattern):
    for i in range(len(mylist)):
        if mylist[i] == pattern[0] and mylist[i:i+len(pattern)] == pattern:
            return (i, i+len(pattern))
    return None, None


text:
from enum import IntEnum

from typing import List

import numpy as np

from . import Base
from . import ByteIO

class FlexOpType(IntEnum):
    CONST = 1
    FETCH1 = 2
    FETCH2 = 3
    ADD = 4
    SUB = 5
    MUL = 6
    DIV = 7
    NEG = 8
    EXP = 9
    OPEN = 10
    CLOSE = 11
    COMMA = 12
    MAX = 13
    MIN = 14
    TWO_WAY_0 = 15
    TWO_WAY_1 = 16
    NWAY = 17
    COMBO = 18
    DOMINATE = 19
    DME_LOWER_EYELID = 20
    DME_UPPER_EYELID = 21

class FlexControllerRemapType(IntEnum):
    PASSTHRU = 0
    # Control 0 -> ramps from 1-0 from 0->0.5. Control 1 -> ramps from 0-1
    # from 0.5->1
    TWO_WAY = 1
    # StepSize = 1 / (control count-1) Control n -> ramps from 0-1-0 from
    # (n-1)*StepSize to n*StepSize to (n+1)*StepSize. A second control is
    # needed to specify amount to use
    NWAY = 2
    EYELID = 3

class FlexController(Base):
    def __init__(self):
        self.name = ''
        self.type = ''
        self.local_to_global = 0
        self.min = 0.0
        self.max = 0.0

    def read(self, reader: ByteIO):
        entry = reader.tell()
        self.type = reader.read_source1_string(entry)
        self.name = reader.read_source1_string(entry)
        self.local_to_global = reader.read_int32()
        self.min, self.max = reader.read_fmt('2f')

    def __repr__(self):
        return f'<FlexController "{self.name}" {self.min}:{self.max}>'

class FlexControllerUI(Base):
    def __init__(self):
        self.name = 0
        self.controller = ''
        self.left_controller = ''
        self.right_controller = ''
        self.nway_controller = ''
        self.remap_type = FlexControllerRemapType(0)
        self.stereo = False
        self.unused = []

    def read(self, reader: ByteIO):
        entry = reader.tell()
        self.name = reader.read_source1_string(entry)
        # TODO: https://github.com/Dmillz89/SourceSDK2013/blob/master/mp/src/public/studio.h#L924
        index0, index1, index2 = reader.read_fmt('3i')
        self.remap_type = FlexControllerRemapType(reader.read_uint8())
        self.stereo = reader.read_uint8()
        reader.skip(2)
        with reader.save_current_pos():
            if self.remap_type == FlexControllerRemapType.NWAY:
                reader.seek(entry + index2)
                reader.skip(4)
                self.nway_controller = reader.read_source1_string(entry + index2)
            elif self.remap_type == FlexControllerRemapType.EYELID:
                reader.seek(entry + index2)
                reader.skip(4)
                self.nway_controller = reader.read_source1_string(entry + index2)

            if not self.stereo:
                reader.seek(entry + index0)
                reader.skip(4)
                self.controller = reader.read_source1_string(entry + index0)
            elif self.stereo:
                reader.seek(entry + index0)
                reader.skip(4)
                self.left_controller = reader.read_source1_string(entry + index0)

                reader.seek(entry + index1)
                reader.skip(4)
                self.right_controller = reader.read_source1_string(entry + index1)
            else:
                raise RuntimeError('Should never reach this')
        pass

    def __repr__(self):
        return f'<FlexControllerUI "{self.name}">'

class FlexRule(Base):
    def __init__(self):
        self.flex_index = 0
        self.flex_ops = []  # type:List[FlexOp]

    def read(self, reader: ByteIO):
        entry = reader.tell()
        self.flex_index = reader.read_uint32()
        op_count = reader.read_uint32()
        op_offset = reader.read_uint32()
        with reader.save_current_pos():
            if op_count > 0 and op_offset != 0:
                reader.seek(entry + op_offset)
                for _ in range(op_count):
                    flex_op = FlexOp()
                    flex_op.read(reader)
                    self.flex_ops.append(flex_op)

class FlexOp(Base):

    def __init__(self):
        self.op = FlexOpType
        self.index = 0
        self.value = 0

    def read(self, reader: ByteIO):
        self.op = FlexOpType(reader.read_uint32())
        if self.op == FlexOpType.CONST:
            self.value = reader.read_float()
        else:
            self.index = reader.read_uint32()

    def __repr__(self):
        return f"FlexOp({self.op.name} {self.value if self.op == FlexOpType.CONST else self.index})"

class VertexAminationType(IntEnum):
    NORMAL = 0
    WRINKLE = 1

class FlexV36(Base):
    def __init__(self):
        self.name = ''
        self.flex_desc_index = 0
        self.targets = [0.0]

        self.partner_index = 0
        self.vertex_anim_type = 0
        self.vertex_animations = np.array([])  # type:np.ndarray

    def __repr__(self) -> str:
        return f'<Flex "{self.name}">'

    def __eq__(self, other: 'FlexV36'):
        return self.flex_desc_index == other.flex_desc_index and self.targets == other.targets

    def __hash__(self):
        return hash(self.flex_desc_index) + hash(self.targets)

    def read(self, reader: ByteIO):
        entry = reader.tell()
        self.flex_desc_index = reader.read_uint32()
        self.name = self.get_value('MDL').flex_names[self.flex_desc_index]

        self.targets = reader.read_fmt('4f')
        vert_count, vert_offset = reader.read_fmt('2I')

        if vert_count > 0 and vert_offset != 0:
            with reader.save_current_pos():
                reader.seek(entry + vert_offset)
                vert_anim_class = VertAnimV36

                self.vertex_animations = np.frombuffer(reader.read(vert_count * vert_anim_class.dtype.itemsize),
                                                       vert_anim_class.dtype)

class FlexV49(FlexV36):
    def read(self, reader: ByteIO):
        entry = reader.tell()
        self.flex_desc_index = reader.read_uint32()
        self.name = self.get_value('MDL').flex_names[self.flex_desc_index]

        self.targets = reader.read_fmt('4f')
        vert_count, vert_offset, self.partner_index = reader.read_fmt('3I')

        self.vertex_anim_type = reader.read_uint8()
        reader.skip(3)
        reader.skip(6 * 4)

        if vert_count > 0 and vert_offset != 0:
            with reader.save_current_pos():
                reader.seek(entry + vert_offset)
                if self.vertex_anim_type == VertexAminationType.WRINKLE:
                    vert_anim_class = VertAnimWrinkleV49
                else:
                    vert_anim_class = VertAnimV49

                self.vertex_animations = np.frombuffer(reader.read(vert_count * vert_anim_class.dtype.itemsize),
                                                       vert_anim_class.dtype)

class VertAnimV49(Base):
    vert_anim_fixed_point_scale = 1 / 4096
    is_wrinkle = False
    dtype = np.dtype(
        [
            ('index', np.uint16, (1,)),
            ('speed', np.uint8, (1,)),
            ('side', np.uint8, (1,)),
            ('vertex_delta', np.float16, (3,)),
            ('normal_delta', np.float16, (3,)),
        ]
    )

class VertAnimWrinkleV49(VertAnimV49):
    is_wrinkle = True
    dtype = np.dtype(
        [
            ('index', np.uint16, (1,)),
            ('speed', np.uint8, (1,)),
            ('side', np.uint8, (1,)),
            ('vertex_delta', np.float16, (3,)),
            ('normal_delta', np.float16, (3,)),
            ('wrinkle_delta', np.float16, (1,)),
        ]
    )

class VertAnimV36(Base):
    vert_anim_fixed_point_scale = 1 / 4096
    is_wrinkle = False
    dtype = np.dtype(
        [
            ('index', np.uint32, (1,)),
            ('vertex_delta', np.float32, (3,)),
            ('normal_delta', np.float32, (3,)),
        ]
    )


text:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

import pyrenn as prn

###
#Read Example Data
df = pd.ExcelFile('example_data.xlsx').parse('compressed_air')
P = np.array([df['P1'].values,df['P2'].values,df['P3'].values])
Y = np.array([df['Y1'].values,df['Y2']])
Ptest_ = np.array([df['P1test'].values,df['P2test'].values,df['P3test'].values])
Ytest_ = np.array([df['Y1test'].values,df['Y2test']])

#define the first timestep t=0 of Test Data as previous (known) data P0test and Y0test
P0test = Ptest_[:,0:1]
Y0test = Ytest_[:,0:1]
#Use the timesteps t = [1..99] as Test Data
Ptest = Ptest_[:,1:100]
Ytest = Ytest_[:,1:100]

###
#Create and train NN

#create feed forward neural network with 1 input, 2 hidden layers with
#4 neurons each and 1 output
#the NN has a recurrent connection with delay of 1 timesteps from the output
# to the first layer
net = prn.CreateNN([3,5,5,2],dIn=[0],dIntern=[],dOut=[1])

#Train NN with training data P=input and Y=target
#Set maximum number of iterations k_max to 500
#Set termination condition for Error E_stop to 1e-5
#The Training will stop after 500 iterations or when the Error <=E_stop
prn.train_LM(P,Y,net,verbose=True,k_max=500,E_stop=1e-5)

###
#Calculate outputs of the trained NN for test data with and without previous input P0 and output Y0
ytest = prn.NNOut(Ptest,net)
y0test = prn.NNOut(Ptest,net,P0=P0test,Y0=Y0test)

###
#Plot results
fig = plt.figure(figsize=(15,10))
ax0 = fig.add_subplot(211)
ax1 = fig.add_subplot(212,sharey=ax0)
fs=18

t = np.arange(0,np.shape(Ptest)[1])/4.0 #timesteps in 15 Minute resolution

#Test Data
ax0.set_title('Test Data',fontsize=fs)
ax0.plot(t,ytest[0],color='b',lw=2,label='NN Output without P0,Y0')
ax0.plot(t,y0test[0],color='g',lw=2,label='NN Output with P0,Y0')
ax0.plot(t,Ytest[0],color='r',marker='None',linestyle=':',lw=3,markersize=8,label='Test Data')
ax0.tick_params(labelsize=fs-2)
ax0.legend(fontsize=fs-2,loc='upper right')
ax0.grid()
plt.setp(ax0.get_xticklabels(), visible=False)
ax0.set_ylabel('Storage Pressure [bar]',fontsize=fs)

ax1.plot(t,ytest[1],color='b',lw=2,label='NN Output without P0,Y0')
ax1.plot(t,y0test[1],color='g',lw=2,label='NN Output with P0,Y0')
ax1.plot(t,Ytest[1],color='r',marker='None',linestyle=':',lw=3,markersize=8,label='Test Data')
ax1.tick_params(labelsize=fs-2)
ax1.grid()
ax1.set_xlabel('Time [h]',fontsize=fs)
ax1.set_ylabel('el. Power [kW]',fontsize=fs)

fig.tight_layout()
plt.show()


text:
#!/usr/bin/env python

import numpy

import chainer
import chainer.functions as F
import chainer.links as L

#from ch2o.test_args import dprint

class LinkInFor(chainer.Chain):

    def __init__(self, num_hidden):
        super(LinkInFor, self).__init__()
        with self.init_scope():
            self.l = L.Linear(num_hidden, num_hidden)

    def forward(self, x, h, indices):
        for i in indices:
            h = h + self.l(x[:, i])
        return h

import testtools
import numpy as np

def main():
    np.random.seed(314)

    batch_size = 3
    num_hidden = 5
    sequence_length = 4

    model = LinkInFor(num_hidden)

    x = np.random.rand(
        batch_size, sequence_length, num_hidden).astype(np.float32)
    h = np.random.rand(batch_size, num_hidden).astype(np.float32)

    args = [x, h, np.arange(sequence_length)]
    #dprint(model(*args))
    testtools.generate_testcase(model, args)

if __name__ == '__main__':
    main()

max_stars_repo_name: disktnk/chainer-compiler

max_issues_repo_name: disktnk/chainer-compiler

max_forks_repo_name: disktnk/chainer-compiler


text:
from requests_html import HTMLSession
from requests.exceptions import ConnectionError
from retry import retry
from typing import List
from time import sleep
import pandas as pd
from numpy import nan
import pickle
from logging import getLogger, StreamHandler, Formatter, DEBUG
logger = getLogger(__name__)
logger.setLevel(DEBUG)
ch = StreamHandler()
ch.setLevel(DEBUG)
formatter = Formatter('%(asctime)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)
logger.propagate = False

def get_articles(base_domain: str, kind: str) -> List[str]:
    base_url = base_domain + kind
    session = HTMLSession()
    r = session.get(base_url + ".html")
    all_articles = pd.Series(list(r.html.links), name=kind)
    articles = all_articles[all_articles.str.startswith("20")]
    logger.debug("Checked: %s", base_url)
    sleep(10)
    r = session.get(base_url + "_list.html")
    all_art_list = pd.Series(list(r.html.links))
    art_list = all_art_list[all_art_list.str.startswith("20")]
    logger.debug("Checked: %s", base_url + "_list.html")
    sleep(10)
    ret_articles = pd.concat([articles, art_list], ignore_index=True, names=[kind])
    return ret_articles

@retry()
def get_description(base_domain: str, code: str, kind: str) -> pd.core.series.Series:
    session = HTMLSession()
    url = base_domain + code
    r = session.get(url)
    sleep(10)
    try:
        raw_title = r.html.find('title', first=True).text
        raw_article = r.html.find('article', first=True).text
        data_row = ["".join(a_text.replace("これは嘘ニュースです", "").replace("\n\n\n新しいアプリで記事を読む", "").split("\n"))
            for a_text in [raw_title, raw_article]]
    except AttributeError:
        data_row = ["404", "Not Found"]
    data = [code, kind] + data_row
    data_sr = pd.Series(data, index=["code", "kind", "title", "article"])
    return data_sr

def main():
    base_domain = "http://kyoko-np.net/"
    try:
        urls = pickle.load(open("urls.pickle", "rb"))
    except (OSError, IOError):
        kinds = ["national", "politics", "business", "sport",
                 "international", "science", "culture", "entertainment"]
        urls = pd.concat(
            [get_articles(base_domain, kind) for kind in kinds],
            axis=1, names=kinds
        )
        urls.columns = kinds
        pickle.dump(urls, open("urls.pickle", "wb"))
    df = pd.DataFrame(columns=["code", "kind", "title", "article"])
    for kind_name in urls:
        raw_kind_urls = urls[kind_name]
        kind_urls = raw_kind_urls[raw_kind_urls.apply(
            lambda x: isinstance(x, str))]
        total = len(kind_urls)
        for i, code in enumerate(kind_urls):
            data_sr = get_description(base_domain, code, kind_urls.name)
            df = df.append(data_sr, ignore_index=True)
            logger.debug(
                "%d / %d done: %s, Category:%s", i, total, code, kind_name
            )

    df.to_csv("dataset.csv.gz", sep='|', header=True, index=False, chunksize=1000, compression='gzip', encoding='utf-8')

if __name__ == '__main__':
    main()

reason: from numpy


text:
"""
Copyright 2016 Deepgram

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import logging
import numpy
from . import EvaluationHook

logger = logging.getLogger(__name__)

###############################################################################
class MnistHook(EvaluationHook):
	""" Post-evaluation hook for MNIST data, which prints summary statistics
		specific to the MNIST data set.
	"""

	###########################################################################
	@classmethod
	def get_name(cls):
		""" Returns the name of the evaluation hook.
		"""
		return 'mnist'

	###########################################################################
	def apply(self, current, original, model=None):
		""" Applies the hook to the data.
		"""
		data, truth = original
		if truth is None:
			logger.warning('MNIST hook only works with ground-truth data. We '
				'will skip this hook.')
			return current

		stats = {
			i : {'correct' : 0, 'total' : 0}
			for i in range(10)
		}

		if len(data) == 1:
			for key in data:
				pass
		elif len(data) > 1:
			if 'labels' in data:
				key = 'labels'
			else:
				logger.error('Cannot parse this data in the MNIST hook. Too '
					'many keys, none of which is "labels".')
				return current
		else:
			logger.error('MNIST hook cannot process this output: it has no '
				'keys.')
			return current

		if key not in truth:
			logger.error('No ground truth data present for MNIST hook.')
			return current

		if len(truth[key]) != len(data[key]):
			logger.warning('There appears to be a different amount of ground '
				'truth information than model output.')
			return current

		decoded_estimate = numpy.argmax(data[key], axis=1)
		decoded_truth = numpy.argmax(truth[key], axis=1)

		for estimate, truth in zip(decoded_estimate, decoded_truth):
			if truth not in stats:
				logger.warning('Out-of-range truth information: %d', truth)
			else:
				stats[truth]['total'] += 1
				if truth == estimate:
					stats[truth]['correct'] += 1

		print(('{: <10s}' * 4).format(
			'LABEL', 'CORRECT', 'TOTAL', 'ACCURACY'
		))
		for i in range(10):
			print(('{: <10d}'*3 + '{: >5.1f}%').format(
				i, stats[i]['correct'], stats[i]['total'],
				stats[i]['correct'] * 100. / stats[i]['total'] \
					if stats[i]['total'] else 0
			))

		correct = sum(x['correct'] for x in stats.values())
		total = sum(x['total'] for x in stats.values())
		print(('{: <10s}' + '{: <10d}'*2 + '{: >5.1f}%').format(
			'ALL', correct, total, correct * 100. / total if total else 0
		))

		return current

### EOF.EOF.EOF.EOF.EOF.EOF.EOF.EOF.EOF.EOF.EOF.EOF.EOF.EOF.EOF.EOF.EOF.EOF.EOF

max_stars_repo_name: greedyuser/kur

max_issues_repo_name: greedyuser/kur

max_forks_repo_name: greedyuser/kur