text:
#!/usr/bin/env python

# Copyright (c) 2013, Carnegie Mellon University
# All rights reserved.
# Authors: Michael Koval <mkoval@cs.cmu.edu>
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# - Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
# - Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
# - Neither the name of Carnegie Mellon University nor the names of its
#   contributors may be used to endorse or promote products derived from this
#   software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import abc
import functools
import logging
import numpy
import openravepy
from ..clone import Clone, CloneException
from ..futures import defer
from ..util import CopyTrajectory, GetTrajectoryTags, SetTrajectoryTags
from .exceptions import (ClonedPlanningError, MetaPlanningError,
                         PlanningError, UnsupportedPlanningError)
from contextlib import contextmanager

logger = logging.getLogger(__name__)

class Tags(object):
    SMOOTH = 'smooth'
    """
    The `SMOOTH` tag means waypoints are close enough together that we can
    approximate derivatives at the waypoints using divided differences. i.e.
    we can safely fit a spline without collision checking.
    """

    CONSTRAINED = 'constrained'
    """
    The `CONSTRAINED` tag means that the geometric path described by these
    waypoints respects a constraint.  This means that the path cannot be
    geometrically altered arbitrarily, at the risk of violating the original
    constraint, it should only be changed in timing.
    """

    PLANNER = 'planner'
    """
    The name of the planner used to generate a trajectory.
    """

    METHOD = 'planning_method'
    """
    The type of planning call used to generate a trajectory.
    """

    PLAN_TIME = 'planning_time'
    """
    The amount of time that was spent by a planner finding a solution.
    """

    POSTPROCESS_TIME = 'postprocess_time'
    """
    The amount of time that was spent modifying the trajectory for execution.
    """

    EXECUTION_TIME = 'execution_time'
    """
    The amount of time that was spent actually running a trajectory.
    """

    DETERMINISTIC_TRAJECTORY = 'deterministic'
    """
    Whether repeating the same planning query will produce the same trajectory.
    """

    DETERMINISTIC_ENDPOINT = 'deterministic_endpoint'
    """
    Whether repeating the same planning query will produce a trajectory with
    the same endpoint as this trajectory.
    """

class LockedPlanningMethod(object):
    """
    Decorate a planning method that locks the calling environment.
    """
    def __init__(self, func):
        self.func = func

    def __call__(self, instance, robot, *args, **kw_args):
        with robot.GetEnv():
            # Perform the actual planning operation.
            traj = self.func(instance, robot, *args, **kw_args)

            # Tag the trajectory with the planner and planning method
            # used to generate it. We don't overwrite these tags if
            # they already exist.
            tags = GetTrajectoryTags(traj)
            tags.setdefault(Tags.PLANNER, instance.__class__.__name__)
            tags.setdefault(Tags.METHOD, self.func.__name__)
            SetTrajectoryTags(traj, tags, append=False)

            return traj

    def __get__(self, instance, instancetype):
        # Bind the self reference and use update_wrapper to propagate the
        # function's metadata (e.g. name and docstring).
        wrapper = functools.partial(self.__call__, instance)
        functools.update_wrapper(wrapper, self.func)
        wrapper.is_planning_method = True
        return wrapper

class ClonedPlanningMethod(LockedPlanningMethod):
    """
    Decorate a planning method that clones the calling environment.
    """
    def __call__(self, instance, robot, *args, **kw_args):
        env = robot.GetEnv()

        # Store the original joint values and indices.
        joint_indices = [robot.GetActiveDOFIndices(), None]
        joint_values = [robot.GetActiveDOFValues(), None]

        try:
            with Clone(env, clone_env=instance.env) as cloned_env:
                cloned_robot = cloned_env.Cloned(robot)

                # Store the cloned joint values and indices.
                joint_indices[1] = cloned_robot.GetActiveDOFIndices()
                joint_values[1] = cloned_robot.GetActiveDOFValues()

                # Check for mismatches in the cloning and hackily reset them.
                # (This is due to a possible bug in OpenRAVE environment
                # cloning where in certain situations, the Active DOF ordering
                # and values do not match the parent environment.  It seems to
                # be exacerbated by multirotation joints, but the exact cause
                # and repeatability is unclear at this point.)
                if not numpy.array_equal(joint_indices[0], joint_indices[1]):
                    logger.warning(
                        "Cloned Active DOF index mismatch: %s != %s",
                        str(joint_indices[0]), str(joint_indices[1]))
                    cloned_robot.SetActiveDOFs(joint_indices[0])

                if not numpy.allclose(joint_values[0], joint_values[1]):
                    logger.warning(
                        "Cloned Active DOF value mismatch: %s != %s",
                        str(joint_values[0]), str(joint_values[1]))
                    cloned_robot.SetActiveDOFValues(joint_values[0])

                traj = super(ClonedPlanningMethod, self).__call__(
                    instance, cloned_robot, *args, **kw_args)
                return CopyTrajectory(traj, env=env)
        except CloneException as e:
            raise ClonedPlanningError(e)

class PlanningMethod(ClonedPlanningMethod):
    def __init__(self, func):
        logger.warn("Please explicitly declare a ClonedPlanningMethod "
                    "instead of using PlanningMethod.")
        super(ClonedPlanningMethod, self).__init__(func)

class Planner(object):
    def has_planning_method(self, method_name):
        if hasattr(self, method_name):
            method = getattr(self, method_name)
            if hasattr(method, 'is_planning_method'):
                return method.is_planning_method
            else:
                return False
        else:
            return False

    def get_planning_method_names(self):
        return filter(lambda method_name: self.has_planning_method(method_name), dir(self))

class BasePlanner(Planner):
    def __init__(self):
        super(BasePlanner, self).__init__()
        self.env = openravepy.Environment()

class MetaPlanner(Planner):
    __metaclass__ = abc.ABCMeta

    def __init__(self):
        super(MetaPlanner, self).__init__()
        self._planners = list()

    def has_planning_method(self, method_name):
        for planner in self._planners:
            if planner.has_planning_method(method_name):
                return True

        return False

    def get_planning_method_names(self):
        method_names = set()
        for planner in self._planners:
            method_names.update(planner.get_planning_method_names())

        return list(method_names)

    @abc.abstractmethod
    def get_planners(self, method_name):
        pass

    def get_planners_recursive(self, method):
        all_planners = set()

        for planner in self.get_planners(method):
            if isinstance(planner, MetaPlanner):
                sub_planners = planner.get_planners_recursive(method)
                all_planners = all_planners.union(sub_planners)
            else:
                all_planners.add(planner)

        return list(all_planners)

    def __dir__(self):
        return self.get_planning_method_names()

    def __getattr__(self, method_name):
        if not self.has_planning_method(method_name):
            raise AttributeError("Object {:s} has no attribute '{:s}'.".format(
                                 repr(self), method_name))

        def meta_wrapper(*args, **kw_args):
            return self.plan(method_name, args, kw_args)

        # Grab docstrings from the delegate planners.
        meta_wrapper.__name__ = method_name
        docstrings = list()
        for planner in self.get_planners_recursive(method_name):
            if planner.has_planning_method(method_name):
                planner_method = getattr(planner, method_name)
                docstrings.append((planner, planner_method))

        # Concatenate the docstrings.
        if docstrings:
            meta_wrapper.__doc__ = ''

            for planner, planner_method in docstrings:
                formatted_docstring = ''

                if isinstance(planner, MetaPlanner):
                    if planner_method.__doc__ is not None:
                        formatted_docstring += planner_method.__doc__
                else:
                    # Header for this planner.
                    formatted_docstring += str(planner) + ': ' + planner_method.__name__ + '\n'
                    formatted_docstring += '-' * (len(formatted_docstring) - 1) + '\n'

                    # Docstring.
                    if planner_method.__doc__ is not None:
                        formatted_docstring += planner_method.__doc__
                    else:
                        formatted_docstring += '<no docstring>\n'

                    # Blank line.
                    formatted_docstring += '\n'

                meta_wrapper.__doc__ += formatted_docstring

        return meta_wrapper

class Sequence(MetaPlanner):
    KNOWN_KWARGS = set(['allow_nondeterministic'])

    def __init__(self, *planners, **kwargs):
        assert self.KNOWN_KWARGS.issuperset(kwargs.keys())

        super(Sequence, self).__init__()
        self._planners = planners
        self._allow_nondeterministic = kwargs.get(
            'allow_nondeterministic', False)

    def __str__(self):
        return 'Sequence({:s})'.format(', '.join(map(str, self._planners)))

    def get_planners(self, method_name):
        return [planner for planner in self._planners
                if planner.has_planning_method(method_name)]

    def plan(self, method, args, kw_args):
        from ..util import Timer

        errors = dict()
        is_sequence_deterministic = True

        for planner in self._planners:
            e = None

            try:
                if planner.has_planning_method(method):
                    logger.info('Sequence - Calling planner "%s".', str(planner))
                    planner_method = getattr(planner, method)

                    with Timer() as timer:
                        output = planner_method(*args, **kw_args)

                    if not is_sequence_deterministic:
                        # TODO: It is overly conservative to set _ENDPOINT,
                        # e.g. for PlanToConfiguration. Unfortunately, there is
                        # no easy way to detect this special case.
                        SetTrajectoryTags(output, {
                            Tags.DETERMINISTIC_TRAJECTORY: False,
                            Tags.DETERMINISTIC_ENDPOINT: False,
                        }, append=True)

                        if not self._allow_nondeterministic:
                            logger.warning(
                                'Tagging trajectory as non-deterministic because an'
                                ' earlier planner in the Sequence threw a'
                                ' non-deterministic PlanningError. Pass the'
                                ' "allow_nondeterministic" to this Sequence'
                                ' constructor if you intended this behavior.')

                    logger.info('Sequence - Planning succeeded after %.3f'
                                ' seconds with "%s".',
                                timer.get_duration(), str(planner))
                    return output
                else:
                    logger.debug('Sequence - Skipping planner "%s"; does not'
                                 ' have "%s" method.', str(planner), method)
            except MetaPlanningError as e:
                pass # Exception handled below.
            except PlanningError as e:
                logger.warning('Planning with %s failed: %s', planner, e)
                # Exception handled below.

            if e is not None:
                if e.deterministic is None:
                    is_sequence_deterministic = False
                    logger.warning(
                        'Planner %s raised a PlanningError without the'
                        ' "deterministic" flag set. Assuming the result'
                        ' is non-deterministic.', planner)
                elif not e.deterministic:
                    is_sequence_deterministic = False

                errors[planner] = e

        raise MetaPlanningError(
            'All planners failed.', errors, deterministic=is_sequence_deterministic)

class Ranked(MetaPlanner):
    def __init__(self, *planners):
        super(Ranked, self).__init__()
        self._planners = planners

    def __str__(self):
        return 'Ranked({0:s})'.format(', '.join(map(str, self._planners)))

    def get_planners(self, method_name):
        return [planner for planner in self._planners
                if planner.has_planning_method(method_name)]

    def plan(self, method, args, kw_args):
        all_planners = self._planners
        futures = []
        results = [None] * len(self._planners)

        # Helper function to call a planner and return its result.
        def call_planner(planner):
            planning_method = getattr(planner, method)
            return planning_method(*args, **kw_args)

        # Find only planners that support the required planning method.
        # Call every planners in parallel using a concurrent executor and
        # return the first non-error result in the ordering when available.
        for index, planner in enumerate(all_planners):
            if not planner.has_planning_method(method):
                results[index] = PlanningError(
                    "{:s} does not implement method {:s}."
                    .format(planner, method))
                continue
            else:
                futures.append((index, defer(call_planner, args=(planner,))))

        # Each time a planner completes, check if we have a valid result
        # (a planner found a solution and all higher-ranked planners had
        # already failed).
        for index, future in futures:
            try:
                return future.result()
            except MetaPlanningError as e:
                results[index] = e
            except PlanningError as e:
                logger.warning("Planning with {:s} failed: {:s}"
                               .format(planner, e))
                results[index] = e
        # TODO: if `cancel()` is supported, call it in a `finally` block here.

        raise MetaPlanningError("All planners failed.",
                                dict(zip(all_planners, results)))

class FirstSupported(MetaPlanner):
    def __init__(self, *planners):
        super(FirstSupported, self).__init__()
        self._planners = planners

    def __str__(self):
        return 'Fallback({:s})'.format(', '.join(map(str, self._planners)))

    def get_planners(self, method_name):
        return [planner for planner in self._planners
                if planner.has_planning_method(method_name)]

    def plan(self, method, args, kw_args):
        for planner in self._planners:
            if planner.has_planning_method(method):
                plan_fn = getattr(planner, method)

                try:
                    return plan_fn(*args, **kw_args)
                except UnsupportedPlanningError:
                    continue

        raise UnsupportedPlanningError()

class MethodMask(MetaPlanner):
    def __init__(self, planner, methods):
        super(MethodMask, self).__init__()
        self._methods = set(methods)
        self._planner = planner
        self._planners = [planner]

    def __str__(self):
        return 'Only({:s}, methods={:s})'.format(
            self._planner, list(self._methods))

    def get_planners(self, method_name):
        if method_name in self._methods:
            return [self._planner]
        else:
            return []

    def plan(self, method, args, kw_args):
        if method in self._methods:
            plan_fn = getattr(self._planner, method)
            return plan_fn(*args, **kw_args)
        else:
            raise UnsupportedPlanningError()

@contextmanager
def save_dof_limits(robot):
    upper, lower = robot.GetDOFLimits()
    try:
        yield
    finally:
        robot.SetDOFLimits(upper, lower)

max_stars_repo_name: MisoRobotics/prpy

max_issues_repo_name: MisoRobotics/prpy

max_forks_repo_name: MisoRobotics/prpy


text:
""" Functions for calculating multiple eQTLS
"""

import pandas as pd
import numpy as np
import statsmodels.api as sm

def ceQTL(counts, dos, cov_mat, rsid):
    """ Convinence function to calculate the single SNP comparison
    """
    acov_mat = cov_mat.copy(deep=True)
    acov_mat['soi'] = dos.ix[rsid, acov_mat.index]
    res = sm.OLS(counts, acov_mat).fit()
    return(res)

def calculate_top_cond(counts, dos, cov_mat, rsid1,
        return_rsid1=False):
    """ Need to mask rsid1 due to perfect multicolinneraity
    """
    acov_mat = cov_mat.copy(deep=True)
    acov_mat[rsid1] = dos.ix[rsid1, acov_mat.index]

    def _calc_model(geno, counts, cov_mat):
        t = cov_mat.copy(deep=True)
        t['ts'] = geno
        return(sm.OLS(counts, t).fit().pvalues['ts'])

    def _calc_model_two(geno, counts, cov_mat, rsid1):
        t = cov_mat.copy(deep=True)
        t['ts'] = geno
        fit = sm.OLS(counts, t).fit()
        return(pd.Series({'aic': fit.aic,
            'rsid2': fit.pvalues['ts']}))

    '''
    pvalues = dos.apply(_calc_model, axis=1, args=(counts,
        acov_mat))
    '''
    pvalues = dos.apply(_calc_model_two, axis=1, args=(counts,
        acov_mat, rsid1))
    # Mask collinearity
    pvalues.ix[rsid1, 'rsid2'] = 1
    return(pvalues)

def calculate_two_snp_model(counts, dos, cov_mat, rsid1, rsid2):
    """
    """
    pass

ext: py

reason: import numpy,import statsmodels


text:
import torch
from torch.utils import data
import pickle
import numpy as np
from skimage.transform import rescale
from os import listdir
from os.path import join

class ToySampledTripletTestDataset(data.Dataset):
    def __init__(self, width, height, depth):

        self.width = width
        self.height = height
        self.depth = depth

        all_original_toms = []
        toy_data_dir = '/pasteur/data/hierarchical-toy-dataset/toy_irregular4/test/im_{}.p'
        for i in range(20):
            curr_path = toy_data_dir.format(str(i))
            toy = pickle.load(open(curr_path, 'rb'))
            padded_toy = np.zeros((64, 64, 64))
            padded_toy[8:58, 8:58, 8:58] = toy
            all_original_toms.append(padded_toy.astype(np.float32))

        self.input_toms = all_original_toms

        mapping = {}
        index = 0

        # 5 x 5
        for l in range(len(all_original_toms)):
            for i in range(6, 56, 1):
                for j in range(6, 56, 1):
                    for k in range(6, 56, 1):
                        mapping[index] = (l, i, j, k, 5)
                        index += 1

        # 10 x 10
        for l in range(len(all_original_toms)):
            for i in range(4, 54, 1):
                for j in range(4, 54, 1):
                    for k in range(4, 54, 1):
                        mapping[index] = (l, i, j, k, 10)
                        index += 1

        # 15 x 15
        for l in range(len(all_original_toms)):
            for i in range(0, 50, 1):
                for j in range(0, 50, 1):
                    for k in range(0, 50, 1):
                        mapping[index] = (l, i, j, k, 15)
                        index += 1

        self.index_mapping = mapping

    def __len__(self):
        return len(self.index_mapping)

    def __getitem__(self, index):

        l, i, j, k, patch_size = self.index_mapping[index]

        original_tom = self.input_toms[l]

        if patch_size == 5:
            original_tomogram = original_tom[i:i+5, j:j+5, k:k+5]
            original_tomogram = rescale(original_tomogram, 2, anti_aliasing=False)
            original_tomogram = np.expand_dims(original_tomogram, 0)

        if patch_size == 10:
            original_tomogram = original_tom[i:i+10, j:j+10, k:k+10]
            original_tomogram = np.expand_dims(original_tomogram, 0)

        if patch_size == 15:
            original_tomogram = original_tom[i:i+15, j:j+15, k:k+15]
            original_tomogram = rescale(original_tomogram, 2/3, anti_aliasing=False)
            original_tomogram = np.expand_dims(original_tomogram, 0)

        return original_tomogram, patch_size

reason: import numpy


text:
import numpy as np

class Vertex:
    def __init__(self, position):
        self.x, self.y, self.z = position

    def __str__(self):
        return f"({self.x}, {self.y}, {self.z})"

    def flatten(self, camera):
        position = self.get_3d_position()
        camera_position = camera.get_3d_position()

        new_vertex_position = position - camera_position

        depth = new_vertex_position[2]

        offset = depth / camera_position[2]

        position_flattened = np.array([new_vertex_position[0], new_vertex_position[1]], dtype='float64')
        position_flattened *= np.abs(offset)

        return tuple(position_flattened)

    def get_3d_position(self):
        return np.array([self.x, self.y, self.z])


text:
"""
==========================
Stochastic test
==========================

This example is designed to test the stochatic optimization algorithms module
for descrete and semicontinous measures from the POT library.

"""

# Author: Kilian Fatras <kilian.fatras@gmail.com>
#
# License: MIT License

import numpy as np
import ot

#############################################################################
# COMPUTE TEST FOR SEMI-DUAL PROBLEM
#############################################################################

#############################################################################
#
# TEST SAG algorithm
# ---------------------------------------------
# 2 identical discrete measures u defined on the same space with a
# regularization term, a learning rate and a number of iteration

def test_stochastic_sag():
    # test sag
    n = 10
    reg = 1
    numItermax = 30000
    rng = np.random.RandomState(0)

    x = rng.randn(n, 2)
    u = ot.utils.unif(n)

    M = ot.dist(x, x)

    G = ot.stochastic.solve_semi_dual_entropic(u, u, M, reg, "sag",
                                               numItermax=numItermax)

    # check constratints
    np.testing.assert_allclose(
        u, G.sum(1), atol=1e-03)  # cf convergence sag
    np.testing.assert_allclose(
        u, G.sum(0), atol=1e-03)  # cf convergence sag

#############################################################################
#
# TEST ASGD algorithm
# ---------------------------------------------
# 2 identical discrete measures u defined on the same space with a
# regularization term, a learning rate and a number of iteration

def test_stochastic_asgd():
    # test asgd
    n = 10
    reg = 1
    numItermax = 10000
    rng = np.random.RandomState(0)

    x = rng.randn(n, 2)
    u = ot.utils.unif(n)

    M = ot.dist(x, x)

    G, log = ot.stochastic.solve_semi_dual_entropic(u, u, M, reg, "asgd",
                                                    numItermax=numItermax, log=True)

    # check constratints
    np.testing.assert_allclose(
        u, G.sum(1), atol=1e-02)  # cf convergence asgd
    np.testing.assert_allclose(
        u, G.sum(0), atol=1e-02)  # cf convergence asgd

#############################################################################
#
# TEST Convergence SAG and ASGD toward Sinkhorn's solution
# --------------------------------------------------------
# 2 identical discrete measures u defined on the same space with a
# regularization term, a learning rate and a number of iteration

def test_sag_asgd_sinkhorn():
    # test all algorithms
    n = 10
    reg = 1
    nb_iter = 10000
    rng = np.random.RandomState(0)

    x = rng.randn(n, 2)
    u = ot.utils.unif(n)
    M = ot.dist(x, x)

    G_asgd = ot.stochastic.solve_semi_dual_entropic(u, u, M, reg, "asgd",
                                                    numItermax=nb_iter)
    G_sag = ot.stochastic.solve_semi_dual_entropic(u, u, M, reg, "sag",
                                                   numItermax=nb_iter)
    G_sinkhorn = ot.sinkhorn(u, u, M, reg)

    # check constratints
    np.testing.assert_allclose(
        G_sag.sum(1), G_sinkhorn.sum(1), atol=1e-02)
    np.testing.assert_allclose(
        G_sag.sum(0), G_sinkhorn.sum(0), atol=1e-02)
    np.testing.assert_allclose(
        G_asgd.sum(1), G_sinkhorn.sum(1), atol=1e-02)
    np.testing.assert_allclose(
        G_asgd.sum(0), G_sinkhorn.sum(0), atol=1e-02)
    np.testing.assert_allclose(
        G_sag, G_sinkhorn, atol=1e-02)  # cf convergence sag
    np.testing.assert_allclose(
        G_asgd, G_sinkhorn, atol=1e-02)  # cf convergence asgd

#############################################################################
# COMPUTE TEST FOR DUAL PROBLEM
#############################################################################

#############################################################################
#
# TEST SGD algorithm
# ---------------------------------------------
# 2 identical discrete measures u defined on the same space with a
# regularization term, a batch_size and a number of iteration

def test_stochastic_dual_sgd():
    # test sgd
    n = 10
    reg = 1
    numItermax = 5000
    batch_size = 10
    rng = np.random.RandomState(0)

    x = rng.randn(n, 2)
    u = ot.utils.unif(n)

    M = ot.dist(x, x)

    G, log = ot.stochastic.solve_dual_entropic(u, u, M, reg, batch_size,
                                               numItermax=numItermax, log=True)

    # check constratints
    np.testing.assert_allclose(
        u, G.sum(1), atol=1e-03)  # cf convergence sgd
    np.testing.assert_allclose(
        u, G.sum(0), atol=1e-03)  # cf convergence sgd

#############################################################################
#
# TEST Convergence SGD toward Sinkhorn's solution
# --------------------------------------------------------
# 2 identical discrete measures u defined on the same space with a
# regularization term, a batch_size and a number of iteration

def test_dual_sgd_sinkhorn():
    # test all dual algorithms
    n = 10
    reg = 1
    nb_iter = 5000
    batch_size = 10
    rng = np.random.RandomState(0)

# Test uniform
    x = rng.randn(n, 2)
    u = ot.utils.unif(n)
    M = ot.dist(x, x)

    G_sgd = ot.stochastic.solve_dual_entropic(u, u, M, reg, batch_size,
                                              numItermax=nb_iter)

    G_sinkhorn = ot.sinkhorn(u, u, M, reg)

    # check constratints
    np.testing.assert_allclose(
        G_sgd.sum(1), G_sinkhorn.sum(1), atol=1e-02)
    np.testing.assert_allclose(
        G_sgd.sum(0), G_sinkhorn.sum(0), atol=1e-02)
    np.testing.assert_allclose(
        G_sgd, G_sinkhorn, atol=1e-02)  # cf convergence sgd

# Test gaussian
    n = 30
    reg = 1
    batch_size = 30

    a = ot.datasets.make_1D_gauss(n, 15, 5)  # m= mean, s= std
    b = ot.datasets.make_1D_gauss(n, 15, 5)
    X_source = np.arange(n, dtype=np.float64)
    Y_target = np.arange(n, dtype=np.float64)
    M = ot.dist(X_source.reshape((n, 1)), Y_target.reshape((n, 1)))
    M /= M.max()

    G_sgd = ot.stochastic.solve_dual_entropic(a, b, M, reg, batch_size,
                                              numItermax=nb_iter)

    G_sinkhorn = ot.sinkhorn(a, b, M, reg)

    # check constratints
    np.testing.assert_allclose(
        G_sgd.sum(1), G_sinkhorn.sum(1), atol=1e-03)
    np.testing.assert_allclose(
        G_sgd.sum(0), G_sinkhorn.sum(0), atol=1e-03)
    np.testing.assert_allclose(
        G_sgd, G_sinkhorn, atol=1e-03)  # cf convergence sgd

max_stars_repo_name: kguerda-idris/POT

max_issues_repo_name: kguerda-idris/POT

max_forks_repo_name: kguerda-idris/POT


text:
#! /usr/bin/env python
# Copyright 2021 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import dpctl
import dpctl.memory as dpctl_mem
import dpctl.tensor as dpt
import numpy as np
import pytest

from numba_dppy.tests._helper import filter_strings
from numba_dppy.utils import (
    as_usm_obj,
    copy_to_numpy_from_usm_obj,
    has_usm_memory,
)

from . import _helper

@pytest.mark.parametrize("filter_str", filter_strings)
def test_has_usm_memory(filter_str):
    a = np.ones(1023, dtype=np.float32)

    with dpctl.device_context(filter_str):
        # test usm_ndarray
        da = dpt.usm_ndarray(a.shape, dtype=a.dtype, buffer="shared")
        usm_mem = has_usm_memory(da)
        assert da.usm_data._pointer == usm_mem._pointer

        # test usm allocated numpy.ndarray
        buf = dpctl_mem.MemoryUSMShared(a.size * a.dtype.itemsize)
        ary_buf = np.ndarray(a.shape, buffer=buf, dtype=a.dtype)
        usm_mem = has_usm_memory(ary_buf)
        assert buf._pointer == usm_mem._pointer

        usm_mem = has_usm_memory(a)
        assert usm_mem is None

@pytest.mark.parametrize("filter_str", filter_strings)
def test_as_usm_obj(filter_str):
    a = np.ones(1023, dtype=np.float32)
    b = a * 3

    with dpctl.device_context(filter_str) as queue:
        a_copy = np.empty_like(a)
        usm_mem = as_usm_obj(a, queue=queue)
        copy_to_numpy_from_usm_obj(usm_mem, a_copy)
        assert np.all(a == a_copy)

        b_copy = np.empty_like(b)
        usm_mem = as_usm_obj(b, queue=queue, copy=False)
        copy_to_numpy_from_usm_obj(usm_mem, b_copy)
        assert np.any(np.not_equal(b, b_copy))

max_stars_repo_name: akharche/numba-dppy

max_issues_repo_name: akharche/numba-dppy

max_forks_repo_name: akharche/numba-dppy

reason: import numpy,from numba


text:
import nelpy as nel
import numpy as np
from math import pi

class TestRegularlySampledAnalogSignalArray:

    def test_copy(self):
        data = np.arange(100)
        fs = 10
        rsasa = nel.RegularlySampledAnalogSignalArray(data, abscissa_vals=data/fs, fs=fs)
        copied_rsasa = rsasa.copy()

        assert hex(id(copied_rsasa)) == hex(id(copied_rsasa._intervalsignalslicer.obj))

    def test_add_signal1D_1(self):
        """Add a signal to an 1D AnalogSignalArray"""
        asa = nel.AnalogSignalArray([1,2,4])
        asa.add_signal([3,4,5])
        assert asa.n_signals == 2

    # def test_add_signal1D_2(self):
    #     """Add a signal to a 1D AnalogSignalArray
    #     Note: should pass on column-wise signals"""
    #     asa = AnalogSignalArray([1,2,4])
    #     asa.add_signal([3,4,5])
    #     assert np.array(asa.data == np.array([[1,2,4],[3,4,5]]).T).all()

    def test_add_signal1D_3(self):
        """Add a signal to a 1D AnalogSignalArray
        Note: should pass on row-wise signals"""
        asa = nel.AnalogSignalArray([1,2,4])
        asa.add_signal([3,4,5])
        assert np.array(asa.data == np.array([[1,2,4],[3,4,5]])).all()

    # def test_add_signal1D_4(self):
    #     """Add a signal to an 2D AnalogSignalArray
    #     Note: should pass on column-wise signals"""
    #     asa = AnalogSignalArray([[1, 2, 4], [7, 8, 9]])
    #     asa.add_signal([3, 4, 5])
    #     assert np.array(asa.data == np.array([[1, 2, 4], [7, 8, 9], [3, 4, 5]]).T).all()

    def test_add_signal1D_5(self):
        """Add a signal to an 2D AnalogSignalArray
        Note: should pass on row-wise signals"""
        asa = nel.AnalogSignalArray([[1, 2, 4], [7, 8, 9]])
        asa.add_signal([3, 4, 5])
        assert np.array(asa.data == np.array([[1, 2, 4], [7, 8, 9], [3, 4, 5]])).all()

    def test_complex_asa_1(self):
        N = 128
        theta = np.array(2*pi/N*np.arange(N))
        exp_theta = np.exp(np.array(theta)*1j)
        casa = nel.AnalogSignalArray(exp_theta)
        assert np.all(np.isclose(casa.abs.data, 1))

    def test_complex_asa_2(self):
        N = 6
        theta = np.array(2*pi/N*np.arange(N))
        exp_theta = np.exp(np.array(theta)*1j)
        casa = nel.AnalogSignalArray(exp_theta)
        expected = np.array([[ 1.0 +0.00000000e+00j, 0.5 +8.66025404e-01j,
            -0.5 +8.66025404e-01j, -1.0 +1.22464680e-16j,
            -0.5 -8.66025404e-01j,  0.5 -8.66025404e-01j]])
        assert np.all(np.isclose(casa.data, expected))

    def test_complex_asa_3(self):
        N = 6
        theta = np.array(2*pi/N*np.arange(N))
        exp_theta = np.exp(np.array(theta)*1j)
        casa = nel.AnalogSignalArray(exp_theta)
        expected = np.array([[ 0. ,  1.04719755,  2.0943951 ,  3.14159265, -2.0943951 ,
        -1.04719755]])
        assert np.all(np.isclose(casa.angle.data, expected))

    def test_asa_data_format1(self):
        asa = nel.AnalogSignalArray([[1, 2, 4], [7, 8, 9]])
        asa.add_signal([3, 4, 5])
        assert np.array(asa._data_rowsig == np.array([[1, 2, 4], [7, 8, 9], [3, 4, 5]])).all()

    def test_asa_data_format2(self):
        asa = nel.AnalogSignalArray([[1, 2, 4], [7, 8, 9]])
        asa.add_signal([3, 4, 5])
        assert np.array(asa._data_colsig == np.array([[1, 7, 3], [2, 8, 4], [4, 9, 5]])).all()

    def test_asa_n_signals(self):
        asa = nel.AnalogSignalArray([[1, 2, 4, 5], [7, 8, 9, 10]])
        asa.add_signal([3, 4, 5, 6])
        assert asa.n_signals == 3

    def test_asa_n_samples(self):
        asa = nel.AnalogSignalArray([[1, 2, 4, 5], [7, 8, 9, 10]])
        asa.add_signal([3, 4, 5, 6])
        assert asa.n_samples == 4

    def test_asa_asarray(self):
        asa = nel.AnalogSignalArray([[1, 2, 4], [7, 8, 9]])
        asa.add_signal([3, 4, 5])
        assert np.array(asa.asarray().yvals == np.array([[1, 2, 4], [7, 8, 9], [3, 4, 5]])).all()

    def test_asa_asarray2(self):
        asa = nel.AnalogSignalArray(np.arange(10), fs=1)
        assert asa(0.5) == np.array([0.5])
        assert asa.asarray(at=0.5).yvals == np.array([0.5])

    def test_asa_mean1(self):
        asa = nel.AnalogSignalArray([[1, 2, 4, 5], [7, 8, 9, 10]])
        asa.add_signal([3, 4, 5, 6])
        assert np.array(asa.mean() == np.array([ 3. ,  8.5,  4.5])).all()

    def test_asa_mean2(self):
        asa = nel.AnalogSignalArray([[1, 2, 4, 5], [7, 8, 9, 10]])
        asa.add_signal([3, 4, 5, 6])
        asa = asa[nel.EpochArray([[0,1.1], [1.9,3.1]])]
        assert np.array(asa.mean() == np.array([ 3. ,  8.5,  4.5])).all()

    def test_asa_mean3(self):
        asa = nel.AnalogSignalArray([[1, 2, 4, 5], [7, 8, 9, 10]])
        asa.add_signal([3, 4, 5, 6])
        asa = asa[nel.EpochArray([[0,1.1], [1.9,3.1]])]
        means = [seg.mean() for seg in asa]
        assert np.array(means == np.array([np.array([ 1.5,  7.5,  3.5]), np.array([ 4.5,  9.5,  5.5])])).all()

class TestHalfOpenIntervals:

    def test_asa_halfopen_1(self):
        asa = nel.AnalogSignalArray([0,1,2,3,4,5,6])
        assert asa.n_samples == 7
        assert asa.support.duration == 7

    def test_asa_halfopen_2(self):
        asa = nel.AnalogSignalArray([0,0,0,1,1,1,2,2,2])
        epochs = nel.utils.get_run_epochs(asa, v1=2, v2=2)
        assert np.allclose(epochs.data, np.array([6, 9]))

    def test_asa_halfopen_3(self):
        asa = nel.AnalogSignalArray([0,0,0,1,1,1,2,2,2])
        epochs = nel.utils.get_run_epochs(asa, v1=1, v2=1)
        assert np.allclose(epochs.data, np.array([3, 9]))

    def test_asa_halfopen_4(self):
        asa = nel.AnalogSignalArray([0,0,0,1,1,1,2,2,2])
        epochs = nel.utils.get_inactive_epochs(asa, v1=1, v2=1)
        assert np.allclose(epochs.data, np.array([0, 6]))


text:
from mpl_toolkits.mplot3d import axes3d
from matplotlib import animation
from matplotlib.animation import HTMLWriter
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.pyplot as pl
import numpy as np
import warnings
import sys, os

def blockPrint():
    """
    Blocks print output for io
    """
    sys.stdout = open(os.devnull, 'w')

def enablePrint():
    """
    Enables print output for io
    """
    sys.stdout = sys.__stdout__

def rotate_3d(df, feat, save_file=True, **kwargs):
    """
    Creates a .gif file of the rotating 3d plot with gives axes

    Parameters
    ----------
    df        : pd.dataFrame :
              dataframe of the results data taken from the prediciton study. Assumes
              results are in df['Predicted'] and there are 4 classes to predict
    feat      : [string] :
              [x, y, z] axis df label for 3d plot
    save_file : bool :
              save file or not

    kwargs include override parameter dicts for matplotlib.animation.FuncAnimation()
    and matplotlib.animation.FuncAnimation.save(). See API for both of these.
    Label is 'anim_param' and 'save_param' respectively.

    Returns
    -------
    rot_animation : object :
        FuncAnimation object containing animation data
    """

    assert all([(i in df.columns) for i in feat]), "Feature(s) not in the dataframe"
    assert len(feat) == 3, "Inputed too many features"

    pred_list = df.predicted.unique()
    # Animation/Saving parameters:
    anim_param = {'frames':np.arange(0,407,2),
                  'interval':50,
                  'repeat':True
                 }
    save_param = {'filename':f'./gif_rotation_{feat[0]}_{feat[1]}_{feat[2]}.gif'.replace(' ', ''),
                  'writer':'imagemagick',
                  'fps':20
                 }
    # Override for making and saving animations
    if 'anim_param' in kwargs:
        anim_param.update(kwargs['anim_param'])
    if 'save_param' in kwargs:
        save_param.update(kwargs['save_param'])

    # Initial conditions
    def init():
        for cat in pred_list:
            ax.scatter(dict_list[cat][feat[0]], dict_list[cat][feat[1]], dict_list[cat][feat[2]], marker = 'o', alpha = 0.2)
        ax.set_xlabel(feat[0])
        ax.set_ylabel(feat[1])
        ax.set_zlabel(feat[2])
        leg = pl.legend(pred_list)
        for lh in leg.legendHandles:
            lh.set_alpha(1)
        return fig,
    # Changing conditions
    def rotate(angle):
        ax.view_init(azim=angle)
        return fig,

    # Disable matplotlib interaction
    pl.ioff()
    # Disable warnings that FuncAnimation() gives
    with warnings.catch_warnings():
        warnings.simplefilter("ignore")
        fig = pl.figure()
        ax = Axes3D(fig)
        dict_list = []
        for tag in pred_list:
            dict_list.append(df[df['predicted']==tag])
        rot_animation = animation.FuncAnimation(fig=fig, func=rotate, init_func=init, **anim_param)
        if save_file:
            rot_animation.save(**save_param)
        # Close plot if using %matplotlib inline
        pl.close()
    return rot_animation


text:
#
#  Copyright 2019 Koyal Bhartia
#  @file    8-puzzle solver.py
#  @author  Koyal Bhartia
#  @date    26/02/2019
#  @version 1.0
#
#  @brief This is the code to solve the 8-puzzle
#
# @Description This code has functions which returns all the possible paths that can be
# traversed given any goal matrix. Given any input matrix it gives the path to the goal matrix
#

import argparse
import numpy as np
import os, sys
import math
from collections import Counter

# @brief Comapares each element of any 2 3*3 matrices
#
#  @param The matrices A and b
#
#  @return Flag indicating if the matrices A and B are equal or not
def compare(A,B):
    for row in range(3):
        for col in range(3):
            if(A[row,col]!=B[row,col]):
                return 0
    return 1

# @brief Checks if the new node created is present in the nodes created earlier
#
#  @param The 3D matrix of all nodes and the new node generated
#
#  @return Flag indicating if the nodes are equal or not
'''
def checkrepeated(Nodes,new_mat):
    for position in range(Nodes.shape[2]):
        if(compare(Nodes[:,:,position],new_mat)):
            return 1
    return 0
'''
# @brief Generates all the possible nodes that can be traveresed along with its directions in
# nodeInfo
#
#  @param The desired goal node and the format of the nodeinfo
#
#  @return The updated 3D node matrix and the complete NodeInfo table
def createAllNodes(Nodes,NodesInfo):

    def countDistinct(arr):
        return len(Counter(arr).keys())

    # @brief Returns the position of the 0 in any given matrix
    #
    #  @param The matrix
    #
    #  @return The 0 positions

    def zero_position(matrix):
        for i in range(0,len(matrix)):
            for j in range(0,len(matrix)):
                if(matrix[i,j]==0):
                    zx=i
                    zy=j
        return zx , zy

    # @brief Swipes the 0 with its adjacent cells (if possible) and keeps updating the node matrix and nodeinfo
    #
    #  @param The matrix in which the 0 has to be swiped, the current parent and child status and the position of the "0" in the matrix
    #
    #  @return The current child position

    def Swipe(matrix,parent,child,row,col):
        checkrepeat=0
        if row is not 0: #top
            new_mat = matrix.copy()
            new_mat[row,col]=matrix[row-1,col]
            new_mat[row-1,col]=0
            #checkrepeat=checkrepeated(Nodes,new_mat)
            #if(checkrepeat==0):
            Nodes[:,:,child]=new_mat
            print(Nodes[:,:,child],"top")
            NodesInfo[child,:,:]=[parent, child, 0, 2] # 2-> top
            child+=1

        if row is not 2: #down
            new_mat = matrix.copy()
            new_mat[row,col]=matrix[row+1,col]
            new_mat[row+1,col]=0
            #checkrepeat=checkrepeated(Nodes,new_mat)
            #if(checkrepeat==0):
            Nodes[:,:,child]=new_mat
            print(Nodes[:,:,child],"dowm")
            NodesInfo[child,:,:]=[parent, child, 0, 8] #8-> down
            child+=1

        if col is not 0: #left
            new_mat = matrix.copy()
            new_mat[row,col]=matrix[row,col-1]
            new_mat[row,col-1]=0
            #checkrepeat=checkrepeated(Nodes,new_mat)
            #if(checkrepeat==0):
            Nodes[:,:,child]=new_mat
            print(Nodes[:,:,child],"left")
            NodesInfo[child,:,:]=[parent, child, 0, 4] #4 -> left
            child+=1

        if col is not 2: #right
            new_mat = matrix.copy()
            new_mat[row,col]=matrix[row,col+1]
            new_mat[row,col+1]=0
            #checkrepeat=checkrepeated(Nodes,new_mat)
            #if(checkrepeat==0):
            Nodes[:,:,child]=new_mat
            print(Nodes[:,:,child],"right")
            NodesInfo[child,:,:]=[parent, child, 0, 6] #6 -> right
            child+=1

        return child

    parent=0
    child=1

    while(child<Total-4):
        print("Child No",child)
        New_Parent=Nodes[:,:,parent]
        print("New_Parent",New_Parent)
        zx, zy = zero_position(New_Parent)
        child=Swipe(New_Parent,parent, child,zx, zy)
        parent+=1

    return Nodes, NodesInfo

# @brief Searches if the input node is present in the list of all possible nodes created earlier
#
#  @param The node list, the input node and the flag as an indiactor of found node
#
#  @return The position where the node is found and the Flag indiacting a successful or an unsuccessful find
def search(Nodes, Input,Found):
    print(Nodes.shape[2],"Nodes.shape[2]")
    for position in range(Nodes.shape[2]):
        current =Nodes[:,:,position]
        if(compare(current,Input)):
            Found=True
            break;
    print(Found,"search pos")
    return position, Found

# @brief Generates the node path from the input node to get to the goal node
#
#  @param The desired goal node, the nodeInfo indiacting all the parent child relationships and the position where the node is found
#
#  @return The path that the node takes to reach the goal node

def NodePath(Nodes,NodesInfo,node_position):
    node_path=[]
    node_path.append(node_position)
    while(node_position!=0):
        print(node_position,"node_position")
        node_position=int(NodesInfo[node_position,0,0])
        node_path.append(node_position)
    nodePathMat=np.zeros((3,3,len(node_path)))
    for i in range(len(node_path)):
        nodePathMat[:,:,i]=Nodes[:,:,node_path[i]]
    return nodePathMat

# @brief Prints the List of nodes, the node info table and the node path to the goal node in 3 different text files
#
#  @param The updated matrices of the all possible nodes, the nodeinfo matrix and the node path from input to the goal
#
#  @return The 3 output text files

def TextOutput(Nodes,NodesInfo,nodePathMat):

    def NodesTransform(Nodes):
        NodesTf=np.zeros((Nodes.shape[2],1,9))
        for i in range(Nodes.shape[2]):
            counter=0
            for col in range(3):
                for row in range(3):
                    NodesTf[i,0,counter]=Nodes[row,col,i].copy()
                    counter+=1
        return NodesTf

    Nodes=NodesTransform(Nodes)
    with open('Nodes.txt', 'w') as file:
        for data in Nodes:
            np.savetxt(file, data, fmt='%-2.0f')

    with open('NodesInfo.txt', 'w') as file:
        for data in NodesInfo:
            np.savetxt(file, data, fmt='%-2.0f')

    with open('NodePath.txt', 'w') as file:
        if(len(nodePathMat)==0):
            data=[]
            file.write("The above combination does not exist")
            np.savetxt(file, data)
        else:
            NodePath=NodesTransform(nodePathMat)
            for data in NodePath:
                np.savetxt(file, data, fmt='%-2.0f')

if __name__ == '__main__':
    #Total no of possible nodes: 9!/2=181441
    Total=181441
    #The desired goal
    Goal=np.mat([[1,2,3],[4,5,6],[7,8,0]])
    #Backup Input if user enters wrong matrix
    Input=np.mat([[1,2,3],[4,5,6],[7,0,8]])
    #Initializing node, nodeinfo matrix
    Nodes=np.zeros((3,3,Total))
    NodesInfo=np.zeros((Total,1,4))
    Nodes[:,:,0]=Goal
    NodesInfo[0,:,:]=[0,0,0,0]
    #Indiactes if the iput node is found
    Found=False
    NodePathMat=[]
    flag=0

    while(flag!=1):
        input("Please enter the nos between 0-8 rowise to start the 8-puzzle after pressing enter")
        try:
            Input=np.zeros((3,3))
            for i in range(3):
                for j in range(3):
                    Input[i][j]=int(input('Element [%d][%d]=' %(i,j)))
                    if (Input[i][j]>8):
                        raise ValueError
            flag=1
            print(flag,"flag")
        except ValueError:
            print("No valid..! Please enter integer less than 8 ...")

    print("Input matrix",Input)
    Nodes, NodesInfo=createAllNodes(Nodes,NodesInfo)
    position,Found=search(Nodes,Input,Found)
    if Found:
        NodePathMat=NodePath(Nodes,NodesInfo,position)
    TextOutput(Nodes,NodesInfo,NodePathMat)


text:
from __future__ import print_function
import h5py
import numpy as np
from six.moves import xrange
import sys
import mpii

keys = ['index','person','imgname','center','scale','part','visible','normalize','torsoangle','multi','istrain']
annot = {k:[] for k in keys}

# Set up index reference for multiperson training
multiRef = np.zeros(mpii.nimages)
trainRef = mpii.annot['img_train'][0][0][0]
allIdxs = np.arange(0,trainRef.shape[0])
with h5py.File('../../data/mpii/annot/multi-idxs.h5','r') as f:
    mTrain = f['train'][:] - 1
    mTest = f['test'][:] - 1
    multiRef[allIdxs[trainRef == 1][mTrain]] = 1
    multiRef[allIdxs[trainRef == 0][mTest]] = 1

# Get image filenames
imgnameRef = mpii.annot['annolist'][0][0][0]['image'][:]

for idx in xrange(mpii.nimages):
    print("\r",idx, end=' ')
    sys.stdout.flush()

    for person in xrange(mpii.numpeople(idx)):
        c,s = mpii.location(idx,person)
        if not c[0] == -1:
            # Add info to annotation list
            annot['index'] += [idx]
            annot['person'] += [person]
            imgname = np.zeros(16)
            refname = str(imgnameRef[idx][0][0][0][0])
            for i in range(len(refname)): imgname[i] = ord(refname[i])
            annot['imgname'] += [imgname]
            annot['center'] += [c]
            annot['scale'] += [s]
            annot['multi'] += [multiRef[idx]]

            if mpii.istrain(idx) == True:
                # Part annotations and visibility
                coords = np.zeros((16,2))
                vis = np.zeros(16)
                for part in xrange(16):
                   coords[part],vis[part] = mpii.partinfo(idx,person,part)
                annot['part'] += [coords]
                annot['visible'] += [vis]
                annot['normalize'] += [mpii.normalization(idx,person)]
                annot['torsoangle'] += [mpii.torsoangle(idx,person)]
                annot['istrain'] += [1]
            else:
                annot['part'] += [-np.ones((16,2))]
                annot['visible'] += [np.zeros(16)]
                annot['normalize'] += [1]
                annot['torsoangle'] += [0]
                if trainRef[idx] == 0:  # Test image
                    annot['istrain'] += [0]
                else:   # Training image (something missing in annot)
                    annot['istrain'] += [2]

print("")

with h5py.File('mpii-annot.h5','w') as f:
    f.attrs['name'] = 'mpii'
    for k in keys:
        f[k] = np.array(annot[k])

max_stars_repo_name: ebartrum/AlphaPose

max_issues_repo_name: XiaoqingWang/AlphaPose

max_forks_repo_name: XiaoqingWang/AlphaPose


text:
from keras.applications.vgg16 import VGG16
from keras.applications.vgg19 import VGG19
from keras.applications.resnet50 import ResNet50
from keras.applications.densenet import DenseNet121
from keras.applications.xception import Xception
from keras.preprocessing import image
import os, pandas as pd
import numpy as np
from sklearn.cluster import KMeans
class Model:
    def __init__(self,model_name):
        self.model=model_name
    def build_model(self):
        if self.model=='Xception' or self.model=='xception':
            return Xception(include_top=False,pooling='avg',weights='imagenet'),'Xception'
        elif self.model=='DenseNet' or self.model=='densenet':
            return DenseNet121(include_top=False,pooling='avg',weights='imagenet'),'DenseNet'
        elif self.model=='ResNet' or self.model=='resnet':
            return ResNet50(include_top=False,pooling='avg',weights='imagenet'),'ResNet'
        elif self.model=='VGG16' or self.model=='vgg16':
            return VGG16(include_top=False,pooling='avg',weights='imagenet'),'VGG16'
        return VGG19(include_top=False,pooling='avg',weights='imagenet'),'VGG19'

def get_features(img_folder,Model):
    fp1=pd.DataFrame(columns=['filename','feature'])
    fp2 = pd.DataFrame(columns=['filename', 'feature'])
    for i in os.listdir(img_folder):#stage1 stage2
        next_folder=os.path.join(img_folder,i)
        for j in os.listdir(next_folder):#Belt st1...
            next_folder_plus=os.path.join(next_folder,j)
            for h in os.listdir(next_folder_plus):
                path=os.path.join(next_folder_plus,h)
                print(path)
                img=image.load_img(path,target_size=(224,224))
                x=image.img_to_array(img)
                x=np.expand_dims(x,axis=0)
                features=Model[0].predict(x)[0]
                fp1=fp1.append({'filename':path,'feature':features},ignore_index=True)

                img = image.load_img(path)
                x = image.img_to_array(img)
                x = np.expand_dims(x, axis=0)
                features = Model[0].predict(x)[0]
                fp2 = fp2.append({'filename': path, 'feature': features}, ignore_index=True)

                print(path+' finished!!')
    fp1.to_csv('data/{}_features_fixed_size.csv'.format(Model[1]),index=False)
    fp2.to_csv('data/{}_features_default_size.csv'.format(Model[1]),index=False)
    print('ALL DONE!')

class ImageCluster(object):
    def __init__(self,base_img_folder,resorted_img_folder,
                 cluster_algo='kmeans',base_model='vgg16',k=None,maxK=None,csv_file_path=None):
        self.base_model, self.base_model_name = Model(model_name=base_model).build_model()
        self.csv_file_path=csv_file_path
        self.cluster_algo=cluster_algo
        self.k=k
        self.maxK=maxK

        self.base_img_folder=base_img_folder
        self.resorted_img_folder=resorted_img_folder
    def get_feature_map(self,resize_shape=None):
        """
        You can fix the code as your actual situation and environment!!!
        设置图片文件夹一定是3层的，
        base_model_folder
            ----label
                ----xxxxxxxx.jpg
        :param resize_shape:
        :return:
        """
        img_path_all=[]
        f=pd.DataFrame(columns=['filename','feature'])
        if resize_shape==None:
            if os.path.isdir(self.base_img_folder):
                for i in os.listdir(self.base_img_folder):
                    next_path=os.path.join(os.path.join(self.base_img_folder,i))
                    if os.path.isdir(next_path):
                        for j in os.listdir(next_path):
                            last_path = os.path.join(next_path, j)
                            for w in os.listdir(last_path):
                                img_path = os.path.join(last_path, w)
                                img_path_all.append(img_path)
                                img = image.load_img(img_path, target_size=resize_shape)
                                x = image.img_to_array(img)
                                x = np.expand_dims(x, axis=0)
                                features = self.base_model.predict(x)[0]
                                f = f.append({'filename': img_path, 'feature': features}, ignore_index=True)
                    else:
                        pass

            else:
                raise ValueError('the base image folder is wrong! PLZ check it out')
        else:
            if os.path.isdir(self.base_img_folder):
                for i in os.listdir(self.base_img_folder):
                    next_path=os.path.join(os.path.join(self.base_img_folder,i))
                    if os.path.isdir(next_path):
                        for j in os.listdir(next_path):
                            last_path = os.path.join(next_path, j)
                            for w in os.listdir(last_path):
                                img_path=os.path.join(last_path,w)
                                img_path_all.append(img_path)
                                img = image.load_img(img_path, target_size=resize_shape)
                                x = image.img_to_array(img)
                                x = np.expand_dims(x, axis=0)
                                features = self.base_model.predict(x)[0]
                                f = f.append({'filename': img_path, 'feature': features}, ignore_index=True)
                                print(img_path,' extracted features')

                    else:
                        pass

            else:
                raise ValueError('the base image folder is wrong! PLZ check it out')

        if len(img_path_all)==0:
            raise ValueError('image loading fails,please check you image path!')
        else:
            print('Have got the feature map for each image')
            f.to_csv('output/base_model_{}_feature_maps.csv'.format(self.base_model_name))
            print('output/base_model_{}_feature_maps.csv has finished!'.format(self.base_model_name))

    def kmeans(self):
        x=[]
        if self.csv_file_path == None:
            self.csv_file = pd.read_csv('output/base_model_{}_feature_maps.csv'.format(self.base_model_name))
        else:
            self.csv_file = pd.read_csv(self.csv_file_path)
        for i in self.csv_file['feature']:
            x.append([float(t) for t in i.strip('[').strip(']').split(' ')])
        x=np.array(x)
        if os.path.exists('output'):
            pass
        else:
            os.mkdir('output')

        if os.path.exists('matplot'):
            pass
        else:
            os.mkdir('matplot')

        def func(k):
            model = KMeans(n_clusters=k, init='k-means++')
            model.fit(x)
            # print('cluster_center', model.cluster_centers_)
            f = pd.DataFrame(columns=['filename', 'label'])
            f['filename'] = self.csv_file['filename']
            f['label'] = model.labels_
            f.to_csv('output/base_model_{}_cluster_kmeans_{}.csv'.format(self.base_model_name,str(k)))
            return model.inertia_

        if self.k==None:
            sse=[]
            for k in range(51,self.maxK+1):
                sse.append(func(k))
            import matplotlib.pyplot as plt
            plt.plot(range(2,self.maxK+1),sse,marker='o')
            plt.xlabel('number of K(cluster)')
            plt.ylabel('SSE Value for each K')
            plt.title('KMeans for ImageCluster')
            plt.savefig('matplot/base_model_{}_KMeans_maxK_{}.png'.format(self.base_model_name,str(self.maxK)))
            plt.show()
        else:
            func(self.k)
    def imagecluster(self):
        if self.cluster_algo.lower()=='kmeans':
            self.kmeans()
        else:
            print('no existing cluster algorithm')
            return

    def resorted_img(self,selected_k_num):
        import shutil
        if os.path.exists(self.resorted_img_folder):
            pass
        else:
            os.mkdir(self.resorted_img_folder)

        resorted_csv=pd.read_csv('output/base_model_{}_cluster_kmeans_{}.csv'.format(self.base_model_name,str(selected_k_num)))
        for i in resorted_csv.index:
            filename=resorted_csv.loc[i,'filename']
            label=resorted_csv.loc[i,'label']
            if os.path.exists(os.path.join(self.resorted_img_folder,str(label))):
                pass
            else:
                os.mkdir(os.path.join(self.resorted_img_folder,str(label)))
            shutil.copy(filename,os.path.join(self.resorted_img_folder,str(label)))
            print(os.path.join(self.resorted_img_folder,str(label))+'\\'+filename+' Copied！')

max_stars_repo_name: TomHacker/ImageCluster

max_issues_repo_name: HandsomeBrotherShuaiLi/ImageCluster

max_forks_repo_name: HandsomeBrotherShuaiLi/ImageCluster


text:
"""
This module contains the tests for the spokestack text to speech manager
"""
import pytest
from unittest import mock

import numpy as np

from spokestack.tts.manager import SequenceIO, TextToSpeechManager, FORMAT_PCM16

def test_invalid():
    client = mock.MagicMock()
    output = mock.MagicMock()
    with pytest.raises(ValueError):
        TextToSpeechManager(client, output, format_="invalid")

def test_synthesize_mp3():
    client = mock.MagicMock()
    client.synthesize.return_value = np.zeros(100000, np.int16).tobytes()
    output = mock.MagicMock()

    with mock.patch("spokestack.tts.manager.MP3Decoder") as patched:
        patched.return_value = np.zeros(160).tobytes()
        manager = TextToSpeechManager(client, output)
        manager.synthesize(utterance="test utterance")

def test_synthesize_raw():
    audio = np.zeros(16000, np.int16)

    client = mock.MagicMock()
    client.synthesize.return_value = [audio]
    output = mock.MagicMock()

    manager = TextToSpeechManager(client, output, format_=FORMAT_PCM16)
    manager.synthesize(utterance="test utterance")

    output.write.assert_called_with(audio.tobytes())

def test_close():
    client = mock.MagicMock()
    output = mock.MagicMock()
    manager = TextToSpeechManager(client, output)
    manager.close()

    assert not manager._client
    assert not manager._output

def test_sequence_io():
    test = (np.ones(1000, np.int16).tobytes() for i in range(10))
    stream = SequenceIO(test)
    for _ in test:
        stream.read()
    # read after StopIteration
    stream.read()

max_stars_repo_name: spokestack/spokestack-python

max_issues_repo_name: spokestack/spokestack-python

max_forks_repo_name: spokestack/spokestack-python


text:
#!/usr/bin/env python
# coding: utf-8

# ## Plot compressed mutation prediction results
#
# Here, we'll look at the results of our mutation prediction experiments with compressed (i.e. PCA projected) data. The idea is similar to `01_classify_stratified/plot_mutation_results.ipynb`, but with slightly more parameters (e.g. number of PCA components).
#
# Notebook parameters:
# * SIG_ALPHA (float): significance cutoff (after FDR correction)

# In[1]:

from pathlib import Path

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from adjustText import adjust_text

import mpmp.config as cfg
import mpmp.utilities.analysis_utilities as au

# In[2]:

# set results directory
results_dir = Path(cfg.results_dirs['mutation'],
                   'compressed_results', 'gene').resolve()
# set significance cutoff after FDR correction
SIG_ALPHA = 0.001

# In[3]:

# load raw data
results_df = au.load_compressed_prediction_results(results_dir,
                                                   'gene',
                                                   old_filenames=True)
print(results_df.shape)
results_df.head()

# In[4]:

# get dataframes comparing each data type to their shuffled baseline
compare_df = pd.DataFrame()
for n_dims in results_df.n_dims.unique():
    for training_data_type in results_df.training_data.unique():
        data_df = results_df[
            (results_df.training_data == training_data_type) &
            (results_df.n_dims == n_dims)
        ]
        # note that we're doing FDR correction separately for each n_dims
        # and training data type
        data_results_df = au.compare_results(data_df,
                                             identifier='identifier',
                                             metric='aupr',
                                             correction=True,
                                             correction_method='fdr_bh',
                                             correction_alpha=SIG_ALPHA,
                                             verbose=False)
        data_results_df['training_data'] = training_data_type
        data_results_df['n_dims'] = n_dims
        compare_df = pd.concat((compare_df, data_results_df))
compare_df.head()

# In[5]:

def label_points(x, y, gene, sig, ax):
    text_labels = []
    a = pd.DataFrame({'x': x, 'y': y, 'gene': gene, 'sig': sig})
    for i, point in a.iterrows():
        if point['y'] > -np.log10(SIG_ALPHA):
            text_labels.append(
                ax.text(point['x'], point['y'], str(point['gene']))
            )
    return text_labels

compare_df = (
    compare_df.rename(columns={'identifier': 'gene'})
              .sort_values(by=['n_dims', 'training_data'])
)
compare_df['nlog10_p'] = -np.log10(compare_df.corr_pval)

sns.set({'figure.figsize': (20, 16)})
sns.set_style('whitegrid')
fig, axarr = plt.subplots(3, 3)

# plot mutation prediction from various data types, in a volcano-like plot
for row_ix, n_dims in enumerate(compare_df.n_dims.unique()):
    for col_ix, training_data_type in enumerate(compare_df.training_data.unique()):
        ax = axarr[row_ix, col_ix]
        data_df = compare_df[
            (compare_df.training_data == training_data_type) &
            (compare_df.n_dims == n_dims)
        ]
        sns.scatterplot(data=data_df, x='delta_mean', y='nlog10_p', hue='reject_null',
                        hue_order=[False, True], ax=ax)
        # add vertical line at 0
        ax.axvline(x=0, linestyle='--', color='black', linewidth=1.25)
        # add horizontal line at statistical significance threshold
        l = ax.axhline(y=-np.log10(SIG_ALPHA), linestyle='--', linewidth=1.25)
        # label horizontal line with significance threshold
        # (matplotlib makes this fairly difficult, sadly)
        ax.text(0.9, -np.log10(SIG_ALPHA)+0.2,
                r'$\mathbf{{\alpha = {}}}$'.format(SIG_ALPHA),
                va='center', ha='center', color=l.get_color(),
                backgroundcolor=ax.get_facecolor())
        ax.set_xlabel('AUPR(signal) - AUPR(shuffled)')
        ax.set_ylabel(r'$-\log_{10}($adjusted $p$-value$)$')
        ax.set_xlim((-0.2, 1.0))
        y_max = max(compare_df.nlog10_p)
        ax.set_ylim((0, y_max+2))
        ax.legend(title=r'Reject $H_0$', loc='upper left')
        ax.set_title(r'{} PCs, {} data ({}/{} significant genes)'.format(
            n_dims, training_data_type,
            np.count_nonzero(data_df.reject_null),
            data_df.shape[0]
        ), size=13, pad=10)

        # label genes and adjust text to not overlap
        # automatic alignment isn't perfect, can align by hand in inkscape if necessary
        text_labels = label_points(data_df['delta_mean'],
                                   data_df['nlog10_p'],
                                   data_df.gene,
                                   data_df.reject_null,
                                   ax)
        adjust_text(text_labels,
                    ax=ax,
                    expand_text=(1., 1.),
                    lim=5)

plt.suptitle('Mutation prediction, signal vs. shuffled results', size=16)
plt.tight_layout(w_pad=2, h_pad=2)
plt.subplots_adjust(top=0.94)

# In[6]:

sns.set({'figure.figsize': (18, 6)})
fig, axarr = plt.subplots(1, 2)

# plot mean performance over all genes in Vogelstein dataset
# sns.pointplot(data=compare_df, x='n_dims', y='delta_mean', hue='training_data', ax=axarr[0])
sns.boxplot(data=compare_df, x='n_dims', y='delta_mean', hue='training_data', ax=axarr[0])
axarr[0].set_title('Mean mutation prediction, performance vs. PCA components')
axarr[0].set_xlabel('Number of PCA components')
axarr[0].set_ylabel('AUPR(signal) - AUPR(shuffled)')
axarr[0].set_ylim(-0.3, max(compare_df.delta_mean + 0.05))

# plot mean performance for genes that are significant for at least one data type
gene_list = compare_df[compare_df.reject_null == True].gene.unique()
sns.boxplot(data=compare_df[compare_df.gene.isin(gene_list)], x='n_dims', y='delta_mean', hue='training_data', ax=axarr[1])
# sns.pointplot(data=compare_df[compare_df.gene.isin(gene_list)],
#               x='n_dims', y='delta_mean', hue='training_data', ax=axarr[1])
axarr[1].set_title('Mean prediction for sig genes only, performance vs. PCA components')
axarr[1].set_xlabel('Number of PCA components')
axarr[1].set_ylabel('AUPR(signal) - AUPR(shuffled)')
axarr[1].set_ylim(-0.2, max(compare_df.delta_mean + 0.05))

# In[7]:

# get results for full gene expression/27k methylation datasets
# we want to compare compressed to full results below
raw_results_dir = Path(cfg.results_dirs['mutation'], 'mutation_imputed_n10_i5', 'gene').resolve()
raw_results_df = au.load_stratified_prediction_results(raw_results_dir, 'gene')

raw_expression_df = (
    raw_results_df[raw_results_df.training_data == 'expression']
        .drop(columns=['training_data'])
)
raw_expression_results_df = au.compare_results(raw_expression_df,
                                               identifier='identifier',
                                               metric='aupr',
                                               correction=True,
                                               correction_method='fdr_bh',
                                               correction_alpha=SIG_ALPHA,
                                               verbose=False)
raw_expression_results_df.rename(columns={'identifier': 'gene'}, inplace=True)

raw_methylation_df = (
    raw_results_df[raw_results_df.training_data == 'methylation']
        .drop(columns=['training_data'])
)
raw_methylation_results_df = au.compare_results(raw_methylation_df,
                                                identifier='identifier',
                                                metric='aupr',
                                                correction=True,
                                                correction_method='fdr_bh',
                                                correction_alpha=SIG_ALPHA,
                                                verbose=False)
raw_methylation_results_df.rename(columns={'identifier': 'gene'}, inplace=True)

# In[8]:

# line plot of performance over increasing number of PCs
# plot dotted line for expression/me performance with all genes?
gene = 'IDH1'
sns.set({'figure.figsize': (8, 5)})
gene_df = compare_df[compare_df.gene == gene]
sns.pointplot(data=gene_df, x='n_dims', y='delta_mean', hue='training_data')
raw_expression_val = (
    raw_expression_results_df[raw_expression_results_df.gene == gene]
        .delta_mean.values[0]
)
plt.axhline(y=raw_expression_val, linestyle='--', color=sns.color_palette()[0])
raw_me_27k_val = (
    raw_methylation_results_df[raw_methylation_results_df.gene == gene]
        .delta_mean.values[0]
)
plt.axhline(y=raw_me_27k_val, linestyle='--', color=sns.color_palette()[1])
plt.title('{} mutation prediction, performance vs. PCA components'.format(gene))
plt.xlabel('Number of PCA components')
plt.ylabel('AUPR(signal) - AUPR(shuffled)')
plt.ylim(0, max(max(gene_df.delta_mean), raw_expression_val, raw_me_27k_val)+0.05)

# In[9]:

def label_points(x, y, gene, sig, ax):
    text_labels = []
    a = pd.DataFrame({'x': x, 'y': y, 'gene': gene, 'sig': sig})
    for i, point in a.iterrows():
        if point['y'] > 1.0:
            text_labels.append(
                ax.text(point['x'], point['y'], str(point['gene']))
            )
    return text_labels

# plot comparisons for all pairwise combinations of training datasets,
# within each choice of compression dimension
import itertools as it

sns.set({'figure.figsize': (20, 16)})
sns.set_style('whitegrid')
fig, axarr = plt.subplots(3, 3)

results_df.sort_values(by=['n_dims', 'training_data'], inplace=True)
for row_ix, n_dims in enumerate(results_df.n_dims.unique()):
    for col_ix, (train_1, train_2) in enumerate(
            it.combinations(results_df.training_data.unique(), 2)):
        train_1_df = results_df[
            (results_df.n_dims == n_dims) &
            (results_df.training_data == train_1)
        ]
        train_2_df = results_df[
            (results_df.n_dims == n_dims) &
            (results_df.training_data == train_2)
        ]
        compare_train_df = au.compare_results(train_1_df,
                                              train_2_df,
                                              identifier='identifier',
                                              metric='aupr',
                                              correction=True,
                                              correction_method='fdr_bh',
                                              correction_alpha=SIG_ALPHA,
                                              verbose=False)
        compare_train_df.rename(columns={'identifier': 'gene'}, inplace=True)
        compare_train_df['nlog10_p'] = -np.log10(compare_train_df.corr_pval)

        ax = axarr[row_ix, col_ix]
        sns.scatterplot(data=compare_train_df, x='delta_mean', y='nlog10_p', hue='reject_null',
                        hue_order=[False, True], ax=ax)
        # add vertical line at 0
        ax.axvline(x=0, linestyle='--', color='black', linewidth=1.25)
        # add horizontal line at statistical significance threshold
        l = ax.axhline(y=-np.log10(SIG_ALPHA), linestyle='--', linewidth=1.25)
        # label horizontal line with significance threshold
        # (matplotlib makes this fairly difficult, sadly)
        ax.text(0.6, -np.log10(SIG_ALPHA)+0.01,
                r'$\mathbf{{\alpha = {}}}$'.format(SIG_ALPHA),
                va='center', ha='center', color=l.get_color(),
                backgroundcolor=ax.get_facecolor())
        # NOTE compare_results function takes df2 - df1, so we have to invert them here
        ax.set_xlabel('AUPR({}) - AUPR({})'.format(train_2, train_1))
        ax.set_ylabel(r'$-\log_{10}($adjusted $p$-value$)$')
        ax.set_xlim((-0.75, 0.75))
        ax.set_ylim((0, 7))
        ax.legend(title=r'Reject $H_0$', loc='upper left')
        ax.set_title(r'{} PCs, {} vs. {}'.format(n_dims, train_2, train_1),
                     size=13, pad=10)

        # label genes and adjust text to not overlap
        # automatic alignment isn't perfect, can align by hand in inkscape if necessary
        text_labels = label_points(compare_train_df['delta_mean'],
                                   compare_train_df['nlog10_p'],
                                   compare_train_df.gene,
                                   compare_train_df.reject_null,
                                   ax)
        adjust_text(text_labels,
                    ax=ax,
                    expand_text=(1., 1.),
                    lim=5)

plt.suptitle('Mutation prediction, comparison of data types', size=16)
plt.tight_layout(w_pad=2, h_pad=2)
plt.subplots_adjust(top=0.94)


text:
import argparse
import os
import numpy as np
import time

from z2_sim.src.QuantumCircuits.Cirq_Code import production
from z2_sim.src.QuantumCircuits.Cirq_Code import io
from z2_sim.src.QuantumCircuits.Cirq_Code.noise.zz_crosstalk_v1 import ZZCrossTalkV1
from z2_sim.src.QuantumCircuits.Cirq_Code.noise.two_local_depol import TwoLocalDepol

import qsimcirq

parser = argparse.ArgumentParser(add_help=True)
parser.add_argument('-resource', metavar='resource', type=str, nargs=1,
                    help='string tag describing this device')
args = parser.parse_args()

resource = args.resource[0]
TROTTER_STEPS = 10
jcoup = 0.8
dt = 0.25

# Initialize simulators and noise model
N_FUSE = 4
# # # # # # # # # # # # # # # # # # # # # # # # #
# CHECK THIS BEFOR RUNNING LOCALLY
N_THREADS = 160
n = 6
# # # # # # # # # # # # # # # # # # # # # # # # #

qsim_options = qsimcirq.QSimOptions(
    max_fused_gate_size = N_FUSE,
    cpu_threads = N_THREADS,
    denormals_are_zeros = True,
)

# "SIM A" ancillafree observable computation using 4x 36-qubit circuit
# The outermost loop in this routine is over the four distinct circuits
# So the first 25% of debug statement timing results will correspond
# to a single choice of initial generator
out, times_out = production.compute_ancillafree_obs_with_intermediate_state_vector(
    n=n,
    trotter_steps=TROTTER_STEPS,
    jcoup=jcoup,
    dt=dt,
    qsim_options=qsim_options,
    all_observables=True,
    obc=True,
    decompbasis="MS",
    verbose=True,
)

# OLD simulation using extra ancilla qubit.
# This has been updated to only compute 21/36 observables.
# out, times_out = production.compute_obs_with_intermediate_state_vector(
#     n=n,
#     trotter_steps=TROTTER_STEPS,
#     jcoup=jcoup,
#     dt=dt,
#     qsim_options=qsim_options,
#     all_observables=True,
#     obc=True,
#     decompbasis="MS",
#     verbose=True,
# )

np.save(f"benchmark_{resource}_trotter{TROTTER_STEPS}.npy", times_out)


text:
from tartan_plugin import *
from data import *
from model import *
from torch.optim import Adam
import numpy as np
import random

def set_random_seed(seed=0):
	# Esp important for ensuring deterministic behavior with CNNs
	torch.backends.cudnn.deterministic = True
	np.random.seed(seed)
	random.seed(seed)
	torch.manual_seed(seed)
	cuda_available = torch.cuda.is_available()
	if cuda_available:
		torch.cuda.manual_seed_all(seed)
	return cuda_available

# Get the model
def get_model(output_classes_dict):
	model = WideResnet(output_classes_dict)
	model.cuda()
	return model

# Instantiate the optimizer
def get_optimizer(model, lr=1e-3):
	optimizer = Adam(model.parameters(), lr=lr)
	return optimizer

def run_epoch(model, dataset, optim, bsz, target_head='target', tartan_=None, mdev_head=None, dev_data=None):
	data_itr = iterator_(dataset[0], bsz)
	dev_head_itr = None if tartan_ is None else iterator_(dev_data, 16) # Iterator for fitting the dev head
	aux_itr = None if tartan_ is None else iterator_(dataset[1], bsz) # Iterator for the auxiliary data
	total_correct, total_egs, total_loss = 0.0, 0.0, 0.0

	for batch_ in data_itr:
		xs, ys = batch_
		model_outs = model(xs, head_name=target_head)
		loss_, acc_stats = model.criterion(model_outs, ys), model.accuracy(model_outs, ys)
		if aux_itr:
			# Get the target gradients
			target_grads = torch.autograd.grad(loss_, model.parameters(), allow_unused=True)

			# Get the auxiliary gradients
			# This is the simple case where there is only 1 auxiliary loss.
			# If there are n, auxiliary losses then we would need n iterators.
			try:
				aux_xs, aux_ys = next(aux_itr)
			except:
				aux_itr = iterator_(dataset[1], bsz)
				aux_xs, aux_ys = next(aux_itr)

			aux_loss_ = model.criterion(model(aux_xs, head_name='aux'), aux_ys)
			aux_grads = torch.autograd.grad(aux_loss_, model.parameters(), allow_unused=True)

			all_gradients = [target_grads, aux_grads]
			# Fit the dev-head
			if mdev_head:
				try:
					dev_batch = next(dev_head_itr)
					meta_batch = next(dev_head_itr)
				except:
					dev_head_itr = None if tartan_ is None else iterator_(dev_data, 16)
					dev_batch = next(dev_head_itr)
					meta_batch = next(dev_head_itr)

				mdev_head.learn_meta_dev_head(dev_batch, model.loss_fn)
				meta_grads = mdev_head.get_meta_grads(meta_batch, model.loss_fn)

				tartan_.update_weight_gradients(all_gradients, meta_grads)

			# Get the combined gradients
			combo_grads = tartan_.weighted_grads(all_gradients)
			# Set the combined gradients
			with torch.no_grad():
				for idx_, param in enumerate(model.parameters()):
					if combo_grads[idx_] is not None:
						param.grad = combo_grads[idx_]

			tartan_.update_weights()
		else:
			# There is no auxiliary data. Just do a normal backward
			loss_.backward()

		total_correct += acc_stats[0][0]
		total_egs += acc_stats[0][1]
		total_loss += loss_ *  acc_stats[0][1]
		if optim:
			optim.step()
			optim.zero_grad()

	return (total_loss / total_egs, total_correct / total_egs)

def graph_task_weights(task_names, weights, name):
	import matplotlib.pyplot as plt

	all_weights = np.stack(weights)
	for i_ in range(all_weights.shape[-1]):
		plt.plot(all_weights[:, i_], label=task_names[i_])
	plt.legend()
	plt.savefig("{}.png".format(name))

def run_tartan(model, data, task_names, bsz=64, n_epochs=30, is_meta_tartan=True):
	# Instantiate the optimizer and the tartan trainer
	optim = get_optimizer(model, lr=3e-4)
	tartan_trainer = TartanTrainer(task_names, is_meta_tartan=is_meta_tartan)
	meta_dev_head = None
	if is_meta_tartan:
		# Setup the dev head if doing meta-tartan
		dev_head = model.get_new_head(NUM_TARGET_CLASSES, 'dev_head')
		meta_dev_head = MetaDevHead(model, dev_head)

	dev_accs, test_accs = [], []
	for epoch in range(n_epochs):
		train_stats = run_epoch(model, data.train, optim, bsz, tartan_=tartan_trainer, mdev_head=meta_dev_head, dev_data=data.dev[0])

		dev_stats = run_epoch(model, data.dev, None, bsz)
		test_stats = run_epoch(model, data.test, None, bsz)
		print('Epoch {}: Train Loss {:.3f}, Train Ac {:.3f}'.format(epoch, *train_stats))
		print('        : Dev   Loss {:.3f}, Dev   Ac {:.3f}'.format(*dev_stats))
		print('        : Test  Loss {:.3f}, Test  Ac {:.3f}'.format(*test_stats))

		dev_accs.append(dev_stats[-1])
		test_accs.append(test_stats[-1])

	if is_meta_tartan:
		graph_task_weights(task_names, tartan_trainer.old_weights, 'meta-tartan')
	return test_accs[np.argmax(dev_accs)]

if __name__ == "__main__":
	data = CIFAR100_MSMammals()
	output_classes_dict = {
							'target': NUM_TARGET_CLASSES,
							'aux': NUM_AUX_CLASSES
					}

	set_random_seed()
	model = get_model(output_classes_dict)
	mttartan_testscore = run_tartan(model, data, ['target', 'aux'], is_meta_tartan=False)

	set_random_seed()
	model = get_model(output_classes_dict)
	metatartan_testscore = run_tartan(model, data, ['target', 'aux'], is_meta_tartan=True)

	print('Test Score | Multitasking-TARTAN   : {:.3f}'.format(mttartan_testscore))
	print('Test Score | Meta-Learning-TARTAN  : {:.3f}'.format(metatartan_testscore))

max_stars_repo_name: ldery/TARTAN

max_issues_repo_name: ldery/TARTAN

max_forks_repo_name: ldery/TARTAN


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
(c) 2014 Brant Faircloth || http://faircloth-lab.org/
All rights reserved.

This code is distributed under a 3-clause BSD license. Please see
LICENSE.txt for more information.

Created on 26 June 2014 17:17 PDT (-0700)
"""

import os
import re
import gzip
import glob
import numpy
import subprocess
from collections import OrderedDict

from phyluce.pth import get_user_param, get_user_path

from Bio import SeqIO

def coverage(log, sample, assembly_pth, assembly, cores, bam):
    log.info("Computing coverage with GATK for {}".format(sample))
    cwd = os.getcwd()
    # move into reference directory
    os.chdir(assembly_pth)
    cmd = [
        get_user_path("binaries", "gatk"),
        "-T",
        "DepthOfCoverage",
        "-R",
        assembly,
        "-I",
        bam,
        "-o",
        "{}-coverage".format(sample),
        "-nt",
        str(cores),
        "--omitIntervalStatistics",
        "--omitLocusTable"
    ]
    gatk_coverage_fname = os.path.join(assembly_pth, '{}.GATK-coverage-out.log'.format(sample))
    with open(gatk_coverage_fname, 'w') as gatk_out:
        proc = subprocess.Popen(cmd, stdout=gatk_out, stderr=subprocess.STDOUT)
        proc.communicate()
    os.chdir(cwd)
    return os.path.join(assembly_pth, "{}-coverage".format(sample))

def compute_coverage_metrics(contig_depth, trim=False):
    metadata = {
        "beginning-length":None,
        "beginning-mean-cov": None,
        "contig-start": None,
        "contig-end": None,
        "trim-start":None,
        "trim-end":None,
        "ending-length":None,
        "ending-mean-cov":None
    }
    depth_array = numpy.array(contig_depth)
    metadata["beginning-length"] = len(depth_array)
    metadata["beginning-mean-cov"] = numpy.round(numpy.mean(depth_array), 2)
    metadata["contig-start"] = 0
    metadata["contig-end"] = len(depth_array)
    if trim:
        # find where coverage is >= 3
        min_cov = depth_array >= 3
        # get positions at edges where True
        true_positions = numpy.where(min_cov==True)
        try:
            metadata["trim-start"] = true_positions[0][0]
            metadata["trim-end"] = true_positions[0][-1] + 1
            good = depth_array[metadata["trim-start"]:metadata["trim-end"]]
        except IndexError as err:
            if err.message == 'index out of bounds':
                good = None
            else:
                raise err
        if good is not None:
            metadata["ending-length"] = len(good)
            metadata["ending-mean-cov"] = numpy.round(numpy.mean(good), 2)
        else:
            metadata["ending-length"] = None
            metadata["ending-mean-cov"] = None
    else:
        metadata["trim-start"] = None
        metadata["trim-end"] = None
        metadata["ending-length"] = len(depth_array)
        metadata["ending-mean-cov"] = numpy.round(numpy.mean(depth_array), 2)
    return metadata

def get_trimmed_coverage_from_output(log, sample, assembly_pth, coverage, assembler):
    log.info("Screening and filtering contigs for coverage (3x ends, 5x avg.)")
    if assembler == "trinity" or assembler == "trinity2":
        regex = re.compile("({}).*:(\d+)".format(get_user_param("headers", "trinity")))
    elif assembler == "velvet":
        regex = re.compile("({}.*):(\d+)".format(get_user_param("headers", "velvet")))
    elif assembler == "abyss":
        regex = re.compile("({}.*):(\d+)".format(get_user_param("headers", "abyss")))
    elif assembler == "idba":
        regex = re.compile("({}.*):(\d+)".format(get_user_param("headers", "idba")))
    elif assembler == "spades":
        regex = re.compile("({}.*):(\d+)".format(get_user_param("headers", "spades")))
    # setup starting values
    previous_match = None
    contig_depth = []
    contig_data = OrderedDict()
    overall_coverage = []
    overall_length = []
    overall_count = 1
    overall_contigs = {}
    pbc = os.path.join(
        assembly_pth,
        '{}-TRIMMED-per-base-coverage.txt.gz'.format(sample)
    )
    pcc = os.path.join(
        assembly_pth,
        '{}-TRIMMED-per-contig-coverage.txt'.format(sample)
    )
    upcc = os.path.join(
        assembly_pth,
        '{}-UNTRIMMED-per-contig-coverage.txt'.format(sample)
    )
    with open(coverage, 'rU') as infile:
        with gzip.open(pbc, 'w') as per_base_cov:
            with open(pcc, 'w') as per_contig_cov:
                with open(upcc, 'w') as unt_per_contig_cov:
                    # read header line
                    gatk_header = infile.readline()
                    # write headers to outfiles
                    per_contig_cov.write("name\tbeginning-length\tbeginning-mean-cov\ttrim-start\ttrim-end\tend-length\tend-mean-cov\n")
                    unt_per_contig_cov.write("name\tbeginning-length\tbeginning-mean-cov\n")
                    per_base_cov.write(gatk_header)
                    for line in infile:
                        ls = line.split()
                        search = regex.search(ls[0])
                        match_name, pos = search.groups()
                        if previous_match is None or match_name == previous_match:
                            # hold onto current match_name
                            previous_match = match_name
                            # compute metrics on current position
                            contig_data[int(pos)] = line
                            contig_depth.append(int(ls[1]))
                        elif match_name != previous_match:
                            metadata = compute_coverage_metrics(contig_depth, trim=True)
                            unt_per_contig_cov.write("{}\t{}\t{}\n".format(
                                    previous_match,
                                    metadata["beginning-length"],
                                    metadata["beginning-mean-cov"]
                                ))
                            if metadata["ending-mean-cov"] >= 5.0:
                                per_contig_cov.write("{}\t{}\t{}\t{}\t{}\t{}\t{}\n".format(
                                    previous_match,
                                    metadata["beginning-length"],
                                    metadata["beginning-mean-cov"],
                                    metadata["trim-start"],
                                    metadata["trim-end"],
                                    metadata["ending-length"],
                                    metadata["ending-mean-cov"]
                                ))
                                for pos, line in contig_data.iteritems():
                                    if pos-1 >= metadata["trim-start"] and pos-1 < metadata["trim-end"]:
                                        per_base_cov.write(line)
                                overall_contigs[previous_match] = metadata
                                overall_count += 1
                                overall_coverage.append(metadata["ending-mean-cov"])
                                overall_length.append(metadata["ending-length"])
                            # reset previous match to current
                            previous_match = match_name
                            # reset containers
                            contig_depth = []
                            contig_data = OrderedDict()
                            # compute metrics on current first position
                            contig_data[int(pos)] = line
                            contig_depth.append(int(ls[1]))
    log.info("\t{} contigs, mean coverage = {:.1f}, mean length = {:.1f}".format(
        overall_count,
        numpy.mean(overall_coverage),
        numpy.mean(overall_length)
    ))
    return overall_contigs

def get_untrimmed_coverage_from_output(log, sample, assembly_pth, coverage, assembler):
    log.info("Screening contigs for coverage")
    if assembler == "trinity" or assembler == "trinity2":
        regex = re.compile("({}).*:(\d+)".format(get_user_param("headers", "trinity")))
    elif assembler == "velvet":
        regex = re.compile("({}.*):(\d+)".format(get_user_param("headers", "velvet")))
    elif assembler == "abyss":
        regex = re.compile("({}.*):(\d+)".format(get_user_param("headers", "abyss")))
    elif assembler == "idba":
        regex = re.compile("({}.*):(\d+)".format(get_user_param("headers", "idba")))
    elif assembler == "spades":
        regex = re.compile("({}.*):(\d+)".format(get_user_param("headers", "spades")))
    # setup starting values
    previous_match = None
    contig_depth = []
    contig_data = OrderedDict()
    overall_coverage = []
    overall_length = []
    overall_count = 1
    overall_contigs = {}
    upcc = os.path.join(
        assembly_pth,
        '{}-UNTRIMMED-per-contig-coverage.txt'.format(sample)
    )
    with open(coverage, 'rU') as infile:
        with open(upcc, 'w') as unt_per_contig_cov:
            # read header line
            gatk_header = infile.readline()
            # write headers to outfiles
            unt_per_contig_cov.write("name\tbeginning-length\tbeginning-mean-cov\n")
            for line in infile:
                ls = line.split()
                search = regex.search(ls[0])
                match_name, pos = search.groups()
                if previous_match is None or match_name == previous_match:
                    # hold onto current match_name
                    previous_match = match_name
                    # compute metrics on current position
                    #contig_data[int(pos)] = line
                    contig_depth.append(int(ls[1]))
                elif match_name != previous_match:
                    metadata = compute_coverage_metrics(contig_depth, trim=False)
                    unt_per_contig_cov.write("{}\t{}\t{}\n".format(
                            previous_match,
                            metadata["beginning-length"],
                            metadata["beginning-mean-cov"]
                        ))
                    overall_contigs[previous_match] = metadata
                    overall_count += 1
                    overall_coverage.append(metadata["beginning-mean-cov"])
                    overall_length.append(metadata["beginning-length"])
                    # reset previous match to current
                    previous_match = match_name
                    # reset containers
                    contig_depth = []
                    contig_data = OrderedDict()
                    # compute metrics on current first position
                    contig_data[int(pos)] = line
                    contig_depth.append(int(ls[1]))
    log.info("\t{} contigs, mean coverage = {:.1f}, mean length = {:.1f}".format(
        overall_count,
        numpy.mean(overall_coverage),
        numpy.mean(overall_length)
    ))
    return overall_contigs

def remove_coverage_files(log, assembly_pth, coverage):
    log.info("Removing GATK coverage files we created (screened files remain)")
    for file in glob.glob("{}*".format(coverage)):
        # gzip coverage file - remove rest
        if file == coverage and not os.path.splitext(coverage)[1] == ".gz":
            log.info("[GZIP] {} (slow)".format(os.path.basename(file)))
            with open(file, "rb") as unzip:
                with gzip.open("{}.gz".format(file), "wb") as zip:
                    zip.writelines(unzip)
            # remove the unzipped file
            os.remove(file)
        else:
            # remove all other files
            log.info("[Remove] {}".format(os.path.basename(file)))
            os.remove(file)

def filter_screened_contigs_from_assembly(log, sample, assembly_pth, assembly, overall_contigs):
    log.info("Filtering assembly FASTAs for coverage")
    keep = set(overall_contigs.keys())
    outf_name = os.path.join(
        assembly_pth,
        "{}-trinity-FILTERED.fasta".format(sample)
    )
    with open(outf_name, 'w') as outf:
        with open(assembly, 'rU') as infile:
            for seq in SeqIO.parse(infile, 'fasta'):
                if seq.id in keep:
                    # get trimming coordinates
                    start = overall_contigs[seq.id]["trim-start"]
                    end = overall_contigs[seq.id]["trim-end"]
                    seq = seq[start:end]
                    seq.description = "len={}".format(len(seq))
                    outf.write(seq.format('fasta'))
    return outf_name


text:
import flywheel
import os
import pandas as pd
import numpy as np
import argparse
import json
from datetime import date, datetime

expectedNumDicomFiles = {
    'dMRI_dir98_AP': 99,
    'dMRI_dir98_AP_SBRef': 1,
    'dMRI_dir98_PA': 99,
    'dMRI_dir98_PA_SBRef': 1,
    'dMRI_dir99_AP': 100,
    'dMRI_dir99_AP_SBRef': 1,
    'dMRI_dir99_PA': 100,
    'dMRI_dir99_PA_SBRef': 1,
    'rfMRI_REST_AP': 420,
    'rfMRI_REST_AP_SBRef': 1,
    'rfMRI_REST_PA': 420,
    'rfMRI_REST_PA_SBRef': 1,
    'SpinEchoFieldMap_AP': 3,
    'SpinEchoFieldMap_PA': 3,
    'SPIRAL_V20_HCP_ASL': 986,
    'SPIRAL_V20_HCP_M0': 68,
    'SPIRAL_V20_HCP_MeanPerf': 34,
    'T1w_MPR': 208,
    'T2w_SPC': 208,
    'tfMRI_GAMBLING_AP': 228,
    'tfMRI_GAMBLING_AP_SBRef': 1,
    'tfMRI_GAMBLING_PA': 228,
    'tfMRI_GAMBLING_PA_SBRef': 1,
    'tfMRI_WM_AP': 365,
    'tfMRI_WM_AP_SBRef': 1,
    'tfMRI_WM_PA': 365,
    'tfMRI_WM_PA_SBRef': 1
    }

fw = flywheel.Client()
project = fw.lookup('pennftdcenter/HCPMultiCenter')

all_info = dict()

for sub in project.subjects():
    sub_id = sub.id
    sub_label = sub.label
    for ses in sub.sessions():
        ses_id = ses.id
        ses_label = ses.label
        for acq in ses.acquisitions():
            acq = acq.reload()
            acq_id = acq.id
            acq_label = acq.label
            acq_series_description = 'NA'
            acq_series_number = 'NA'
            acq_study_date = 'NA'
            acq_found_dicom = False
            acq_num_dicom_files = 0
            acq_missing_info = False
            acq_complete = False
            for f in acq.files:
                if f.type != 'dicom':
                    next
                acq_found_dicom = True
                file_info = f.info
                if (not file_info):
                    acq_missing_info = True

                extension = os.path.splitext(f.name)[1]
                acq_num_dicom_files = 1
                if (extension == '.zip'):
                    acq_num_dicom_files = f.zip_member_count
                try:
                    acq_series_description = file_info['SeriesDescription']
                except KeyError:
                    acq_series_description = 'NA'
                try:
                    acq_series_number = file_info['SeriesNumber']
                except KeyError:
                    acq_series_number = 'NA'
                try:
                    acq_study_date = file_info['StudyDate']
                except KeyError:
                    print('No study date for acquisition {}/{}/{} {}'.format(sub.label, ses.label, acq_label, acq.id))
                    acq_study_date = 'NA'
                break
            acq_ignore = (acq_series_description != 'NA' and not (acq_series_description in expectedNumDicomFiles.keys()))
            if (acq_series_description != 'NA' and (acq_series_description in expectedNumDicomFiles.keys())):
                acq_complete = (acq_num_dicom_files == expectedNumDicomFiles[acq_series_description])
            all_info[acq_id] = [sub_id, sub_label, ses_id, ses_label, acq_label, acq_study_date, acq_series_number, acq_series_description,
                                acq_missing_info, acq_ignore, acq_found_dicom, acq_num_dicom_files, acq_complete]

with open('HCPMultiCenterCompleteness.txt','w') as data:
    data.write(str(all_info))

with open('HCPMultiCenterCompleteness.json', 'w') as convert_file:
    convert_file.write(json.dumps(all_info))

df = pd.DataFrame.from_dict(all_info, orient='index').reset_index()
df.columns=['AcquisitionID', 'SubjectID', 'SubjectLabel', 'SessionID', 'SessionLabel', 'AcquisitionLabel', 'StudyDate', 'SeriesNumber', 'SeriesDescription',
                            'MissingInfo', 'Ignore', 'FoundDicom', 'NumDicomFiles','Complete']

# Export dataframe to csv
filename = f'HCPMultiCenterCompleteness.csv'
df.to_csv(filename,index=False)


text:
import numpy as np

class VerifyParameterType():
    @staticmethod
    def verify(value):
        for cls in __class__.__subclasses__():
            if cls.verify(value):
                return cls

        raise ValueError('Parameter malformed : {}'.format(value))

class VerifyParameterNumListOfTuples(VerifyParameterType):
    @staticmethod
    def verify(value):
        if type(value) == list:
            if len(value) >= 1:
                if type(value[0]) == tuple:
                    return True
        return False

class VerifyParameterNumList(VerifyParameterType):
    @staticmethod
    def verify(value):
        if type(value) == list:
            if len(value) > 1:
                if type(value[0]) != tuple:
                    return True
        return False

class VerifyParameterNumListSingleEntry(VerifyParameterType):
    @staticmethod
    def verify(value):
        if type(value) == list:
            if len(value) == 1:
                if type(value[0]) != tuple:
                    return True
        return False

class VerifyParameterNumSingleValue(VerifyParameterType):
    @staticmethod
    def verify(value):
        if type(value) == int or type(value) == float:
            return True
        return False

if __name__ == '__main__':
    print( VerifyParameterType.verify([0]) )
    print( VerifyParameterType.verify([(0, 10)]) )
    print( VerifyParameterType.verify([0, 10, 20]) )
    print( VerifyParameterType.verify(1) )
    # print( VerifyParameterType.verify('a') )