text:
# emacs: -*- coding: utf-8; mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
#
#   See COPYING file distributed along with the PyMVPA package for the
#   copyright and license terms.
#
### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
"""Wrap the libsvm package into a very simple class interface."""

__docformat__ = "restructuredtext"

import numpy as np

import operator

from mvpa2.base import warning
from mvpa2.base.state import ConditionalAttribute
from mvpa2.base.learner import FailedToTrainError

from mvpa2.clfs.base import accepts_dataset_as_samples, accepts_samples_as_dataset
from mvpa2.clfs._svmbase import _SVM

from mvpa2.clfs.libsvmc import _svm
from mvpa2.kernels.libsvm import LinearLSKernel
from mvpa2.clfs.libsvmc.sens import LinearSVMWeights

from mvpa2.support.due import due, Doi, BibTeX

if __debug__:
    from mvpa2.base import debug

# we better expose those since they are mentioned in docstrings
# although pylint would not be happy

def _data2ls(data):
    return np.asarray(data).astype(float)

class SVM(_SVM):
    """Support Vector Machine Classifier.

    This is a simple interface to the libSVM package.
    """

    # Since this is internal feature of LibSVM, this conditional attribute is present
    # here
    probabilities = ConditionalAttribute(
        enabled=False, doc="Estimates of samples probabilities as provided by LibSVM"
    )

    # TODO p is specific for SVR
    _KNOWN_PARAMS = ["epsilon", "probability", "shrinking", "weight_label", "weight"]

    # _KNOWN_KERNEL_PARAMS = [ 'cache_size' ]

    _KNOWN_SENSITIVITIES = {
        "linear": LinearSVMWeights,
    }
    _KNOWN_IMPLEMENTATIONS = {
        "C_SVC": (
            _svm.svmc.C_SVC,
            ("C",),
            ("binary", "multiclass", "oneclass"),
            "C-SVM classification",
        ),
        "NU_SVC": (
            _svm.svmc.NU_SVC,
            ("nu",),
            ("binary", "multiclass", "oneclass"),
            "nu-SVM classification",
        ),
        "ONE_CLASS": (_svm.svmc.ONE_CLASS, (), ("oneclass-binary",), "one-class-SVM"),
        "EPSILON_SVR": (
            _svm.svmc.EPSILON_SVR,
            ("C", "tube_epsilon"),
            ("regression",),
            "epsilon-SVM regression",
        ),
        "NU_SVR": (
            _svm.svmc.NU_SVR,
            ("nu", "tube_epsilon"),
            ("regression", "oneclass"),
            "nu-SVM regression",
        ),
    }

    __default_kernel_class__ = LinearLSKernel
    __tags__ = _SVM.__tags__ + ["libsvm"]

    def __init__(self, **kwargs):
        # XXX Determine which parameters depend on each other and implement
        # safety/simplifying logic around them
        # already done for: nr_weight
        # thought: weight and weight_label should be a dict
        """Interface class to LIBSVM classifiers and regressions.

        Default implementation (C/nu/epsilon SVM) is chosen depending
        on the given parameters (C/nu/tube_epsilon).
        """

        svm_impl = kwargs.get("svm_impl", None)
        # Depending on given arguments, figure out desired SVM
        # implementation
        if svm_impl is None:
            for arg, impl in [
                ("tube_epsilon", "EPSILON_SVR"),
                ("C", "C_SVC"),
                ("nu", "NU_SVC"),
            ]:
                if arg in kwargs:
                    svm_impl = impl
                    if __debug__:
                        debug(
                            "SVM",
                            "No implementation was specified. Since "
                            "%s is given among arguments, assume %s" % (arg, impl),
                        )
                    break
            if svm_impl is None:
                svm_impl = "C_SVC"
                if __debug__:
                    debug("SVM", 'Assign C_SVC "by default"')
        kwargs["svm_impl"] = svm_impl

        # init base class
        _SVM.__init__(self, **kwargs)

        self._svm_type = self._KNOWN_IMPLEMENTATIONS[svm_impl][0]

        if "nu" in self._KNOWN_PARAMS and "epsilon" in self._KNOWN_PARAMS:
            # overwrite eps param with new default value (information
            # taken from libSVM docs
            self.params["epsilon"]._set_default(0.001)

        self.__model = None
        """Holds the trained SVM."""

    @due.dcite(
        Doi("10.1145/1961189.1961199"),
        description="LIBSVM: A library for support vector machines",
        path="libsvm",
        tags=["implementation"],
    )
    # TODO: conditioned citations for nu-SVM and one-class
    #    B. Schölkopf, A. Smola, R. Williamson, and P. L. Bartlett. New support vector algorithms. Neural Computation, 12, 2000, 1207-1245.
    #    B. Schölkopf, J. Platt, J. Shawe-Taylor, A. J. Smola, and R. C. Williamson. Estimating the support of a high-dimensional distribution. Neural Computation, 13, 2001, 1443-1471.
    def _train(self, dataset):
        """Train SVM"""
        super(SVM, self)._train(dataset)
        targets_sa_name = self.get_space()  # name of targets sa
        targets_sa = dataset.sa[targets_sa_name]  # actual targets sa

        # libsvm needs doubles
        src = _data2ls(dataset)

        # libsvm cannot handle literal labels
        labels = self._attrmap.to_numeric(targets_sa.value).tolist()

        svmprob = _svm.SVMProblem(labels, src)

        # Translate few params
        TRANSLATEDICT = {"epsilon": "eps", "tube_epsilon": "p"}
        args = []
        for paramname, param in list(self.params.items()) + list(
            self.kernel_params.items()
        ):
            if paramname in TRANSLATEDICT:
                argname = TRANSLATEDICT[paramname]
            elif paramname in _svm.SVMParameter.default_parameters:
                argname = paramname
            else:
                if __debug__:
                    debug(
                        "SVM_",
                        "Skipping parameter %s since it is not known "
                        "to libsvm" % paramname,
                    )
                continue
            args.append((argname, param.value))

        # ??? All those parameters should be fetched if present from
        # **kwargs and create appropriate parameters within .params or
        # .kernel_params
        libsvm_param = _svm.SVMParameter(
            kernel_type=self.params.kernel.as_raw_ls(),  # Just an integer ID
            svm_type=self._svm_type,
            **dict(args)
        )

        """Store SVM parameters in libSVM compatible format."""

        if "C" in self.params:  # svm_type in [_svm.svmc.C_SVC]:
            Cs = self._get_cvec(dataset)
            if len(Cs) > 1:
                C0 = abs(Cs[0])
                scale = 1.0 / C0  # *np.sqrt(C0))
                # so we got 1 C per label
                uls = self._attrmap.to_numeric(targets_sa.unique)
                if len(Cs) != len(uls):
                    raise ValueError(
                        "SVM was parameterized with %d Cs but there are %d "
                        "labels in the dataset" % (len(Cs), len(targets_sa.unique))
                    )
                weight = [c * scale for c in Cs]
                # All 3 need to be set to take an effect
                libsvm_param._set_parameter("weight", weight)
                libsvm_param._set_parameter("nr_weight", len(weight))
                libsvm_param._set_parameter("weight_label", uls)
            libsvm_param._set_parameter("C", Cs[0])

        try:
            self.__model = _svm.SVMModel(svmprob, libsvm_param)
        except Exception as e:
            raise FailedToTrainError(str(e))

    @accepts_samples_as_dataset
    def _predict(self, data):
        """Predict values for the data"""
        # libsvm needs doubles
        src = _data2ls(data)
        ca = self.ca

        predictions = [self.model.predict(p) for p in src]

        if ca.is_enabled("estimates"):
            if self.__is_regression__:
                estimates = [self.model.predict_values_raw(p)[0] for p in src]
            else:
                # if 'trained_targets' are literal they have to be mapped
                if np.issubdtype(self.ca.trained_targets.dtype, "c") or np.issubdtype(
                    self.ca.trained_targets.dtype, "U"
                ):
                    trained_targets = self._attrmap.to_numeric(self.ca.trained_targets)
                else:
                    trained_targets = self.ca.trained_targets
                nlabels = len(trained_targets)
                # XXX We do duplicate work. model.predict calls
                # predict_values_raw internally and then does voting or
                # thresholding. So if speed becomes a factor we might
                # want to move out logic from libsvm over here to base
                # predictions on obtined values, or adjust libsvm to
                # spit out values from predict() as well
                if nlabels == 2 and self._svm_impl != "ONE_CLASS":
                    # Apperently libsvm reorders labels so we need to
                    # track (1,0) values instead of (0,1) thus just
                    # lets take negative reverse
                    estimates = [
                        self.model.predict_values(p)[
                            (trained_targets[1], trained_targets[0])
                        ]
                        for p in src
                    ]
                    if len(estimates) > 0:
                        if __debug__:
                            debug(
                                "SVM",
                                "Forcing estimates to be ndarray and reshaping"
                                " them into 1D vector",
                            )
                        estimates = np.asarray(estimates).reshape(len(estimates))
                else:
                    # In multiclass we return dictionary for all pairs
                    # of labels, since libsvm does 1-vs-1 pairs
                    estimates = [self.model.predict_values(p) for p in src]
            ca.estimates = estimates

        if ca.is_enabled("probabilities"):
            # XXX Is this really necesssary? yoh don't think so since
            # assignment to ca is doing the same
            # self.probabilities = [ self.model.predict_probability(p)
            #                       for p in src ]
            try:
                ca.probabilities = [self.model.predict_probability(p) for p in src]
            except TypeError:
                warning(
                    "Current SVM %s doesn't support probability " % self
                    + " estimation."
                )
        return predictions

    def summary(self):
        """Provide quick summary over the SVM classifier"""
        s = super(SVM, self).summary()
        if self.trained:
            s += "\n #SVs:%d" % self.__model.get_total_n_sv()
            try:
                param = self.__model.model.param
                C = param.C
                # extract information of how many SVs sit inside the margin,
                # i.e. so called 'bounded SVs'
                inside_margin = np.sum(
                    # take 0.99 to avoid rounding issues
                    np.abs(self.__model.get_sv_coef())
                    >= 0.99 * param.C
                )
                s += " #bounded_SVs:%d" % inside_margin
                s += (" used_C:%-5g" % C).rstrip()
            except:
                pass
        return s

    def _untrain(self):
        """Untrain libsvm's SVM: forget the model"""
        if __debug__ and "SVM" in debug.active:
            debug("SVM", "Untraining %s and destroying libsvm model" % self)
        super(SVM, self)._untrain()
        del self.__model
        self.__model = None

    model = property(fget=lambda self: self.__model)
    """Access to the SVM model."""

# try to configure libsvm 'noise reduction'. Due to circular imports,
# we can't check externals here since it would not work.
try:
    # if externals.exists('libsvm verbosity control'):
    if __debug__ and "LIBSVM" in debug.active:
        debug("LIBSVM", "Setting verbosity for libsvm to 255")
        _svm.svmc.svm_set_verbosity(255)
    else:
        _svm.svmc.svm_set_verbosity(0)
except AttributeError:
    warning("Available LIBSVM has no way to control verbosity of the output")

# Assign SVM class to limited set of LinearSVMWeights
LinearSVMWeights._LEGAL_CLFS = [SVM]

max_stars_repo_name: mortonne/PyMVPA

max_issues_repo_name: mortonne/PyMVPA

max_forks_repo_name: mortonne/PyMVPA


text:
# --------------------------------------------------------
# Fast R-CNN
# Copyright (c) 2015 Microsoft
# Licensed under The MIT License [see LICENSE for details]
# Written by Ross Girshick
# --------------------------------------------------------

"""Test a Fast R-CNN network on an imdb (image database)."""

from fast_rcnn.config import cfg, get_output_dir
from fast_rcnn.bbox_transform import clip_boxes, bbox_transform_inv
import argparse
from utils.timer import Timer
import numpy as np
import cv2
import os
os.environ['GLOG_minloglevel'] = '2'
import caffe
from fast_rcnn.nms_wrapper import nms
import cPickle
from utils.blob import im_list_to_blob
import os

def _get_image_blob(im):
    """Converts an image into a network input.

    Arguments:
        im (ndarray): a color image in BGR order

    Returns:
        blob (ndarray): a data blob holding an image pyramid
        im_scale_factors (list): list of image scales (relative to im) used
            in the image pyramid
    """
    im_orig = im.astype(np.float32, copy=True)
    im_orig -= cfg.PIXEL_MEANS

    im_shape = im_orig.shape
    im_size_min = np.min(im_shape[0:2])
    im_size_max = np.max(im_shape[0:2])

    processed_ims = []
    im_scale_factors = []

    for target_size in cfg.TEST.SCALES:
        im_scale = float(target_size) / float(im_size_min)
        # Prevent the biggest axis from being more than MAX_SIZE
        if np.round(im_scale * im_size_max) > cfg.TEST.MAX_SIZE:
            im_scale = float(cfg.TEST.MAX_SIZE) / float(im_size_max)
        im = cv2.resize(im_orig, None, None, fx=im_scale, fy=im_scale,
                        interpolation=cv2.INTER_LINEAR)
        im_scale_factors.append(im_scale)
        processed_ims.append(im)

    # Create a blob to hold the input images
    blob = im_list_to_blob(processed_ims)

    return blob, np.array(im_scale_factors)

def _get_rois_blob(im_rois, im_scale_factors):
    """Converts RoIs into network inputs.

    Arguments:
        im_rois (ndarray): R x 4 matrix of RoIs in original image coordinates
        im_scale_factors (list): scale factors as returned by _get_image_blob

    Returns:
        blob (ndarray): R x 5 matrix of RoIs in the image pyramid
    """
    rois, levels = _project_im_rois(im_rois, im_scale_factors)
    rois_blob = np.hstack((levels, rois))
    return rois_blob.astype(np.float32, copy=False)

def _project_im_rois(im_rois, scales):
    """Project image RoIs into the image pyramid built by _get_image_blob.

    Arguments:
        im_rois (ndarray): R x 4 matrix of RoIs in original image coordinates
        scales (list): scale factors as returned by _get_image_blob

    Returns:
        rois (ndarray): R x 4 matrix of projected RoI coordinates
        levels (list): image pyramid levels used by each projected RoI
    """
    im_rois = im_rois.astype(np.float, copy=False)

    if len(scales) > 1:
        widths = im_rois[:, 2] - im_rois[:, 0] + 1
        heights = im_rois[:, 3] - im_rois[:, 1] + 1

        areas = widths * heights
        scaled_areas = areas[:, np.newaxis] * (scales[np.newaxis, :] ** 2)
        diff_areas = np.abs(scaled_areas - 224 * 224)
        levels = diff_areas.argmin(axis=1)[:, np.newaxis]
    else:
        levels = np.zeros((im_rois.shape[0], 1), dtype=np.int)

    rois = im_rois * scales[levels]

    return rois, levels

def _get_blobs(im, rois, mask=False):
    """Convert an image and RoIs within that image into network inputs."""
    blobs = {'data' : None, 'rois' : None}
    blobs['data'], im_scale_factors = _get_image_blob(im)
    if not cfg.TEST.HAS_RPN or mask:
        blobs['rois'] = _get_rois_blob(rois, im_scale_factors)
    return blobs, im_scale_factors

def im_detect(net, im, boxes=None):
    """Detect object classes in an image given object proposals.

    Arguments:
        net (caffe.Net): Fast R-CNN network to use
        im (ndarray): color image to test (in BGR order)
        boxes (ndarray): R x 4 array of object proposals or None (for RPN)

    Returns:
        scores (ndarray): R x K array of object class scores (K includes
            background as object category 0)
        boxes (ndarray): R x (4*K) array of predicted bounding boxes
    """
    blobs, im_scales = _get_blobs(im, boxes)

    # When mapping from image ROIs to feature map ROIs, there's some aliasing
    # (some distinct image ROIs get mapped to the same feature ROI).
    # Here, we identify duplicate feature ROIs, so we only compute features
    # on the unique subset.
    if cfg.DEDUP_BOXES > 0 and not cfg.TEST.HAS_RPN:
        v = np.array([1, 1e3, 1e6, 1e9, 1e12])
        hashes = np.round(blobs['rois'] * cfg.DEDUP_BOXES).dot(v)
        _, index, inv_index = np.unique(hashes, return_index=True,
                                        return_inverse=True)
        blobs['rois'] = blobs['rois'][index, :]
        boxes = boxes[index, :]

    if cfg.TEST.HAS_RPN:
        im_blob = blobs['data']
        blobs['im_info'] = np.array(
            [[im_blob.shape[2], im_blob.shape[3], im_scales[0]]],
            dtype=np.float32)

    # reshape network inputs
    net.blobs['data'].reshape(*(blobs['data'].shape))
    if cfg.TEST.HAS_RPN:
        net.blobs['im_info'].reshape(*(blobs['im_info'].shape))
    else:
        net.blobs['rois'].reshape(*(blobs['rois'].shape))

    # do forward
    forward_kwargs = {'data': blobs['data'].astype(np.float32, copy=False)}
    if cfg.TEST.HAS_RPN:
        forward_kwargs['im_info'] = blobs['im_info'].astype(np.float32, copy=False)
    else:
        forward_kwargs['rois'] = blobs['rois'].astype(np.float32, copy=False)
    blobs_out = net.forward(**forward_kwargs)

    if cfg.TEST.HAS_RPN:
        assert len(im_scales) == 1, "Only single-image batch implemented"
        rois = net.blobs['rois'].data.copy()
        # unscale back to raw image space
        boxes = rois[:, 1:5] / im_scales[0]

    if cfg.TEST.SVM:
        # use the raw scores before softmax under the assumption they
        # were trained as linear SVMs
        scores = net.blobs['cls_score'].data
    else:
        # use softmax estimated probabilities
        scores = blobs_out['cls_prob']

    if cfg.TEST.BBOX_REG:
        # Apply bounding-box regression deltas
        box_deltas = blobs_out['bbox_pred']
        pred_boxes = bbox_transform_inv(boxes, box_deltas)
        pred_boxes = clip_boxes(pred_boxes, im.shape)
    else:
        # Simply repeat the boxes, once for each class
        pred_boxes = np.tile(boxes, (1, scores.shape[1]))

    if cfg.DEDUP_BOXES > 0 and not cfg.TEST.HAS_RPN:
        # Map scores and predictions back to the original set of boxes
        scores = scores[inv_index, :]
        pred_boxes = pred_boxes[inv_index, :]

    return scores, pred_boxes

def im_det(net, im, boxes=None):
    """Detect object classes in an image given object proposals.

    Arguments:
        net (caffe.Net): Fast R-CNN network to use
        im (ndarray): color image to test (in BGR order)
        boxes (ndarray): R x 4 array of object proposals or None (for RPN)

    Returns:
        scores (ndarray): R x K array of object class scores (K includes
            background as object category 0)
        boxes (ndarray): R x (4*K) array of predicted bounding boxes
    """
    blobs, im_scales = _get_blobs(im, boxes)

    # When mapping from image ROIs to feature map ROIs, there's some aliasing
    # (some distinct image ROIs get mapped to the same feature ROI).
    # Here, we identify duplicate feature ROIs, so we only compute features
    # on the unique subset.
    if cfg.DEDUP_BOXES > 0 and not cfg.TEST.HAS_RPN:
        v = np.array([1, 1e3, 1e6, 1e9, 1e12])
        hashes = np.round(blobs['rois'] * cfg.DEDUP_BOXES).dot(v)
        _, index, inv_index = np.unique(hashes, return_index=True,
                                        return_inverse=True)
        blobs['rois'] = blobs['rois'][index, :]
        boxes = boxes[index, :]

    if cfg.TEST.HAS_RPN:
        im_blob = blobs['data']
        blobs['im_info'] = np.array(
            [[im_blob.shape[2], im_blob.shape[3], im_scales[0]]],
            dtype=np.float32)

    # reshape network inputs
    net.blobs['data'].reshape(*(blobs['data'].shape))
    if cfg.TEST.HAS_RPN:
        net.blobs['im_info'].reshape(*(blobs['im_info'].shape))
    else:
        net.blobs['rois'].reshape(*(blobs['rois'].shape))

    # do forward
    print(im_blob.shape)
    forward_kwargs = {'data': blobs['data'].astype(np.float32, copy=False)}
    if cfg.TEST.HAS_RPN:
        forward_kwargs['im_info'] = blobs['im_info'].astype(np.float32, copy=False)
    else:
        forward_kwargs['rois'] = blobs['rois'].astype(np.float32, copy=False)
    blobs_out = net.forward(**forward_kwargs)

    if cfg.TEST.HAS_RPN:
        assert len(im_scales) == 1, "Only single-image batch implemented"
        rois = net.blobs['rois'].data.copy()
        # unscale back to raw image space
        boxes = rois[:, 1:5] / im_scales[0]

    scores = blobs_out['cls_prob']

    if cfg.TEST.BBOX_REG:
        # Apply bounding-box regression deltas
        box_deltas = blobs_out['bbox_pred']
        pred_boxes = bbox_transform_inv(boxes, box_deltas)
        pred_boxes = clip_boxes(pred_boxes, im.shape)
    else:
        # Simply repeat the boxes, once for each class
        pred_boxes = np.tile(boxes, (1, scores.shape[1]))

    if cfg.DEDUP_BOXES > 0 and not cfg.TEST.HAS_RPN:
        # Map scores and predictions back to the original set of boxes
        scores = scores[inv_index, :]
        pred_boxes = pred_boxes[inv_index, :]

    feat = net.blobs['res4f'].data.copy()
    return scores, pred_boxes, feat

def im_seg(net, im, feat, boxes=None):
    """Detect object classes in an image given object proposals.

    Arguments:
        net (caffe.Net): Fast R-CNN network to use
        im (ndarray): color image to test (in BGR order)
        boxes (ndarray): R x 4 array of object proposals or None (for RPN)

    Returns:
        scores (ndarray): R x K array of object class scores (K includes
            background as object category 0)
        boxes (ndarray): R x (4*K) array of predicted bounding boxes
    """
    blobs, im_scales = _get_blobs(im, boxes, mask=True)
    # blobs['rois']=blobs['rois'][:,1:]

    # reshape network inputs
    net.blobs['res4f'].reshape(*(feat.shape))
    net.blobs['rois'].reshape(*(blobs['rois'].shape))

    # do forward
    # print(feat)
    net.blobs['res4f'].data[...] = feat
    net.blobs['rois'].data[...] = blobs['rois']
    forward_kwargs = {'rois': blobs['rois'].astype(np.float32, copy=False), 'res4f': feat}
    blobs_out = net.forward()

    seg_result = blobs_out['mask_prob']
    # print(seg_result)
    # print(np.unique(seg_result))
    # print(np.unique(feat))
    seg_result = np.array(seg_result > 0.5, dtype=np.int32)

    return seg_result

def im_detect_rpn(net, im):
    """Detect object classes in an image given object proposals.

    Arguments:
        net (caffe.Net): Fast R-CNN network to use
        im (ndarray): color image to test (in BGR order)
        boxes (ndarray): R x 4 array of object proposals or None (for RPN)

    Returns:
        scores (ndarray): R x K array of object class scores (K includes
            background as object category 0)
        boxes (ndarray): R x (4*K) array of predicted bounding boxes
    """
    blobs, im_scales = _get_blobs(im, None)

    # When mapping from image ROIs to feature map ROIs, there's some aliasing
    # (some distinct image ROIs get mapped to the same feature ROI).
    # Here, we identify duplicate feature ROIs, so we only compute features
    # on the unique subset.
    im_blob = blobs['data']
    blobs['im_info'] = np.array(
        [[im_blob.shape[2], im_blob.shape[3], im_scales[0]]],
        dtype=np.float32)

    # reshape network inputs
    net.blobs['data'].reshape(*(blobs['data'].shape))
    net.blobs['im_info'].reshape(*(blobs['im_info'].shape))

    # do forward
    forward_kwargs = {'data': blobs['data'].astype(np.float32, copy=False)}
    forward_kwargs['im_info'] = blobs['im_info'].astype(np.float32, copy=False)
    blobs_out = net.forward(**forward_kwargs)

    assert len(im_scales) == 1, "Only single-image batch implemented"
    rois = net.blobs['rois'].data.copy()
    # unscale back to raw image space
    boxes = rois[:, 1:5] / im_scales[0]
    scores = net.blobs['scores'].data

    return scores, boxes

def im_detect_mask(net, im, boxes=None):
    """Detect object classes in an image given object proposals.

    Arguments:
        net (caffe.Net): Fast R-CNN network to use
        im (ndarray): color image to test (in BGR order)
        boxes (ndarray): R x 4 array of object proposals or None (for RPN)

    Returns:
        masks (ndarray): R x M x M array of predicted masks
    """
    boxes = boxes.reshape((-1,4))
    blobs, im_scales = _get_blobs(im, boxes)

    # When mapping from image ROIs to feature map ROIs, there's some aliasing
    # (some distinct image ROIs get mapped to the same feature ROI).
    # Here, we identify duplicate feature ROIs, so we only compute features
    # on the unique subset.
    # reshape network inputs
    net.blobs['data'].reshape(*(blobs['data'].shape))
    net.blobs['rois'].reshape(*(blobs['rois'].shape))

    # do forward
    forward_kwargs = {'data': blobs['data'].astype(np.float32, copy=False)}
    forward_kwargs['rois'] = blobs['rois'].astype(np.float32, copy=False)
    blobs_out = net.forward(**forward_kwargs)

    masks = blobs_out['mask_prob']

    return masks

def vis_detections(im, class_name, dets, thresh=0.3):
    """Visual debugging of detections."""
    import matplotlib.pyplot as plt
    im = im[:, :, (2, 1, 0)]
    for i in xrange(np.minimum(10, dets.shape[0])):
        bbox = dets[i, :4]
        score = dets[i, -1]
        if score > thresh:
            plt.cla()
            plt.imshow(im)
            plt.gca().add_patch(
                plt.Rectangle((bbox[0], bbox[1]),
                              bbox[2] - bbox[0],
                              bbox[3] - bbox[1], fill=False,
                              edgecolor='g', linewidth=3)
                )
            plt.title('{}  {:.3f}'.format(class_name, score))
            plt.show()

def apply_nms(all_boxes, thresh):
    """Apply non-maximum suppression to all predicted boxes output by the
    test_net method.
    """
    num_classes = len(all_boxes)
    num_images = len(all_boxes[0])
    nms_boxes = [[[] for _ in xrange(num_images)]
                 for _ in xrange(num_classes)]
    for cls_ind in xrange(num_classes):
        for im_ind in xrange(num_images):
            dets = all_boxes[cls_ind][im_ind]
            if dets == []:
                continue
            # CPU NMS is much faster than GPU NMS when the number of boxes
            # is relative small (e.g., < 10k)
            # TODO(rbg): autotune NMS dispatch
            keep = nms(dets, thresh, force_cpu=True)
            if len(keep) == 0:
                continue
            nms_boxes[cls_ind][im_ind] = dets[keep, :].copy()
    return nms_boxes

def test_net(net, imdb, max_per_image=400, thresh=-np.inf, vis=False, mask=False):
    """Test a Fast R-CNN network on an image database."""
    num_images = len(imdb.image_index)
    # all detections are collected into:
    #    all_boxes[cls][image] = N x 5 array of detections in
    #    (x1, y1, x2, y2, score)
    all_boxes = [[[] for _ in xrange(num_images)]
                 for _ in xrange(imdb.num_classes)]

    output_dir = get_output_dir(imdb, net)

    # timers
    _t = {'im_detect' : Timer(), 'misc' : Timer()}

    if not cfg.TEST.HAS_RPN:
        roidb = imdb.roidb

    for i in xrange(num_images):
        # filter out any ground truth boxes
        if cfg.TEST.HAS_RPN:
            box_proposals = None
        else:
            # The roidb may contain ground-truth rois (for example, if the roidb
            # comes from the training or val split). We only want to evaluate
            # detection on the *non*-ground-truth rois. We select those the rois
            # that have the gt_classes field set to 0, which means there's no
            # ground truth.
            box_proposals = roidb[i]['boxes'][roidb[i]['gt_classes'] == 0]

        im = cv2.imread(imdb.image_path_at(i))
        _t['im_detect'].tic()
        scores, boxes = im_detect(net, im, box_proposals)
        _t['im_detect'].toc()

        _t['misc'].tic()
        # skip j = 0, because it's the background class
        for j in xrange(1, imdb.num_classes):
            inds = np.where(scores[:, j] > thresh)[0]
            cls_scores = scores[inds, j]
            if cfg.TEST.AGNOSTIC:
                cls_boxes = boxes[inds, 4:8]
            else:
                cls_boxes = boxes[inds, j*4:(j+1)*4]
            cls_dets = np.hstack((cls_boxes, cls_scores[:, np.newaxis])) \
                .astype(np.float32, copy=False)
            keep = nms(cls_dets, cfg.TEST.NMS)
            cls_dets = cls_dets[keep, :]
            if vis:
                vis_detections(im, imdb.classes[j], cls_dets)
            all_boxes[j][i] = cls_dets

        # Limit to max_per_image detections *over all classes*
        if max_per_image > 0:
            image_scores = np.hstack([all_boxes[j][i][:, -1]
                                      for j in xrange(1, imdb.num_classes)])
            if len(image_scores) > max_per_image:
                image_thresh = np.sort(image_scores)[-max_per_image]
                for j in xrange(1, imdb.num_classes):
                    keep = np.where(all_boxes[j][i][:, -1] >= image_thresh)[0]
                    all_boxes[j][i] = all_boxes[j][i][keep, :]
        # _t['misc'].toc()
        #
        # _t['im_detect'].tic()
        # seg = im_seg(net, im, all_boxes[:][i])
        # _t['im_detect'].toc()
        #
        # _t['misc'].tic()

        print 'im_detect: {:d}/{:d} {:.3f}s {:.3f}s' \
              .format(i + 1, num_images, _t['im_detect'].average_time,
                      _t['misc'].average_time)

    det_file = os.path.join(output_dir, 'detections.pkl')
    with open(det_file, 'wb') as f:
        cPickle.dump(all_boxes, f, cPickle.HIGHEST_PROTOCOL)

    print 'Evaluating detections'
    imdb.evaluate_detections(all_boxes, output_dir)

def test_rpn(net, imdb, max_per_image=400, thresh=-np.inf, vis=False):
    """Test a Fast R-CNN network on an image database."""
    num_images = len(imdb.image_index)
    # all detections are collected into:
    #    all_boxes[cls][image] = N x 5 array of detections in
    #    (x1, y1, x2, y2, score)
    all_boxes = [[[] for _ in xrange(num_images)]
                 for _ in xrange(2)]

    output_dir = get_output_dir(imdb, net)

    # timers
    _t = {'im_detect' : Timer(), 'misc' : Timer()}

    if not cfg.TEST.HAS_RPN:
        roidb = imdb.roidb

    for i in xrange(num_images):
        im = cv2.imread(imdb.image_path_at(i))
        _t['im_detect'].tic()
        scores, boxes = im_detect_rpn(net, im)
        _t['im_detect'].toc()

        _t['misc'].tic()
        # skip j = 0, because it's the background class
        inds = np.where(scores[:] > thresh)[0]
        cls_scores = scores[inds]
        cls_boxes = boxes[inds, :]
        cls_dets = np.hstack((cls_boxes, cls_scores)) \
            .astype(np.float32, copy=False)
        keep = nms(cls_dets, cfg.TEST.NMS)
        cls_dets = cls_dets[keep, :]
        if vis:
            vis_detections(im, imdb.classes[1], cls_dets)
        all_boxes[1][i] = cls_dets

        # Limit to max_per_image detections *over all classes*
        if max_per_image > 0:
            image_scores = np.hstack([all_boxes[1][i][:, -1]
                                      for j in xrange(1, 2)])
            if len(image_scores) > max_per_image:
                image_thresh = np.sort(image_scores)[-max_per_image]
                for j in xrange(1, 2):
                    keep = np.where(all_boxes[j][i][:, -1] >= image_thresh)[0]
                    all_boxes[j][i] = all_boxes[j][i][keep, :]
        _t['misc'].toc()

        print 'im_detect: {:d}/{:d} {:.3f}s {:.3f}s' \
              .format(i + 1, num_images, _t['im_detect'].average_time,
                      _t['misc'].average_time)

    det_file = os.path.join(output_dir, 'detections.pkl')
    with open(det_file, 'wb') as f:
        cPickle.dump(all_boxes, f, cPickle.HIGHEST_PROTOCOL)

    print 'Evaluating detections'
    imdb.evaluate_detections(all_boxes, output_dir)

def test_net_mask(net, net_mask, imdb, max_per_image=400, thresh=-np.inf, vis=False, save_path="./output/"):
    """Test a Fast R-CNN network on an image database."""
    num_images = len(imdb.image_index)
    # all detections are collected into:
    #    all_boxes[cls][image] = N x 5 array of detections in
    #    (x1, y1, x2, y2, score)
    all_boxes = [[[] for _ in xrange(num_images)]
                 for _ in xrange(imdb.num_classes)]

    output_dir = get_output_dir(imdb, net)

    if not os.path.exists(save_path):
        os.makedirs(save_path)

    # timers
    _t = {'im_detect' : Timer(), 'im_seg' : Timer(),'misc' : Timer()}

    if not cfg.TEST.HAS_RPN:
        roidb = imdb.roidb

    for i in xrange(num_images):
        # filter out any ground truth boxes
        if cfg.TEST.HAS_RPN:
            box_proposals = None
        else:
            # The roidb may contain ground-truth rois (for example, if the roidb
            # comes from the training or val split). We only want to evaluate
            # detection on the *non*-ground-truth rois. We select those the rois
            # that have the gt_classes field set to 0, which means there's no
            # ground truth.
            box_proposals = roidb[i]['boxes'][roidb[i]['gt_classes'] == 0]

        im = cv2.imread(imdb.image_path_at(i))
        print(im.shape)
        _t['im_detect'].tic()
        scores, boxes, feat = im_det(net, im, box_proposals)
        _t['im_detect'].toc()

        _t['misc'].tic()
        # skip j = 0, because it's the background class
        for j in xrange(1, imdb.num_classes):
            inds = np.where(scores[:, j] > thresh)[0]
            cls_scores = scores[inds, j]
            if cfg.TEST.AGNOSTIC:
                cls_boxes = boxes[inds, 4:8]
            else:
                cls_boxes = boxes[inds, j*4:(j+1)*4]
            cls_dets = np.hstack((cls_boxes, cls_scores[:, np.newaxis])) \
                .astype(np.float32, copy=False)
            keep = nms(cls_dets, cfg.TEST.NMS)
            cls_dets = cls_dets[keep, :]
            if vis:
                vis_detections(im, imdb.classes[j], cls_dets)
            all_boxes[j][i] = cls_dets

        # Limit to max_per_image detections *over all classes*
        if max_per_image > 0:
            image_scores = np.hstack([all_boxes[j][i][:, -1]
                                      for j in xrange(1, imdb.num_classes)])
            if len(image_scores) > max_per_image:
                image_thresh = np.sort(image_scores)[-max_per_image]
                for j in xrange(1, imdb.num_classes):
                    keep = np.where(all_boxes[j][i][:, -1] >= image_thresh)[0]
                    all_boxes[j][i] = all_boxes[j][i][keep, :]

        print 'im_detection: {:d}/{:d} {:.3f}s {:.3f}s' \
              .format(i + 1, num_images, _t['im_detect'].average_time,
                      _t['misc'].average_time)

        _t['im_seg'].tic()
        out_mask = np.zeros((im.shape[0], im.shape[1]))
        for j in xrange(1, imdb.num_classes):
            ins_index = 1
            boxes_this_im = all_boxes[j][i][:, :-1]
            seg = im_seg(net_mask, im, feat, boxes_this_im)
            # print(seg.shape)
            for ii in xrange(seg.shape[0]):
                seg_now = seg[ii][0]
                # seg_now = np.transpose(seg_now)
                box_now = boxes_this_im[ii]
                box_now = box_now.astype(int)
                # im_now=im[box_now[1]: box_now[3], box_now[0]:box_now[2]]
                # cv2.imshow("test", im_now)
                # cv2.waitKey()
                seg_org_size = cv2.resize(seg_now, (box_now[2] - box_now[0], box_now[3] - box_now[1]), interpolation=cv2.INTER_NEAREST)
                # print((seg_org_size*99999).shape)
                # cv2.imshow("seg", seg_org_size*99999)
                # cv2.waitKey()
                seg_org_size = seg_org_size*ins_index
                out_mask[box_now[1]: box_now[3], box_now[0]:box_now[2]] = seg_org_size
                ins_index += 1
        _t['im_seg'].toc()

        mask_save_path = os.path.join(save_path, os.path.basename(imdb.image_path_at(i)).replace(".jpg", ".png"))
        cv2.imwrite(mask_save_path, out_mask*100)

        print 'im_seg: {:d}/{:d} {:.3f}s' \
              .format(i + 1, num_images, _t['im_seg'].average_time)

    det_file = os.path.join(output_dir, 'detections.pkl')
    with open(det_file, 'wb') as f:
        cPickle.dump(all_boxes, f, cPickle.HIGHEST_PROTOCOL)

    print 'Evaluating detections'
    imdb.evaluate_detections(all_boxes, output_dir)

def test_net_mask_reload(net_proto, net_mask_proto, weights, imdb, max_per_image=400, thresh=-np.inf, vis=False, save_path="./output/"):
    """Test a Fast R-CNN network on an image database."""
    num_images = len(imdb.image_index)
    # all detections are collected into:
    #    all_boxes[cls][image] = N x 5 array of detections in
    #    (x1, y1, x2, y2, score)

    caffe.set_mode_gpu()
    caffe.set_device(cfg.GPU_ID)

    all_boxes = [[[] for _ in xrange(num_images)]
                 for _ in xrange(imdb.num_classes)]

    net = caffe.Net(net_proto, weights, caffe.TEST)
    net.name = os.path.splitext(os.path.basename(weights))[0]
    output_dir = get_output_dir(imdb, net)

    if not os.path.exists(save_path):
        os.makedirs(save_path)

    # timers
    _t = {'im_detect' : Timer(), 'im_seg' : Timer(),'misc' : Timer()}

    if not cfg.TEST.HAS_RPN:
        roidb = imdb.roidb

    del net
    for i in xrange(num_images):
        # filter out any ground truth boxes
        if cfg.TEST.HAS_RPN:
            box_proposals = None
        else:
            # The roidb may contain ground-truth rois (for example, if the roidb
            # comes from the training or val split). We only want to evaluate
            # detection on the *non*-ground-truth rois. We select those the rois
            # that have the gt_classes field set to 0, which means there's no
            # ground truth.
            box_proposals = roidb[i]['boxes'][roidb[i]['gt_classes'] == 0]

        net = caffe.Net(net_proto, weights, caffe.TEST)
        net.name = os.path.splitext(os.path.basename(weights))[0]
        im = cv2.imread(imdb.image_path_at(i))
        print(im.shape)
        _t['im_detect'].tic()
        scores, boxes, feat = im_det(net, im, box_proposals)
        _t['im_detect'].toc()

        _t['misc'].tic()
        # skip j = 0, because it's the background class
        for j in xrange(1, imdb.num_classes):
            inds = np.where(scores[:, j] > thresh)[0]
            cls_scores = scores[inds, j]
            if cfg.TEST.AGNOSTIC:
                cls_boxes = boxes[inds, 4:8]
            else:
                cls_boxes = boxes[inds, j*4:(j+1)*4]
            cls_dets = np.hstack((cls_boxes, cls_scores[:, np.newaxis])) \
                .astype(np.float32, copy=False)
            keep = nms(cls_dets, cfg.TEST.NMS)
            cls_dets = cls_dets[keep, :]
            if vis:
                vis_detections(im, imdb.classes[j], cls_dets)
            all_boxes[j][i] = cls_dets

        # Limit to max_per_image detections *over all classes*
        if max_per_image > 0:
            image_scores = np.hstack([all_boxes[j][i][:, -1]
                                      for j in xrange(1, imdb.num_classes)])
            if len(image_scores) > max_per_image:
                image_thresh = np.sort(image_scores)[-max_per_image]
                for j in xrange(1, imdb.num_classes):
                    keep = np.where(all_boxes[j][i][:, -1] >= image_thresh)[0]
                    all_boxes[j][i] = all_boxes[j][i][keep, :]

        print 'im_detection: {:d}/{:d} {:.3f}s {:.3f}s' \
              .format(i + 1, num_images, _t['im_detect'].average_time,
                      _t['misc'].average_time)
        del net

        net_mask = caffe.Net(net_mask_proto, weights, caffe.TEST)
        net_mask.name = os.path.splitext(os.path.basename(weights))[0]
        _t['im_seg'].tic()
        out_mask = np.zeros((im.shape[0], im.shape[1]))
        for j in xrange(1, imdb.num_classes):
            ins_index = 1
            boxes_this_im = all_boxes[j][i][:, :-1]
            seg = im_seg(net_mask, im, feat, boxes_this_im)
            # print(seg.shape)
            for ii in xrange(seg.shape[0]):
                seg_now = seg[ii][0]
                # seg_now = np.transpose(seg_now)
                box_now = boxes_this_im[ii]
                box_now = box_now.astype(int)
                if box_now[2] == box_now[0] or box_now[3] == box_now[1]:
                    continue
                # im_now=im[box_now[1]: box_now[3], box_now[0]:box_now[2]]
                # cv2.imshow("test", im_now)
                # cv2.waitKey()
                seg_org_size = cv2.resize(seg_now, (box_now[2] - box_now[0], box_now[3] - box_now[1]), interpolation=cv2.INTER_NEAREST)
                # print((seg_org_size*99999).shape)
                # cv2.imshow("seg", seg_org_size*99999)
                # cv2.waitKey()
                seg_org_size = seg_org_size*ins_index
                out_mask[box_now[1]: box_now[3], box_now[0]:box_now[2]] = seg_org_size
                ins_index += 1
        _t['im_seg'].toc()

        mask_save_path = os.path.join(save_path, os.path.basename(imdb.image_path_at(i)).replace(".jpg", ".png"))
        cv2.imwrite(mask_save_path, out_mask*10)

        print 'im_seg: {:d}/{:d} {:.3f}s' \
              .format(i + 1, num_images, _t['im_seg'].average_time)
        del net_mask

    det_file = os.path.join(output_dir, 'detections.pkl')
    with open(det_file, 'wb') as f:
        cPickle.dump(all_boxes, f, cPickle.HIGHEST_PROTOCOL)

    print 'Evaluating detections'
    imdb.evaluate_detections(all_boxes, output_dir)

ext: py

max_stars_repo_name: VitoChien/py-mask-rcnn-multiGPU

max_issues_repo_name: VitoChien/py-mask-rcnn-multiGPU

max_forks_repo_name: VitoChien/py-mask-rcnn-multiGPU

reason: import numpy


text:
# -*- coding: utf-8 -*-

from fnc_libs import *
from keras import optimizers as op
from util import *
import random
import keras.backend as K
from keras.preprocessing import sequence
from keras.models import Sequential, Model
from keras.layers import Dense, Layer, Dropout, Activation, Input, Merge, Multiply
from keras.layers import Embedding
from keras.layers import Conv1D, GlobalMaxPooling1D
from keras.utils.np_utils import to_categorical
import numpy as np
import pandas as pd
from keras import regularizers

from utils.score import report_score, LABELS, score_submission

# Set file names
file_train_instances = "/fncdata/train_stances.csv"
file_train_bodies = "/fncdata/train_bodies.csv"
file_test_instances = "/fncdata/competition_test_stances.csv"
file_test_bodies = "/fncdata/test_bodies.csv"

file_head = "/fncdata/competition_test_stances.csv"
file_body = "/fncdata/test_bodies.csv"

test_dh,test_db,test_stances = load_data(file_head,file_body)

#file_test_stances = "labeled_ts.csv"
#dataa = pd.read_csv(file_test_stances)
#test_ = dataa.values
#test_stances = test_[:,2]

new_test_stances = []
for i in test_stances:
    if i == 'unrelated':
        new_test_stances.append(3)
    elif i == 'agree':
        new_test_stances.append(0)
    elif i == 'disagree':
        new_test_stances.append(1)
    else:
        new_test_stances.append(2)

# Initialise hyperparameters
r = random.Random()
lim_unigram = 5000
target_size = 4
hidden_size = 100
train_keep_prob = 0.6
l2_alpha = 0.00001
learn_rate = 0.01
clip_ratio = 5
batch_size_train = 500
epochs = 90
reg = 0.00005
# Load data sets
raw_train = FNCData(file_train_instances, file_train_bodies)
raw_test = FNCData(file_test_instances, file_test_bodies)
n_train = len(raw_train.instances)

print ("processing data .....")
# Process data sets
train_set, train_stances, bow_vectorizer, tfreq_vectorizer, tfidf_vectorizer = \
    pipeline_train(raw_train, raw_test, lim_unigram=lim_unigram)
feature_size = len(train_set[0])
test_set = pipeline_test(raw_test, bow_vectorizer, tfreq_vectorizer, tfidf_vectorizer)

model = Sequential()
print("creating model")
model.add(Dense(500, input_dim=10001, activation='relu',\
                kernel_regularizer=regularizers.l2(reg)))
                #activity_regularizer=regularizers.l1(0.001)))
#model.add(Dropout(0.4))
#model.add(Dense(100, activation='relu'))
model.add(Dropout(0.6))
model.add(Dense(100, activation='relu',\
                kernel_regularizer=regularizers.l2(reg)))
                #activity_regularizer=regularizers.l1(0.001)))
model.add(Dropout(0.4))
model.add(Dense(4, activation='softmax'))
print("compiling")

opt = op.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0, clipvalue=5.)

model.compile( loss='categorical_crossentropy',optimizer=opt,metrics=['accuracy'])
Y_train = to_categorical(train_stances, 4)
Y_test = to_categorical(new_test_stances, 4)
train_set = np.array(train_set)
test_set = np.array(test_set)
print("fitting")
model.fit(train_set,Y_train, batch_size=500, epochs=90, verbose=0)
predictions = model.predict(test_set)
predictions = [i.argmax()for i in predictions]
predictions = np.array(predictions)

string_predicted = []

for i,j in enumerate(predictions):
    if j == 3:
        string_predicted.append("unrelated")
    elif j == 0:
        string_predicted.append("agree")
    elif j == 1:
        string_predicted.append("disagree")
    elif j == 2:
        string_predicted.append("discuss")

report_score(test_stances, string_predicted)

'''
# Define model
# Create placeholders
features_pl = tf.placeholder(tf.float32, [None, feature_size], 'features')
stances_pl = tf.placeholder(tf.int64, [None], 'stances')
keep_prob_pl = tf.placeholder(tf.float32)

# Infer batch size
batch_size = tf.shape(features_pl)[0]

# Define multi-layer perceptron
hidden_layer = tf.nn.dropout(tf.nn.relu(tf.contrib.layers.linear(features_pl, hidden_size)), keep_prob=keep_prob_pl)
logits_flat = tf.nn.dropout(tf.contrib.layers.linear(hidden_layer, target_size), keep_prob=keep_prob_pl)
logits = tf.reshape(logits_flat, [batch_size, target_size])

# Define L2 loss
tf_vars = tf.trainable_variables()
l2_loss = tf.add_n([tf.nn.l2_loss(v) for v in tf_vars if 'bias' not in v.name]) * l2_alpha

# Define overall loss
loss = tf.reduce_sum(tf.nn.sparse_softmax_cross_entropy_with_logits(logits, stances_pl) + l2_loss)

# Define prediction
softmaxed_logits = tf.nn.softmax(logits)
predict = tf.arg_max(softmaxed_logits, 1)

# Load model
if mode == 'load':
    with tf.Session() as sess:
        load_model(sess)

        # Predict
        test_feed_dict = {features_pl: test_set, keep_prob_pl: 1.0}
        test_pred = sess.run(predict, feed_dict=test_feed_dict)

# Train model
if mode == 'train':

    # Define optimiser
    opt_func = tf.train.AdamOptimizer(learn_rate)
    grads, _ = tf.clip_by_global_norm(tf.gradients(loss, tf_vars), clip_ratio)
    opt_op = opt_func.apply_gradients(zip(grads, tf_vars))

    # Perform training
    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())

        for epoch in range(epochs):
            total_loss = 0
            indices = list(range(n_train))
            r.shuffle(indices)

            for i in range(n_train // batch_size_train):
                batch_indices = indices[i * batch_size_train: (i + 1) * batch_size_train]
                batch_features = [train_set[i] for i in batch_indices]
                batch_stances = [train_stances[i] for i in batch_indices]

                batch_feed_dict = {features_pl: batch_features, stances_pl: batch_stances, keep_prob_pl: train_keep_prob}
                _, current_loss = sess.run([opt_op, loss], feed_dict=batch_feed_dict)
                total_loss += current_loss

        # Predict
        test_feed_dict = {features_pl: test_set, keep_prob_pl: 1.0}
        test_pred = sess.run(predict, feed_dict=test_feed_dict)

# Save predictions
save_predictions(test_pred, file_predictions)
'''

max_stars_repo_path: fake news challenge (FNC-1)/ucl_fnc.py

max_issues_repo_path: fake news challenge (FNC-1)/ucl_fnc.py

max_issues_repo_name: kshitizbhansali/DeepLearn

max_forks_repo_path: fake news challenge (FNC-1)/ucl_fnc.py

max_forks_repo_name: kshitizbhansali/DeepLearn


text:
from sklearn.naive_bayes import GaussianNB
import numpy as np
from keras.preprocessing.sequence import pad_sequences

def run_gaussian(load_data):
    (x_train, y_train), (x_test, y_test) = load_data
    train_test = x_test
    new_list = []
    for i in range(0, len(train_test)):
       new_list.append(np.array(train_test[i]))
    new_list = np.array(new_list)
    train_test = new_list
    train_test = pad_sequences(train_test)
    gaus = GaussianNB()
    pred = gaus.fit(train_test, y_test).predict(train_test)
    print("Number of mislabeled points out of a total %d points : %d"
        % (train_test.shape[0],(y_test != pred).sum()))
    a = train_test.shape[0]
    b = (y_test != pred).sum()
    print("Acc precent: %d" % (float(b)/float(a)*100.0))

def testing(load_data):
    (x_train, y_train), (x_test, y_test) = load_data
    print("type is: ",type(x_test[0]))
    new_list = []
    for i in range(0, len(x_test)):
       new_list.append(np.array(x_test[i]))
    new_list = np.array(new_list)
    print(new_list[0])
    print("type of newlist: ",type(new_list))
    print("type of newlist item: ",type(new_list[0]))


text:
import collections
import inspect
import typing
from contextlib import suppress
from copy import deepcopy
from enum import Enum

import magicgui
import numpy as np
from magicgui.widgets import ComboBox, Widget, create_widget
from napari.layers.base import Layer
from packaging.version import parse as parse_version
from qtpy.QtCore import Signal
from qtpy.QtGui import QHideEvent, QPainter, QPaintEvent, QResizeEvent
from qtpy.QtWidgets import (
    QAbstractSpinBox,
    QApplication,
    QCheckBox,
    QComboBox,
    QFormLayout,
    QLabel,
    QLineEdit,
    QMessageBox,
    QScrollArea,
    QStackedLayout,
    QVBoxLayout,
    QWidget,
)
from superqt import QEnumComboBox

from PartSeg.common_gui.error_report import ErrorDialog
from PartSegCore.algorithm_describe_base import AlgorithmDescribeBase, AlgorithmProperty, ROIExtractionProfile
from PartSegCore.channel_class import Channel
from PartSegCore.segmentation.algorithm_base import (
    ROIExtractionAlgorithm,
    ROIExtractionResult,
    SegmentationLimitException,
)
from PartSegImage import Image

from ..common_backend.base_settings import BaseSettings
from ..common_backend.segmentation_thread import SegmentationThread
from .universal_gui_part import ChannelComboBox, CustomDoubleSpinBox, CustomSpinBox

def update(d, u):
    if not isinstance(d, typing.MutableMapping):
        d = {}
    for k, v in u.items():
        d[k] = update(d.get(k, {}), v) if isinstance(v, collections.abc.Mapping) else v
    return d

def _pretty_print(data, indent=2) -> str:
    if isinstance(data, typing.Mapping):
        res = "\n"
        for k, v in data.items():
            res += f"{' ' * indent}{k}: {_pretty_print(v, indent+2)}\n"
        return res[:-1]
    return str(data)

class ProfileSelect(QComboBox):
    def __init__(self):
        super().__init__()
        self._settings = None

    def add_settings(self, settings: BaseSettings):
        self._settings = settings
        if hasattr(settings, "roi_profiles"):
            self.addItems(settings.roi_profiles.keys())

    def get_value(self):
        if self._settings is not None and hasattr(self._settings, "roi_profiles"):
            return self._settings.roi_profiles[self.currentText()]
        return None

    def set_value(self, value):
        if (
            self._settings is not None
            and hasattr(self._settings, "roi_profiles")
            and value.name in self._settings.roi_profiles
        ):
            self.setCurrentText(value.name)

class QtAlgorithmProperty(AlgorithmProperty):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self._widget = self._get_field()
        self.change_fun = self.get_change_signal(self._widget)
        self._getter, self._setter = self.get_getter_and_setter_function(self._widget)
        with suppress(TypeError, ValueError):
            self._setter(self._widget, self.default_value)

    def get_value(self):
        return self._getter(self._widget)

    def recursive_get_values(self):
        if isinstance(self._widget, SubAlgorithmWidget):
            return self._widget.recursive_get_values()
        return self.get_value()

    def set_value(self, val):
        """set value of widget"""
        with suppress(TypeError, ValueError):
            return self._setter(self._widget, val)

    def get_field(self) -> QWidget:
        """
        Get representing widget
        :return:
        :rtype:
        """
        return self._widget

    @classmethod
    def from_algorithm_property(cls, ob):
        """
        Create class instance base on :py:class:`.AlgorithmProperty` instance

        :type ob: AlgorithmProperty | str
        :param ob: AlgorithmProperty object or label
        :return: QtAlgorithmProperty | QLabel
        """
        if isinstance(ob, AlgorithmProperty):
            return cls(
                name=ob.name,
                user_name=ob.user_name,
                default_value=ob.default_value,
                options_range=ob.range,
                single_steep=ob.single_step,
                value_type=ob.value_type,
                possible_values=ob.possible_values,
                help_text=ob.help_text,
                per_dimension=ob.per_dimension,
            )
        if isinstance(ob, str):
            return QLabel(ob)
        raise ValueError(f"unknown parameter type {type(ob)} of {ob}")

    @staticmethod
    def _get_numeric_field(ap: AlgorithmProperty):
        if issubclass(ap.value_type, int):
            res = CustomSpinBox()
            if not isinstance(ap.default_value, int):
                raise ValueError(
                    f"Incompatible types. default_value should be type of int. Is {type(ap.default_value)}"
                )
        else:  # issubclass(ap.value_type, float):
            res = CustomDoubleSpinBox()
            if not isinstance(ap.default_value, float):
                raise ValueError(
                    f"Incompatible types. default_value should be type of float. Is {type(ap.default_value)}"
                )
        if ap.default_value < res.minimum():
            res.setMinimum(ap.default_value)
        if ap.range is not None:
            res.setRange(*ap.range)
        return res

    @classmethod
    def _get_field_from_value_type(cls, ap: AlgorithmProperty):
        if issubclass(ap.value_type, Channel):
            res = ChannelComboBox()
            res.change_channels_num(10)
        elif issubclass(ap.value_type, AlgorithmDescribeBase):
            res = SubAlgorithmWidget(ap)
        elif issubclass(ap.value_type, bool):
            res = QCheckBox()
        elif issubclass(ap.value_type, (float, int)):
            res = cls._get_numeric_field(ap)
        elif issubclass(ap.value_type, Enum):
            # noinspection PyTypeChecker
            res = QEnumComboBox(enum_class=ap.value_type)
        elif issubclass(ap.value_type, str):
            res = QLineEdit()
        elif issubclass(ap.value_type, ROIExtractionProfile):
            res = ProfileSelect()
        elif issubclass(ap.value_type, list):
            res = QComboBox()
            res.addItems(list(map(str, ap.possible_values)))
        else:
            res = create_widget(annotation=ap.value_type, options={})
        return res

    def _get_field(self) -> typing.Union[QWidget, Widget]:
        """
        Get proper widget for given field type. Overwrite if would like to support new data types.
        """
        if self.per_dimension:
            self.per_dimension = False
            prop = self.from_algorithm_property(self)
            self.per_dimension = True
            res = ListInput(prop, 3)
        elif not inspect.isclass(self.value_type):
            res = create_widget(annotation=self.value_type, options={})
        elif hasattr(self.value_type, "get_object"):
            res = self.value_type.get_object()
        else:
            res = self._get_field_from_value_type(self)
        tool_tip_text = self.help_text or ""
        tool_tip_text += f" default value: {_pretty_print(self.default_value)}"
        if isinstance(res, QWidget):
            res.setToolTip(tool_tip_text)
        if isinstance(res, Widget):
            res.tooltip = tool_tip_text  # pylint: disable=W0201 # false positive
        return res

    @staticmethod
    def get_change_signal(widget: typing.Union[QWidget, Widget]):
        if isinstance(widget, Widget):
            return widget.changed
        if isinstance(widget, QComboBox):
            return widget.currentIndexChanged
        if isinstance(widget, QCheckBox):
            return widget.stateChanged
        if isinstance(widget, QAbstractSpinBox):
            return widget.valueChanged
        if isinstance(widget, QLineEdit):
            return widget.textChanged
        if isinstance(widget, SubAlgorithmWidget):
            return widget.values_changed
        if isinstance(widget, ListInput):
            return widget.change_signal
        if hasattr(widget, "values_changed"):
            return widget.values_changed
        raise ValueError(f"Unsupported type: {type(widget)}")

    @staticmethod
    def get_getter_and_setter_function(
        widget: typing.Union[QWidget, Widget],
    ) -> typing.Tuple[
        typing.Callable[
            [
                QWidget,
            ],
            typing.Any,
        ],
        typing.Callable[[QWidget, typing.Any], None],  # noqa E231
    ]:
        """
        For each widget type return proper functions. This functions need instance as first argument

        :return: (getter, setter)
        """
        if isinstance(widget, Widget):
            return _value_get, _value_set
        if isinstance(widget, ProfileSelect):
            return widget.__class__.get_value, widget.__class__.set_value
        if isinstance(widget, ChannelComboBox):
            return widget.__class__.get_value, widget.__class__.set_value
        if isinstance(widget, QEnumComboBox):
            return widget.__class__.currentEnum, widget.__class__.setCurrentEnum
        if isinstance(widget, QComboBox):
            return widget.__class__.currentText, widget.__class__.setCurrentText
        if isinstance(widget, QCheckBox):
            return widget.__class__.isChecked, widget.__class__.setChecked
        if isinstance(widget, CustomSpinBox):
            return widget.__class__.value, widget.__class__.setValue
        if isinstance(widget, CustomDoubleSpinBox):
            return widget.__class__.value, widget.__class__.setValue
        if isinstance(widget, QLineEdit):
            return widget.__class__.text, widget.__class__.setText
        if isinstance(widget, SubAlgorithmWidget):
            return widget.__class__.get_values, widget.__class__.set_values
        if isinstance(widget, ListInput):
            return widget.__class__.get_value, widget.__class__.set_value
        if hasattr(widget, "get_value") and hasattr(widget, "set_value"):
            return widget.__class__.get_value, widget.__class__.set_value
        raise ValueError(f"Unsupported type: {type(widget)}")

class ListInput(QWidget):
    change_signal = Signal()

    def __init__(self, property_el: QtAlgorithmProperty, length):
        super().__init__()
        self.input_list = [property_el.from_algorithm_property(property_el) for _ in range(length)]
        layout = QVBoxLayout()
        for el in self.input_list:
            el.change_fun.connect(self.change_signal.emit)
            layout.addWidget(el.get_field())
        self.setLayout(layout)

    def get_value(self):
        return [x.get_value() for x in self.input_list]

    def set_value(self, value):
        if not isinstance(value, (list, tuple)):
            value = [value for _ in range(len(self.input_list))]
        for f, val in zip(self.input_list, value):
            f.set_value(val)

def _any_arguments(fun):
    if parse_version(magicgui.__version__) >= parse_version("0.3.0"):

        def _any():
            fun()

    else:

        def _any(*_):
            fun()

    return _any

class FormWidget(QWidget):
    value_changed = Signal()

    def __init__(
        self,
        fields: typing.List[AlgorithmProperty],
        start_values=None,
        dimension_num=1,
        settings: typing.Optional[BaseSettings] = None,
    ):
        super().__init__()
        if start_values is None:
            start_values = {}
        self.widgets_dict: typing.Dict[str, QtAlgorithmProperty] = {}
        self.channels_chose: typing.List[typing.Union[ChannelComboBox, SubAlgorithmWidget]] = []
        layout = QFormLayout()
        layout.setContentsMargins(10, 0, 10, 0)
        # layout.setVerticalSpacing(0)
        element_list = self._element_list(fields)
        for el in element_list:
            if isinstance(el, QLabel):
                layout.addRow(el)
                continue
            self._add_to_layout(layout, el, start_values, settings)
        self.setLayout(layout)
        self.value_changed.connect(self.update_size)

    def _add_to_layout(self, layout, ap: QtAlgorithmProperty, start_values: typing.MutableMapping, settings):
        label = QLabel(ap.user_name)
        if ap.help_text:
            label.setToolTip(ap.help_text)
        self.widgets_dict[ap.name] = ap
        ap.change_fun.connect(_any_arguments(self.value_changed.emit))
        if isinstance(ap.get_field(), SubAlgorithmWidget):
            layout.addRow(label, ap.get_field().choose)
            layout.addRow(ap.get_field())
            if ap.name in start_values:
                ap.get_field().set_starting(start_values[ap.name])
            ap.change_fun.connect(_any_arguments(self.value_changed.emit))
            return
        if isinstance(ap.get_field(), Widget):
            layout.addRow(label, ap.get_field().native)
            return
        layout.addRow(label, ap.get_field())
        # noinspection PyUnresolvedReferences
        if issubclass(ap.value_type, Channel):
            # noinspection PyTypeChecker
            self.channels_chose.append(ap.get_field())
        if issubclass(ap.value_type, ROIExtractionProfile):
            # noinspection PyTypeChecker
            ap.get_field().add_settings(settings)
        if ap.name in start_values:
            with suppress(KeyError, ValueError, TypeError):
                ap.set_value(start_values[ap.name])

    @staticmethod
    def _element_list(fields) -> typing.Iterable[QtAlgorithmProperty]:
        return map(QtAlgorithmProperty.from_algorithm_property, fields)

    def has_elements(self):
        return len(self.widgets_dict) > 0

    def update_size(self):
        self.setMinimumHeight(self.layout().minimumSize().height())

    def get_values(self):
        return {name: el.get_value() for name, el in self.widgets_dict.items()}

    def recursive_get_values(self):
        return {name: el.recursive_get_values() for name, el in self.widgets_dict.items()}

    def set_values(self, values: dict):
        for name, value in values.items():
            if name in self.widgets_dict:
                self.widgets_dict[name].set_value(value)

    def image_changed(self, image: Image):
        if not image:
            return
        for channel_widget in self.channels_chose:
            if isinstance(channel_widget, ChannelComboBox):
                channel_widget.change_channels_num(image.channels)
            else:
                channel_widget.change_channels_num(image)

class SubAlgorithmWidget(QWidget):
    values_changed = Signal()

    def __init__(self, algorithm_property: AlgorithmProperty):
        super().__init__()
        if not isinstance(algorithm_property.possible_values, typing.MutableMapping):
            raise ValueError(
                "algorithm_property.possible_values should be dict." f"It is {type(algorithm_property.possible_values)}"
            )
        if not isinstance(algorithm_property.default_value, str):
            raise ValueError(
                "algorithm_property.default_value should be str." f"It is {type(algorithm_property.default_value)}"
            )
        self.starting_values = {}
        self.property = algorithm_property
        self.widgets_dict: typing.Dict[str, FormWidget] = {}
        # TODO protect for recursion
        widget = FormWidget(algorithm_property.possible_values[algorithm_property.default_value].get_fields())
        widget.layout().setContentsMargins(0, 0, 0, 0)
        widget.value_changed.connect(self.values_changed)

        self.widgets_dict[algorithm_property.default_value] = widget
        self.choose = QComboBox(self)
        self.choose.addItems(list(algorithm_property.possible_values.keys()))
        self.setContentsMargins(0, 0, 0, 0)

        self.choose.setCurrentText(algorithm_property.default_value)

        self.choose.currentTextChanged.connect(self.algorithm_choose)
        # self.setStyleSheet("border: 1px solid red")
        layout = QVBoxLayout()
        layout.setContentsMargins(4, 4, 4, 4)
        layout.addWidget(widget)
        if not widget.has_elements():
            widget.hide()
            self.hide()
        tmp_widget = QWidget(self)
        # tmp_widget.setMinimumHeight(5000)
        layout.addWidget(tmp_widget)
        self.tmp_widget = tmp_widget
        self.setLayout(layout)

    def set_starting(self, starting_values):
        self.starting_values = starting_values
        # self.set_values(starting_values)

    def set_values(self, val: typing.Mapping):
        if not isinstance(val, typing.Mapping):
            return
        self.choose.setCurrentText(val["name"])
        if val["name"] not in self.widgets_dict:
            self.algorithm_choose(val["name"])
        if val["name"] in self.widgets_dict:
            self.widgets_dict[val["name"]].set_values(val["values"])

    def recursive_get_values(self):
        return {name: el.recursive_get_values() for name, el in self.widgets_dict.items()}

    def get_values(self):
        name = self.choose.currentText()
        values = self.widgets_dict[name].get_values()
        return {"name": name, "values": values}

    def change_channels_num(self, image: Image):
        for i in range(self.layout().count()):
            el = self.layout().itemAt(i)
            if el.widget() and isinstance(el.widget(), FormWidget):
                el.widget().image_changed(image)

    def algorithm_choose(self, name):
        if name not in self.widgets_dict:
            if name not in self.property.possible_values:
                return
            start_dict = {} if name not in self.starting_values else self.starting_values[name]
            try:
                self.widgets_dict[name] = FormWidget(
                    self.property.possible_values[name].get_fields(), start_values=start_dict
                )
            except KeyError as e:
                raise e
            self.widgets_dict[name].layout().setContentsMargins(0, 0, 0, 0)
            self.layout().addWidget(self.widgets_dict[name])
            self.widgets_dict[name].value_changed.connect(self.values_changed)
        widget = self.widgets_dict[name]
        for i in range(self.layout().count()):
            lay_elem = self.layout().itemAt(i)
            if lay_elem.widget():
                lay_elem.widget().hide()
        if widget.has_elements():
            self.show()
            widget.show()
        else:
            self.hide()
        self.values_changed.emit()

    def showEvent(self, _event):
        # workaround for changing size
        self.tmp_widget.hide()

    def paintEvent(self, event: QPaintEvent):
        name = self.choose.currentText()
        if self.widgets_dict[name].has_elements() and event.rect().top() == 0 and event.rect().left() == 0:
            painter = QPainter(self)
            painter.drawRect(event.rect())

class BaseAlgorithmSettingsWidget(QScrollArea):
    values_changed = Signal()
    algorithm_thread: SegmentationThread

    def __init__(self, settings: BaseSettings, name, algorithm: typing.Type[ROIExtractionAlgorithm]):
        """
        For algorithm which works on one channel
        """
        super().__init__()
        self.settings = settings
        self.widget_list = []
        self.name = name
        self.algorithm = algorithm
        main_layout = QVBoxLayout()
        self.info_label = QLabel()
        self.info_label.setHidden(True)
        main_layout.addWidget(self.info_label)
        start_values = settings.get(f"algorithm_widget_state.{name}", {})
        self.form_widget = self._form_widget(algorithm, start_values=start_values)
        self.form_widget.value_changed.connect(self.values_changed.emit)
        # self.form_widget.setMinimumHeight(1500)
        self.setWidget(self.form_widget)
        value_dict = self.settings.get(f"algorithms.{self.name}", {})
        self.set_values(value_dict)
        # self.settings.image_changed[Image].connect(self.image_changed)
        self.algorithm_thread = SegmentationThread(algorithm())
        self.algorithm_thread.info_signal.connect(self.show_info)
        self.algorithm_thread.exception_occurred.connect(self.exception_occurred)

    @staticmethod
    def _form_widget(algorithm, start_values) -> FormWidget:
        return FormWidget(algorithm.get_fields(), start_values=start_values)

    @staticmethod
    def exception_occurred(exc: Exception):
        if isinstance(exc, SegmentationLimitException):
            mess = QMessageBox()
            mess.setIcon(QMessageBox.Critical)
            mess.setText("During segmentation process algorithm meet limitations:\n" + "\n".join(exc.args))
            mess.setWindowTitle("Segmentation limitations")
            mess.exec_()
            return
        if isinstance(exc, RuntimeError) and exc.args[0].startswith(
            "Exception thrown in SimpleITK KittlerIllingworthThreshold"
        ):
            mess = QMessageBox()
            mess.setIcon(QMessageBox.Critical)
            mess.setText("Fail to apply Kittler Illingworth to current data\n" + exc.args[0].split("\n")[1])
            mess.setWindowTitle("Segmentation limitations")
            mess.exec_()
            return
        dial = ErrorDialog(exc, "Error during segmentation", f"{QApplication.instance().applicationName()}")
        dial.exec_()

    def show_info(self, text):
        self.info_label.setText(text)
        self.info_label.setVisible(True)

    def image_changed(self, image: Image):
        self.form_widget.image_changed(image)
        self.algorithm_thread.algorithm.set_image(image)

    def set_mask(self, mask):
        self.algorithm_thread.algorithm.set_mask(mask)

    def mask(self) -> typing.Optional[np.ndarray]:
        return self.algorithm_thread.algorithm.mask

    def set_values(self, values_dict):
        self.form_widget.set_values(values_dict)

    def get_values(self):
        return self.form_widget.get_values()

    def channel_num(self):
        return self.channels_chose.currentIndex()

    def execute(self, exclude_mask=None):
        values = self.get_values()
        self.settings.set(f"algorithms.{self.name}", deepcopy(values))
        self.algorithm_thread.set_parameters(**values)
        self.algorithm_thread.start()

    def hideEvent(self, a0: QHideEvent):
        self.algorithm_thread.clean()

    def resizeEvent(self, event: QResizeEvent):
        if self.height() < self.form_widget.height():
            self.setMinimumWidth(self.form_widget.width() + 20)
        else:
            self.setMinimumWidth(self.form_widget.width() + 10)
        super().resizeEvent(event)

    def recursive_get_values(self):
        return self.form_widget.recursive_get_values()

class AlgorithmSettingsWidget(BaseAlgorithmSettingsWidget):
    def execute(self, exclude_mask=None):
        self.algorithm_thread.algorithm.set_image(self.settings.image)
        super().execute(exclude_mask)

class InteractiveAlgorithmSettingsWidget(BaseAlgorithmSettingsWidget):
    algorithm_thread: SegmentationThread

    def __init__(self, settings, name, algorithm: typing.Type[ROIExtractionAlgorithm], selector: typing.List[QWidget]):
        super().__init__(settings, name, algorithm)
        self.selector = selector
        self.algorithm_thread.finished.connect(self.enable_selector)
        self.algorithm_thread.started.connect(self.disable_selector)
        # noinspection PyUnresolvedReferences
        if hasattr(settings, "mask_changed"):
            settings.mask_changed.connect(self.change_mask)

    def value_updated(self):
        if not self.parent().interactive:
            return
        self.execute()

    def change_mask(self):
        if not self.isVisible():
            return
        self.algorithm_thread.algorithm.set_mask(self.settings.mask)

    def disable_selector(self):
        for el in self.selector:
            el.setDisabled(True)

    def enable_selector(self):
        for el in self.selector:
            el.setEnabled(True)

    def get_segmentation_profile(self) -> ROIExtractionProfile:
        return ROIExtractionProfile("", self.algorithm.get_name(), self.get_values())

class AlgorithmChooseBase(QWidget):
    finished = Signal()
    started = Signal()
    result = Signal(ROIExtractionResult)
    value_changed = Signal()
    progress_signal = Signal(str, int)
    algorithm_changed = Signal(str)

    def __init__(
        self, settings: BaseSettings, algorithms: typing.Dict[str, typing.Type[ROIExtractionAlgorithm]], parent=None
    ):
        super().__init__(parent)
        self.settings = settings
        self.algorithms = algorithms
        settings.algorithm_changed.connect(self.updated_algorithm)
        self.stack_layout = QStackedLayout()
        self.algorithm_choose = QComboBox()
        self.algorithm_dict: typing.Dict[str, BaseAlgorithmSettingsWidget] = {}
        self.algorithm_choose.currentTextChanged.connect(self.change_algorithm)
        self.add_widgets_to_algorithm()

        # self.setMinimumWidth(370)

        self.setContentsMargins(0, 0, 0, 0)
        layout = QVBoxLayout()
        layout.setContentsMargins(0, 0, 0, 0)
        layout.addWidget(self.algorithm_choose)
        layout.addLayout(self.stack_layout)
        self.setLayout(layout)

    @staticmethod
    def _algorithm_widget(settings, name, val) -> InteractiveAlgorithmSettingsWidget:
        return InteractiveAlgorithmSettingsWidget(settings, name, val, [])

    def add_widgets_to_algorithm(self):
        self.algorithm_choose.blockSignals(True)
        self.algorithm_choose.clear()
        for name, val in self.algorithms.items():
            self.algorithm_choose.addItem(name)
            widget = self._algorithm_widget(self.settings, name, val)
            self.algorithm_dict[name] = widget
            widget.algorithm_thread.execution_done.connect(self.result.emit)
            widget.algorithm_thread.finished.connect(self.finished.emit)
            widget.algorithm_thread.started.connect(self.started.emit)
            widget.algorithm_thread.progress_signal.connect(self.progress_signal.emit)
            widget.values_changed.connect(self.value_changed.emit)
            self.stack_layout.addWidget(widget)
        name = self.settings.get("current_algorithm", "")
        self.algorithm_choose.blockSignals(False)
        if name:
            self.algorithm_choose.setCurrentText(name)

    def reload(self, algorithms=None):
        if algorithms is not None:
            self.algorithms = algorithms
        for _ in range(self.stack_layout.count()):
            widget: InteractiveAlgorithmSettingsWidget = self.stack_layout.takeAt(0).widget()
            widget.algorithm_thread.execution_done.disconnect()
            widget.algorithm_thread.finished.disconnect()
            widget.algorithm_thread.started.disconnect()
            widget.algorithm_thread.progress_signal.disconnect()
            widget.values_changed.disconnect()
            widget.deleteLater()
        self.algorithm_dict = {}
        self.add_widgets_to_algorithm()

    def updated_algorithm(self):
        self.change_algorithm(
            self.settings.last_executed_algorithm,
            self.settings.get(f"algorithms.{self.settings.last_executed_algorithm}"),
        )

    def recursive_get_values(self):
        result = {key: widget.recursive_get_values() for key, widget in self.algorithm_dict.items()}

        self.settings.set("algorithm_widget_state", update(self.settings.get("algorithm_widget_state", {}), result))
        return result

    def change_algorithm(self, name, values: dict = None):
        self.settings.set("current_algorithm", name)
        widget = self.stack_layout.currentWidget()
        self.blockSignals(True)
        if name != widget.name:
            widget = self.algorithm_dict[name]
            self.stack_layout.setCurrentWidget(widget)
            widget.image_changed(self.settings.image)
            if hasattr(widget, "set_mask") and hasattr(self.settings, "mask"):
                widget.set_mask(self.settings.mask)
        elif values is None:
            self.blockSignals(False)
            return
        if values is not None:
            widget.set_values(values)
        self.algorithm_choose.setCurrentText(name)
        self.blockSignals(False)
        self.algorithm_changed.emit(name)

    def current_widget(self) -> InteractiveAlgorithmSettingsWidget:
        return self.stack_layout.currentWidget()

    def current_parameters(self) -> ROIExtractionProfile:
        widget = self.current_widget()
        return ROIExtractionProfile("", widget.name, widget.get_values())

    def get_info_text(self):
        return self.current_widget().algorithm_thread.get_info_text()

class AlgorithmChoose(AlgorithmChooseBase):
    def __init__(
        self, settings: BaseSettings, algorithms: typing.Dict[str, typing.Type[ROIExtractionAlgorithm]], parent=None
    ):
        super().__init__(settings, algorithms, parent)
        self.settings.image_changed.connect(self.image_changed)

    def image_changed(self):
        current_widget: InteractiveAlgorithmSettingsWidget = self.stack_layout.currentWidget()
        current_widget.image_changed(self.settings.image)
        if hasattr(self.settings, "mask") and hasattr(current_widget, "change_mask"):
            current_widget.change_mask()

# AbstractAlgorithmSettingsWidget.register(AlgorithmSettingsWidget)

def _value_get(self):
    return self.value

def _value_set(self, value):
    if isinstance(self, ComboBox) and issubclass(self.annotation, Layer) and isinstance(value, str):
        for el in self.choices:
            if el.name == value:
                self.value = el
                return
    self.value = value

max_stars_repo_name: neuromusic/PartSeg

max_issues_repo_name: neuromusic/PartSeg

max_forks_repo_name: neuromusic/PartSeg


text:
import os
import math
from itertools import chain

import numpy as np
import torch
import torch.nn as nn
from torch.optim import Adam
from firelab import BaseTrainer
from torchvision.datasets import MNIST, FashionMNIST
from torchvision.transforms import ToTensor
from torch.utils.data import DataLoader
from tqdm import tqdm

from src.models import LineModel, SimpleModel
from src.utils import validate, validate_weights, linerp, get_weights_linerp
from src.utils import compute_weights_entropy_linerp, compute_activations_entropy_linerp
from src.plotting_utils import generate_linerp_plot, generate_acts_entropy_linerp_plot, generate_weights_entropy_linerp_plot

class LineTrainer(BaseTrainer):
    def __init__(self, config):
        super(LineTrainer, self).__init__(config)

    def init_dataloaders(self):
        batch_size = self.config.hp.batch_size
        project_path = self.config.firelab.project_path
        data_dir = os.path.join(project_path, self.config.data_dir)

        data_train = FashionMNIST(data_dir, download=True, train=True, transform=ToTensor())
        data_test = FashionMNIST(data_dir, download=True, train=False, transform=ToTensor())

        self.train_dataloader = DataLoader(data_train, batch_size=batch_size, num_workers=3, shuffle=True)
        self.val_dataloader = DataLoader(data_test, batch_size=batch_size, num_workers=3, shuffle=False)

    def init_models(self):
        self.model = LineModel().to(self.config.firelab.device_name)

    def init_criterions(self):
        self.criterion = nn.CrossEntropyLoss(reduction='none')

    def init_optimizers(self):
        self.optim = Adam(self.model.parameters(), lr=self.config.hp.lr)

    def train_on_batch(self, batch):
        x = batch[0].to(self.config.firelab.device_name)
        y = batch[1].to(self.config.firelab.device_name)

        preds = self.model(x)
        clf_loss = self.criterion(preds, y).mean()
        acc = (preds.argmax(dim=1) == y).float().mean()

        dist = (self.model.w_1 - self.model.w_2).norm()
        w_1_len = self.model.w_1.norm()
        w_2_len = self.model.w_2.norm()

        if self.config.hp.dist_reg_coef == 0:
            final_loss = clf_loss
        else:
            final_loss = clf_loss - self.config.hp.dist_reg_coef * dist

        self.optim.zero_grad()
        final_loss.backward()
        self.optim.step()

        self.writer.add_scalar('Train/loss', clf_loss.item(), self.num_iters_done)
        self.writer.add_scalar('Train/acc', acc.item(), self.num_iters_done)
        self.writer.add_scalar('Reg/distance2', dist.item(), self.num_iters_done)
        self.writer.add_scalar('Reg/w_1_norm2', w_1_len.item(), self.num_iters_done)
        self.writer.add_scalar('Reg/w_2_norm2', w_2_len.item(), self.num_iters_done)

    def validate(self):
        self.model.eval()
        clf_loss, acc = validate(self.model, self.val_dataloader, self.criterion)

        self.writer.add_scalar('Val/loss', clf_loss.item(), self.num_iters_done)
        self.writer.add_scalar('Val/acc', acc.item(), self.num_iters_done)

    def after_training_hook(self):
        self.visualize_linerp()
        self.visualize_entropy_linerp()

    def visualize_linerp(self):
        dummy_model = SimpleModel().to(self.config.firelab.device_name)

        w1to2_linerp_scores_train = linerp(self.model.w_1, self.model.w_2, dummy_model, self.train_dataloader)
        w1to2_linerp_scores_test = linerp(self.model.w_1, self.model.w_2, dummy_model, self.val_dataloader)

        self.writer.add_figure('w_1 to w_2 linerp/accuracy', generate_linerp_plot(
            [s[1] for s in w1to2_linerp_scores_train],
            [s[1] for s in w1to2_linerp_scores_test],
        ))

        self.writer.add_figure('w_1 to w_2 linerp/loss', generate_linerp_plot(
            [s[0] for s in w1to2_linerp_scores_train],
            [s[0] for s in w1to2_linerp_scores_test],
        ))

    def visualize_entropy_linerp(self):
        # We are visualizing only between w_1 and w_2 here
        dummy_model = SimpleModel().to(self.config.firelab.device_name)

        # acts_ent_linerp_train = compute_activations_entropy_linerp(self.model.w_1, self.model.w_2, dummy_model.nn, self.train_dataloader)
        acts_ent_linerp_test = compute_activations_entropy_linerp(self.model.w_1, self.model.w_2, dummy_model.nn, self.val_dataloader)
        weights_ent_linerp = compute_weights_entropy_linerp(self.model.w_1, self.model.w_2)

        # self.writer.add_figure('activations_entropy_linerp/train', generate_acts_entropy_linerp_plot(acts_ent_linerp_train))
        self.writer.add_figure('activations_entropy_linerp/test', generate_acts_entropy_linerp_plot(acts_ent_linerp_test))
        self.writer.add_figure('weights_entropy_linerp', generate_weights_entropy_linerp_plot(weights_ent_linerp))

        for i, w in enumerate(get_weights_linerp(self.model.w_1, self.model.w_2)):
            self.writer.add_histogram('Weights histogram', w.detach().cpu(), i)

max_stars_repo_name: universome/loss-patterns

max_issues_repo_name: universome/loss-patterns

max_forks_repo_name: universome/loss-patterns