text:
import logging
import tempfile
import os
import torch
from collections import OrderedDict
from tqdm import tqdm

from maskrcnn_benchmark.modeling.roi_heads.mask_head.inference import Masker
from maskrcnn_benchmark.structures.bounding_box import BoxList
from maskrcnn_benchmark.structures.boxlist_ops import boxlist_iou

from pycocotools.cocoeval import COCOeval, Params

def do_coco_evaluation(
    dataset,
    predictions,
    box_only,
    output_folder,
    iou_types,
    expected_results,
    expected_results_sigma_tol,
):
    logger = logging.getLogger("maskrcnn_benchmark.inference")

    if box_only:
        logger.info("Evaluating bbox proposals")
        areas = {"all": "", "small": "s", "medium": "m", "large": "l"}
        res = COCOResults("box_proposal")
        for limit in [100, 1000]:
            for area, suffix in areas.items():
                stats = evaluate_box_proposals(
                    predictions, dataset, area=area, limit=limit
                )
                key = "AR{}@{:d}".format(suffix, limit)
                res.results["box_proposal"][key] = stats["ar"].item()
        logger.info(res)
        check_expected_results(res, expected_results, expected_results_sigma_tol)
        if output_folder:
            torch.save(res, os.path.join(output_folder, "box_proposals.pth"))
        return
    logger.info("Preparing results for COCO format")
    coco_results = {}
    if "bbox" in iou_types:
        logger.info("Preparing bbox results")
        coco_results["bbox"] = prepare_for_coco_detection(predictions, dataset)
    if "segm" in iou_types:
        logger.info("Preparing segm results")
        coco_results["segm"] = prepare_for_coco_segmentation(predictions, dataset)
    if 'keypoints' in iou_types:
        logger.info('Preparing keypoints results')
        coco_results['keypoints'] = prepare_for_coco_keypoint(predictions, dataset)
    if 'depth' in iou_types:
        logger.info('Preparing depth results')
        coco_results['depth'] = prepare_for_coco_regression(predictions, dataset)

    results = COCOResults(*iou_types)
    logger.info("Evaluating predictions")
    for iou_type in iou_types:
        with tempfile.NamedTemporaryFile() as f:
            file_path = f.name
            if output_folder:
                file_path = os.path.join(output_folder, iou_type + ".json")
            res = evaluate_predictions_on_coco(
                dataset.coco, coco_results[iou_type], file_path, iou_type
            )
            results.update(res)
    logger.info(results)
    check_expected_results(results, expected_results, expected_results_sigma_tol)
    if output_folder:
        torch.save(results, os.path.join(output_folder, "coco_results.pth"))
    return results, coco_results

def prepare_for_coco_detection(predictions, dataset):
    # assert isinstance(dataset, COCODataset)
    coco_results = []
    for image_id, prediction in enumerate(predictions):
        original_id = dataset.id_to_img_map[image_id]
        if len(prediction) == 0:
            continue

        img_info = dataset.get_img_info(image_id)
        image_width = img_info["width"]
        image_height = img_info["height"]
        prediction = prediction.resize((image_width, image_height))
        prediction = prediction.convert("xywh")

        boxes = prediction.bbox.tolist()
        scores = prediction.get_field("scores").tolist()
        labels = prediction.get_field("labels").tolist()

        mapped_labels = [dataset.contiguous_category_id_to_json_id[i] for i in labels]

        coco_results.extend(
            [
                {
                    "image_id": original_id,
                    "category_id": mapped_labels[k],
                    "bbox": box,
                    "score": scores[k],
                }
                for k, box in enumerate(boxes)
                # if hasattr(dataset, "class_filter_list") and mapped_labels[k] in dataset.class_filter_list
            ]
        )
    return coco_results

def prepare_for_coco_segmentation(predictions, dataset):
    import pycocotools.mask as mask_util
    import numpy as np

    masker = Masker(threshold=0.5, padding=1)
    # assert isinstance(dataset, COCODataset)
    coco_results = []
    for image_id, prediction in tqdm(enumerate(predictions)):
        original_id = dataset.id_to_img_map[image_id]
        if len(prediction) == 0:
            continue

        img_info = dataset.get_img_info(image_id)
        image_width = img_info["width"]
        image_height = img_info["height"]
        prediction = prediction.resize((image_width, image_height))
        masks = prediction.get_field("mask")
        # t = time.time()
        # Masker is necessary only if masks haven't been already resized.
        if list(masks.shape[-2:]) != [image_height, image_width]:
            masks = masker(masks.expand(1, -1, -1, -1, -1), prediction)
            masks = masks[0]
        # logger.info('Time mask: {}'.format(time.time() - t))
        # prediction = prediction.convert('xywh')

        # boxes = prediction.bbox.tolist()
        scores = prediction.get_field("scores").tolist()
        labels = prediction.get_field("labels").tolist()

        # rles = prediction.get_field('mask')

        rles = [
            mask_util.encode(np.array(mask[0, :, :, np.newaxis], order="F"))[0]
            for mask in masks
        ]
        for rle in rles:
            rle["counts"] = rle["counts"].decode("utf-8")

        mapped_labels = [dataset.contiguous_category_id_to_json_id[i] for i in labels]

        coco_results.extend(
            [
                {
                    "image_id": original_id,
                    "category_id": mapped_labels[k],
                    "segmentation": rle,
                    "score": scores[k],
                }
                for k, rle in enumerate(rles)
            ]
        )
    return coco_results

def prepare_for_coco_keypoint(predictions, dataset):
    # assert isinstance(dataset, COCODataset)
    coco_results = []
    for image_id, prediction in enumerate(predictions):
        original_id = dataset.id_to_img_map[image_id]
        if len(prediction.bbox) == 0:
            continue

        # TODO replace with get_img_info?
        image_width = dataset.coco.imgs[original_id]['width']
        image_height = dataset.coco.imgs[original_id]['height']
        prediction = prediction.resize((image_width, image_height))
        prediction = prediction.convert('xywh')

        boxes = prediction.bbox.tolist()
        scores = prediction.get_field('scores').tolist()
        labels = prediction.get_field('labels').tolist()
        keypoints = prediction.get_field('keypoints')
        keypoints = keypoints.resize((image_width, image_height))
        keypoints = keypoints.keypoints.view(keypoints.keypoints.shape[0], -1).tolist()

        mapped_labels = [dataset.contiguous_category_id_to_json_id[i] for i in labels]

        coco_results.extend([{
            'image_id': original_id,
            'category_id': mapped_labels[k],
            'keypoints': keypoint,
            'score': scores[k]} for k, keypoint in enumerate(keypoints)])
    return coco_results

def prepare_for_coco_regression(predictions, dataset):
    # assert isinstance(dataset, COCODataset)
    coco_results = []
    for image_id, prediction in enumerate(predictions):
        original_id = dataset.id_to_img_map[image_id]
        if len(prediction.bbox) == 0:
            continue

        # TODO replace with get_img_info?
        image_width = dataset.coco.imgs[original_id]['width']
        image_height = dataset.coco.imgs[original_id]['height']
        prediction = prediction.resize((image_width, image_height))
        prediction = prediction.convert('xywh')

        boxes = prediction.bbox.tolist()
        scores = prediction.get_field('scores').tolist()
        labels = prediction.get_field('labels').tolist()
        regs = prediction.get_field('depths').tolist() # .convert("depth")

        mapped_labels = [dataset.contiguous_category_id_to_json_id[i] for i in labels]

        coco_results.extend([{
            'image_id': original_id,
            'category_id': mapped_labels[k],
            'bbox': boxes[k],
            'depth': reg,
            'score': scores[k]} for k, reg in enumerate(regs)])
    return coco_results

# inspired from Detectron
def evaluate_box_proposals(
    predictions, dataset, thresholds=None, area="all", limit=None
):
    """Evaluate detection proposal recall metrics. This function is a much
    faster alternative to the official COCO API recall evaluation code. However,
    it produces slightly different results.
    """
    # Record max overlap value for each gt box
    # Return vector of overlap values
    areas = {
        "all": 0,
        "small": 1,
        "medium": 2,
        "large": 3,
        "96-128": 4,
        "128-256": 5,
        "256-512": 6,
        "512-inf": 7,
    }
    area_ranges = [
        [0 ** 2, 1e5 ** 2],  # all
        [0 ** 2, 32 ** 2],  # small
        [32 ** 2, 96 ** 2],  # medium
        [96 ** 2, 1e5 ** 2],  # large
        [96 ** 2, 128 ** 2],  # 96-128
        [128 ** 2, 256 ** 2],  # 128-256
        [256 ** 2, 512 ** 2],  # 256-512
        [512 ** 2, 1e5 ** 2],
    ]  # 512-inf
    assert area in areas, "Unknown area range: {}".format(area)
    area_range = area_ranges[areas[area]]
    gt_overlaps = []
    num_pos = 0

    for image_id, prediction in enumerate(predictions):
        original_id = dataset.id_to_img_map[image_id]

        img_info = dataset.get_img_info(image_id)
        image_width = img_info["width"]
        image_height = img_info["height"]
        prediction = prediction.resize((image_width, image_height))

        # sort predictions in descending order
        # TODO maybe remove this and make it explicit in the documentation
        inds = prediction.get_field("objectness").sort(descending=True)[1]
        prediction = prediction[inds]

        ann_ids = dataset.coco.getAnnIds(imgIds=original_id)
        anno = dataset.coco.loadAnns(ann_ids)
        gt_boxes = [obj["bbox"] for obj in anno if obj["iscrowd"] == 0]
        gt_boxes = torch.as_tensor(gt_boxes).reshape(-1, 4)  # guard against no boxes
        gt_boxes = BoxList(gt_boxes, (image_width, image_height), mode="xywh").convert(
            "xyxy"
        )
        gt_areas = torch.as_tensor([obj["area"] for obj in anno if obj["iscrowd"] == 0])

        if len(gt_boxes) == 0:
            continue

        valid_gt_inds = (gt_areas >= area_range[0]) & (gt_areas <= area_range[1])
        gt_boxes = gt_boxes[valid_gt_inds]

        num_pos += len(gt_boxes)

        if len(gt_boxes) == 0:
            continue

        if len(prediction) == 0:
            continue

        if limit is not None and len(prediction) > limit:
            prediction = prediction[:limit]

        overlaps = boxlist_iou(prediction, gt_boxes)

        _gt_overlaps = torch.zeros(len(gt_boxes))
        for j in range(min(len(prediction), len(gt_boxes))):
            # find which proposal box maximally covers each gt box
            # and get the iou amount of coverage for each gt box
            max_overlaps, argmax_overlaps = overlaps.max(dim=0)

            # find which gt box is 'best' covered (i.e. 'best' = most iou)
            gt_ovr, gt_ind = max_overlaps.max(dim=0)
            assert gt_ovr >= 0
            # find the proposal box that covers the best covered gt box
            box_ind = argmax_overlaps[gt_ind]
            # record the iou coverage of this gt box
            _gt_overlaps[j] = overlaps[box_ind, gt_ind]
            assert _gt_overlaps[j] == gt_ovr
            # mark the proposal box and the gt box as used
            overlaps[box_ind, :] = -1
            overlaps[:, gt_ind] = -1

        # append recorded iou coverage level
        gt_overlaps.append(_gt_overlaps)
    gt_overlaps = torch.cat(gt_overlaps, dim=0)
    gt_overlaps, _ = torch.sort(gt_overlaps)

    if thresholds is None:
        step = 0.05
        thresholds = torch.arange(0.5, 0.95 + 1e-5, step, dtype=torch.float32)
    recalls = torch.zeros_like(thresholds)
    # compute recall for each iou threshold
    for i, t in enumerate(thresholds):
        recalls[i] = (gt_overlaps >= t).float().sum() / float(num_pos)
    # ar = 2 * np.trapz(recalls, thresholds)
    ar = recalls.mean()
    return {
        "ar": ar,
        "recalls": recalls,
        "thresholds": thresholds,
        "gt_overlaps": gt_overlaps,
        "num_pos": num_pos,
    }

def evaluate_predictions_on_coco(
    coco_gt, coco_results, json_result_file, iou_type="bbox"
):
    import json

    with open(json_result_file, "w") as f:
        json.dump(coco_results, f)

    from pycocotools.coco import COCO
    from pycocotools.cocoeval import COCOeval

    # coco_dt = coco_gt.loadRes(coco_results)
    coco_dt = coco_gt.loadRes(str(json_result_file)) if coco_results else COCO()
    if iou_type == "depth":
        coco_eval = COCORegeval(coco_gt, coco_dt, iou_type)
    else:
        coco_eval = COCOeval(coco_gt, coco_dt, iou_type)
    coco_eval.evaluate()
    coco_eval.accumulate()
    coco_eval.summarize()
    return coco_eval

class COCOResults(object):
    METRICS = {
        "bbox": ["AP", "AP50", "AP75", "APs", "APm", "APl"],
        "segm": ["AP", "AP50", "AP75", "APs", "APm", "APl"],
        "box_proposal": [
            "AR@100",
            "ARs@100",
            "ARm@100",
            "ARl@100",
            "AR@1000",
            "ARs@1000",
            "ARm@1000",
            "ARl@1000",
        ],
        "keypoints": ["AP", "AP50", "AP75", "APm", "APl"],
        "depth": ["AP", "D>1.25**2", "D>1.25", "APm", "APl"],
    }

    def __init__(self, *iou_types):
        allowed_types = ("box_proposal", "bbox", "segm", "keypoints", "depth")
        assert all(iou_type in allowed_types for iou_type in iou_types)
        results = OrderedDict()
        for iou_type in iou_types:
            results[iou_type] = OrderedDict(
                [(metric, -1) for metric in COCOResults.METRICS[iou_type]]
            )
        self.results = results

    def update(self, coco_eval):
        if coco_eval is None:
            return
        from pycocotools.cocoeval import COCOeval

        assert isinstance(coco_eval, COCOeval)
        s = coco_eval.stats
        iou_type = coco_eval.params.iouType
        res = self.results[iou_type]
        metrics = COCOResults.METRICS[iou_type]
        for idx, metric in enumerate(metrics):
            res[metric] = s[idx]

    def __repr__(self):
        results = '\n'
        for task, metrics in self.results.items():
            results += 'Task: {}\n'.format(task)
            metric_names = metrics.keys()
            metric_vals = ['{:.4f}'.format(v) for v in metrics.values()]
            results += (', '.join(metric_names) + '\n')
            results += (', '.join(metric_vals) + '\n')
        return results

def check_expected_results(results, expected_results, sigma_tol):
    if not expected_results:
        return

    logger = logging.getLogger("maskrcnn_benchmark.inference")
    for task, metric, (mean, std) in expected_results:
        actual_val = results.results[task][metric]
        lo = mean - sigma_tol * std
        hi = mean + sigma_tol * std
        ok = (lo < actual_val) and (actual_val < hi)
        msg = (
            "{} > {} sanity check (actual vs. expected): "
            "{:.3f} vs. mean={:.4f}, std={:.4}, range=({:.4f}, {:.4f})"
        ).format(task, metric, actual_val, mean, std, lo, hi)
        if not ok:
            msg = "FAIL: " + msg
            logger.error(msg)
        else:
            msg = "PASS: " + msg
            logger.info(msg)

import numpy as np
import datetime
import time
from collections import defaultdict
import copy

class RegParams(Params):
    def setRegParams(self):
        self.imgIds = []
        self.catIds = []
        # np.arange causes trouble.  the data point on arange is slightly larger than the true value
        self.iouThrs = np.linspace(.5, 0.95, np.round((0.95 - .5) / .05) + 1, endpoint=True)
        self.recThrs = np.linspace(.0, 1.00, np.round((1.00 - .0) / .01) + 1, endpoint=True)
        self.maxDets = [20]
        self.areaRng = [[0 ** 2, 1e5 ** 2], [32 ** 2, 96 ** 2], [96 ** 2, 1e5 ** 2]]
        self.areaRngLbl = ['all', 'medium', 'large']
        self.useCats = 1
        self.scoreThrs = 0.7

    def __init__(self, iouType='segm'):
        if iouType == 'segm' or iouType == 'bbox':
            self.setDetParams()
        elif iouType == 'keypoints':
            self.setKpParams()
        elif iouType == 'depth':
            self.setRegParams()
        else:
            raise Exception('iouType not supported')
        self.iouType = iouType
        # useSegm is deprecated
        self.useSegm = None

class COCORegeval(COCOeval):
    def __init__(self, cocoGt=None, cocoDt=None, iouType='segm'):
        '''
        Initialize CocoEval using coco APIs for gt and dt
        :param cocoGt: coco object with ground truth annotations
        :param cocoDt: coco object with detection results
        :return: None
        '''
        if not iouType:
            print('iouType not specified. use default iouType segm')
        self.cocoGt   = cocoGt              # ground truth COCO API
        self.cocoDt   = cocoDt              # detections COCO API
        self.evalImgs = defaultdict(list)   # per-image per-category evaluation results [KxAxI] elements
        self.eval     = {}                  # accumulated evaluation results
        self._gts = defaultdict(list)       # gt for evaluation
        self._dts = defaultdict(list)       # dt for evaluation
        self.params = RegParams(iouType=iouType) # parameters
        self._paramsEval = {}               # parameters for evaluation
        self.stats = []                     # result summarization
        self.ious = {}                      # ious between all gts and dts
        if not cocoGt is None:
            self.params.imgIds = sorted(cocoGt.getImgIds())
            self.params.catIds = sorted(cocoGt.getCatIds())

    def evaluate(self):
        tic = time.time()
        print('Running per image evaluation...')
        p = self.params
        # add backward compatibility if useSegm is specified in params
        if not p.useSegm is None:
            p.iouType = 'segm' if p.useSegm == 1 else 'bbox'
            print('useSegm (deprecated) is not None. Running {} evaluation'.format(p.iouType))
        print('Evaluate annotation type *{}*'.format(p.iouType))
        p.imgIds = list(np.unique(p.imgIds))
        if p.useCats:
            p.catIds = list(np.unique(p.catIds))
        p.maxDets = sorted(p.maxDets)
        self.params=p

        self._prepare()
        # loop through images, area range, max detection number
        catIds = p.catIds if p.useCats else [-1]

        if p.iouType == 'segm' or p.iouType == 'bbox':
            computeIoU = self.computeIoU
        elif p.iouType == 'keypoints':
            computeIoU = self.computeOks
        elif p.iouType == 'depth':
            computeIoU = self.computeRegErr
        self.ious = {(imgId, catId): computeIoU(imgId, catId) \
                        for imgId in p.imgIds
                        for catId in catIds}

        evaluateImg = self.evaluateImg
        maxDet = p.maxDets[-1]
        self.evalImgs = [evaluateImg(imgId, catId, areaRng, maxDet)
                 for catId in catIds
                 for areaRng in p.areaRng
                 for imgId in p.imgIds
             ]
        # print(self.evalImgs)
        self._paramsEval = copy.deepcopy(self.params)
        toc = time.time()
        print('DONE (t={:0.2f}s).'.format(toc-tic))

    def computeRegErr(self, imgId, catId):
        p = self.params
        # dimention here should be Nxm
        gts = self._gts[imgId, catId]
        dts = self._dts[imgId, catId]
        inds = np.argsort([-d['score'] for d in dts], kind='mergesort')
        dts = [dts[i] for i in inds]
        # dts = [d for d in dts if d['score']>p.scoreThrs]
        if len(dts) > p.maxDets[-1]:
            dts = dts[0:p.maxDets[-1]]
        # if len(gts) == 0 and len(dts) == 0:
        if len(gts) == 0 or len(dts) == 0:
            return []
        ious = np.zeros((len(dts), len(gts)))
        # compute oks between each detection and ground truth object
        total_error = 0
        for j, gt in enumerate(gts):
            # create bounds for ignore regions(double the gt bbox)
            g = np.array(gt['disp_unity'])
            # g = np.array(gt['disp_base'])
            # g = np.array(gt['depth'])
            # g = np.array(gt['height_rw'])

            # k1 = np.count_nonzero(vg > 0)
            bb = gt['bbox']
            x0 = bb[0] - bb[2]; x1 = bb[0] + bb[2] * 2
            y0 = bb[1] - bb[3]; y1 = bb[1] + bb[3] * 2
            # depth_g = g/bb[3]
            depth_g = g
            for i, dt in enumerate(dts):
                d = np.array(dt['depth'])
                dbb = dt['bbox']
                bb_height = dbb[3]
                # depth_d = d/bb_height
                depth_d = d
                # print(depth_d,depth_g)
                dx = depth_d - depth_g
                # print(xd, bb_height, xg)
                # e = np.abs(dx) #/ depth_g
                # print(dx*2200)
                total_error += np.abs(dx*2200)
                ious[i, j] = min(depth_d/depth_g, depth_g/depth_d)# np.exp(-e) #/ e.shape[0]
                # print(depth_g, depth_d)
        total_error /= len(dts)* len(gts)
        # print(total_error)
        return ious

    def summarize(self):
        '''
        Compute and display summary metrics for evaluation results.
        Note this functin can *only* be applied on the default parameter setting
        '''
        def _summarize( ap=1, iouThr=None, areaRng='all', maxDets=100 ):
            p = self.params
            iStr = ' {:<18} {} @[ IoU={:<9} | area={:>6s} | maxDets={:>3d} ] = {:0.3f}'
            titleStr = 'Average Precision' if ap == 1 else 'Average Recall'
            typeStr = '(AP)' if ap==1 else '(AR)'
            iouStr = '{:0.2f}:{:0.2f}'.format(p.iouThrs[0], p.iouThrs[-1]) \
                if iouThr is None else '{:0.2f}'.format(iouThr)

            aind = [i for i, aRng in enumerate(p.areaRngLbl) if aRng == areaRng]
            mind = [i for i, mDet in enumerate(p.maxDets) if mDet == maxDets]
            if ap == 1:
                # dimension of precision: [TxRxKxAxM]
                s = self.eval['precision']
                # IoU
                if iouThr is not None:
                    t = np.where(iouThr == p.iouThrs)[0]
                    s = s[t]
                s = s[:,:,:,aind,mind]
            else:
                # dimension of recall: [TxKxAxM]
                s = self.eval['recall']
                if iouThr is not None:
                    t = np.where(iouThr == p.iouThrs)[0]
                    s = s[t]
                s = s[:,:,aind,mind]
            if len(s[s>-1])==0:
                mean_s = -1
            else:
                mean_s = np.mean(s[s>-1])
            print(iStr.format(titleStr, typeStr, iouStr, areaRng, maxDets, mean_s))
            return mean_s
        def _summarizeDets():
            stats = np.zeros((12,))
            stats[0] = _summarize(1)
            stats[1] = _summarize(1, iouThr=.5, maxDets=self.params.maxDets[2])
            stats[2] = _summarize(1, iouThr=.75, maxDets=self.params.maxDets[2])
            stats[3] = _summarize(1, areaRng='small', maxDets=self.params.maxDets[2])
            stats[4] = _summarize(1, areaRng='medium', maxDets=self.params.maxDets[2])
            stats[5] = _summarize(1, areaRng='large', maxDets=self.params.maxDets[2])
            stats[6] = _summarize(0, maxDets=self.params.maxDets[0])
            stats[7] = _summarize(0, maxDets=self.params.maxDets[1])
            stats[8] = _summarize(0, maxDets=self.params.maxDets[2])
            stats[9] = _summarize(0, areaRng='small', maxDets=self.params.maxDets[2])
            stats[10] = _summarize(0, areaRng='medium', maxDets=self.params.maxDets[2])
            stats[11] = _summarize(0, areaRng='large', maxDets=self.params.maxDets[2])
            return stats
        def _summarizeKps():
            stats = np.zeros((10,))
            stats[0] = _summarize(1, maxDets=20)
            stats[1] = _summarize(1, maxDets=20, iouThr=.5)
            stats[2] = _summarize(1, maxDets=20, iouThr=.75)
            stats[3] = _summarize(1, maxDets=20, areaRng='medium')
            stats[4] = _summarize(1, maxDets=20, areaRng='large')
            stats[5] = _summarize(0, maxDets=20)
            stats[6] = _summarize(0, maxDets=20, iouThr=.5)
            stats[7] = _summarize(0, maxDets=20, iouThr=.75)
            stats[8] = _summarize(0, maxDets=20, areaRng='medium')
            stats[9] = _summarize(0, maxDets=20, areaRng='large')
            return stats
        def _summarizeRegs():
            stats = np.zeros((10,))
            stats[0] = _summarize(1, maxDets=20)
            stats[1] = _summarize(1, maxDets=20, iouThr=.65)
            stats[2] = _summarize(1, maxDets=20, iouThr=.8)
            stats[3] = _summarize(1, maxDets=20, areaRng='medium')
            stats[4] = _summarize(1, maxDets=20, areaRng='large')
            stats[5] = _summarize(0, maxDets=20)
            stats[6] = _summarize(0, maxDets=20, iouThr=.65)
            stats[7] = _summarize(0, maxDets=20, iouThr=.8)
            stats[8] = _summarize(0, maxDets=20, areaRng='medium')
            stats[9] = _summarize(0, maxDets=20, areaRng='large')
            return stats
        if not self.eval:
            raise Exception('Please run accumulate() first')
        iouType = self.params.iouType
        if iouType == 'segm' or iouType == 'bbox':
            summarize = _summarizeDets
        elif iouType == 'keypoints':
            summarize = _summarizeKps
        elif iouType == 'depth':
            summarize = _summarizeRegs
        self.stats = summarize()


text:
import numpy as np
import scipy.signal as signal
import matplotlib.pyplot as plt
import recommender_config
# import warnings
#warnings.simplefilter(action='ignore', category=FutureWarning) #some modules such as ForecastingGridSearchCV when imported raise an annoying future warning

### Sktime native forecasters
from sktime.forecasting.model_selection import ForecastingGridSearchCV
from sktime.forecasting.model_selection import SlidingWindowSplitter
from sktime.forecasting.compose import make_reduction

from sktime.forecasting.naive import NaiveForecaster
from sktime.forecasting.trend import PolynomialTrendForecaster
from sktime.forecasting.arima import AutoARIMA
from sktime.forecasting.theta import ThetaForecaster

# pandas
import pandas as pd

### sklearn imports
from sklearn.neighbors import KNeighborsRegressor
from sklearn.svm import SVR
from sklearn.tree import DecisionTreeRegressor

### Statistical tests
import statsmodels.api as sm
# from statsmodels.tools.sm_exceptions import ConvergenceWarning
import pymannkendall as mk
import ruptures as rpt

### Metrics
def mae(y, y_hat):
    return np.mean(np.abs(y - y_hat))

def rmse(y_pred,y_test):
    return np.sqrt(np.mean(np.power(y_pred - y_test,2)))

def smape(y_pred,y_test):
    nominator = np.abs(y_test - y_pred)
    denominator = np.abs(y_test) + np.abs(y_pred)
    return np.mean(2.0 * nominator / denominator) #the 2 in the nominator is because of symmetry

def mean_overprov(y_prov, y_test, margin=1.0, granularity=15):
    N = len(y_test)
    difference_over  = np.array(y_prov)  - np.array(y_test) * margin
    over_provision = difference_over > 0
    over_acc = sum(difference_over[over_provision]) / (granularity*N)
    return over_acc

def mean_underprov(y_prov, y_test, margin=1.0, granularity=15):
    N = len(y_test)
    difference_under = np.array(y_test) - np.array(y_prov) * margin
    under_provision = difference_under > 0
    under_acc = sum(difference_under[under_provision])  / (granularity*N)
    return under_acc

def perc_overprov(y_prov, y_test, margin=1.0, granularity=15):
    N = len(y_test)
    difference_over  = np.array(y_prov)  - np.array(y_test) * margin
    over_provision = difference_over > 0
    over_perc = (sum(over_provision == True)  / N)*100
    return over_perc

def perc_underprov(y_prov, y_test, margin=1.0, granularity=15):
    N = len(y_test)
    difference_under = np.array(y_test) - np.array(y_prov) * margin
    under_provision = difference_under > 0
    under_perc = (sum(under_provision == True) /N)*100
    return under_perc

### Functions to process time series
def convolution_filter(y, period):
    # Prepare Filter
    if period % 2 == 0:
        filt = np.array([.5] + [1] * (period - 1) + [.5]) / period
    else:
        filt = np.repeat(1. / period, period)

    # Signal Convolution
    conv_signal = signal.convolve(y, filt, mode='valid')

    # Padding (2-Sided Convolution)
    trim_head = int(np.ceil(len(filt) / 2.) - 1) or None
    trim_tail = int(np.ceil(len(filt) / 2.) - len(filt) % 2) or None

    if trim_head:
        conv_signal = np.r_[conv_signal, [np.nan] * trim_tail]
    if trim_tail:
        conv_signal = np.r_[[np.nan] * trim_head, conv_signal]

    return conv_signal

def compute_ses(y, alpha):
    nobs = len(y)  # X from the slides

    # Forecast Array
    fh = np.full(nobs + 1, np.nan)  # Initialize the Forecast array to NaNs # S from the slides
    fh[0] = y[0]  # Initialization of first value (instead of NaN)
    fh[1] = y[0]  # Initialization of first forecast

    # Simple Exponential Smoothing
    for t in range(2, nobs + 1):
        fh[t] = alpha * y[t - 1] + (1 - alpha) * fh[t - 1]  # s[t] = alpha * y....

    return (fh[:nobs], fh[nobs])

def forecast_ses(fh_next, start, end):
    ## Forecast Array
    fh_forecast = np.full(end - start, np.nan)
    fh_forecast[:] = fh_next

    return fh_forecast

def seasonal_decompose(y, period):
    nobs = len(y)

    # At least two observable periods in the trace
    if nobs < 2 * period:
        raise ValueError('lengh of signal must be larger than (2 * period)')

    # Convolution to retrieve step-by-step trend
    trend = convolution_filter(y, period)

    # Multiplicative de-trending to Retrieve average Season (period pattern)
    detrended = y / trend
    period_averages = np.array([np.nanmean(detrended[i::period], axis=0) for i in range(period)])
    period_averages /= np.mean(period_averages, axis=0)

    return period_averages  # "season" for deseasonalize

def deseasonalize(y, season):
    nobs = len(y)
    period = len(season)

    seasonal = np.tile(season, (nobs // period) + 1).T[:nobs]
    return y / seasonal

def reseasonalize(y, season, start):
    nobs = len(y)
    period = len(season)

    shift = period - (start % period)
    season = np.concatenate((season[-shift:], season[:-shift]))

    seasonal = np.tile(season, (nobs // period) + 1).T[:nobs]
    return y * seasonal

def compute_trend(y):
    lm = np.polyfit(np.arange(len(y)), y, 1)

    slope = lm[0]
    intercept = lm[1]
    drift = (slope * np.arange(0, len(y))) + intercept

    return (slope, intercept, drift)

def retrend(y, start, end, slope, intercept, type_t='mult'):
    if type_t == 'mult':
        #version 1
        drift = (slope * np.arange(start, end)) + intercept
        pred = y * (drift / np.mean(y))
    elif type_t =='add':
        #version 2 --> linear trend
        drift = slope * np.arange(start, end)
        pred = y + drift
    elif type_t=='no':
        pred = y
    return pred

def scan(trace, max_period = 100):

    errors = []
    test_size = len(trace) //8 # size of the test set for testing cycles
    if max_period is None: max_period = len(trace) //3 # max period to be examined

    y_train, y_test = trace[:-test_size],trace[-test_size:]

    true_max_period = min(max_period, int(len(y_train)/2))
    period_values = range(1, true_max_period)
    model = ThetaModel()

    for sp_val in period_values:
        model.fit(y_train, sp=sp_val)
        y_pred = model.forecast(test_size)[1]
        current_error = smape(y_pred, y_test)
        errors.append(current_error)

    period = period_values[np.argmin(errors)]
    return period, errors

## Functions to plot time series
def plot_series(series_list, labels, title='', legend=True):
    if len(series_list) != len(labels):
        raise ValueError('Number of series and labels must be the same')
    data = {}
    for i, series in enumerate(series_list):
        data[labels[i]] = series
    df = pd.DataFrame(data)
    df.plot(figsize=(12, 6))
    plt.title(title)
    if legend:
        plt.legend()
    plt.savefig("./tmp/" + title + ".png")
    plt.show()

### Own implementation of Theta Model
class ThetaModel:
    def __init__(self, alpha = 0.2):
        self.alpha = alpha

    def fit (self, y, sp):

        ## 1. Deseasonalize & Detrend
        season = seasonal_decompose(y, sp)            	                       ### THIS IS THE SEASON
        deseason = deseasonalize(y, season)                                      ### THIS IS THE DESEASONALIZED AND DETRENDED

        ## 2. Obtain Drift (general Trend) for later
        slope, intercept, drift = compute_trend(deseason)                              ### THIS IS THE SLOPE, INTERCEPT AND DRIFT

        ## 3. Obtain Simple Exponential Smoothing (SES)
        fitted, y_next = compute_ses(deseason, self.alpha)                                  ### THIS IS THE MODEL (Fitted, Next)

        ## Save "Model"
        self.season = season
        self.deseason = deseason
        self.slope = slope
        self.intercept = intercept
        self.drift = drift
        self.fitted = fitted
        self.next = y_next
        self.dataset = y
        self.last = len(y)

    def forecast (self, n_forecast):
        ## Get new boundaries
        start = self.last
        end = self.last + n_forecast
        ## 1. Forecast
        y_pred_1 = forecast_ses(self.next, start, end)

        ## 2. Re-Trend [FIRST]
        y_pred_2 = retrend(y_pred_1, start, end, self.slope, self.intercept, type_t='mult')

        ## 3. Re-Seasonalize [SECOND]
        y_pred = reseasonalize(y_pred_2, self.season, start)

        ## Join Full Trace
        full_trace_pred = np.concatenate((self.dataset, y_pred))

        return full_trace_pred, y_pred

### Forecasters with autotunning
def Theta_forecast(y, window, max_period=100, output=False, title=''):
    sp, _ = scan(y, max_period)
    forecaster = ThetaModel()
    forecaster.fit(y, sp)
    _, y_pred = forecaster.forecast(window)
    if output: plot_series([pd.Series(y), pd.Series(y_pred, index=np.arange(len(y), len(y) + window))],
                           labels=["y", "y_pred"], title=title)
    if output: print("Detected period: {}".format(sp))
    return y_pred

def Theta_forecast_sktime(y, window, max_period = 100, output =False, title=''):
    y = pd.Series(y)
    fh = np.arange(1, window+1)
    forecaster = ThetaForecaster()
    sps = np.arange(1,max_period)
    param_grid = {"sp": sps}
    cv = SlidingWindowSplitter(window_length=int(len(y) * recommender_config.WINDOW_SPILLTER), fh = fh)
    gscv = ForecastingGridSearchCV(forecaster, strategy="refit", cv=cv, param_grid=param_grid)
    gscv.fit(y)
    y_pred = gscv.best_forecaster_.predict(fh)
    if output: plot_series([y, y_pred], labels=["y","y_pred"], title=title)
    return y_pred.values

def Naive_forecast(y, window, max_period = 100, output =False, title=''):
    y = pd.Series(y)
    fh = np.arange(1, window+1)
    forecaster = NaiveForecaster()
    sps = np.arange(1,max_period)
    param_grid = {"strategy" : ["last", "drift"], "sp": sps}
    cv = SlidingWindowSplitter(window_length=int(len(y) * recommender_config.WINDOW_SPILLTER), fh = fh)
    gscv = ForecastingGridSearchCV(forecaster, strategy="refit", cv=cv, param_grid=param_grid)
    gscv.fit(y)
    y_pred = gscv.predict(fh)
    if output: plot_series([y, y_pred], labels=["y","y_pred"], title=title)
    return y_pred.values

def lr_forecast(y, window, output =False, title=''):
    y = pd.Series(y)
    fh = np.arange(1, window+1)
    forecaster = PolynomialTrendForecaster()
    param_grid = {"degree" : [1,2]}
    cv = SlidingWindowSplitter(window_length=int(len(y) * recommender_config.WINDOW_SPILLTER), fh = fh)
    gscv = ForecastingGridSearchCV(forecaster, strategy="refit", cv=cv, param_grid=param_grid)
    gscv.fit(y, fh=fh)
    y_pred = gscv.predict(fh)
    if output: plot_series([y, y_pred], labels=["y","y_pred"], title=title)
    return y_pred.values

def autoarima_forecast(y, window, output =False, title=''):
    y = pd.Series(y)
    fh = np.arange(1, window+1)
    forecaster = AutoARIMA()
    forecaster.fit(y, fh=fh)
    y_pred = forecaster.predict(fh)
    if output: plot_series([y, y_pred], labels=["y","y_pred"], title=title)
    return y_pred.values

def KNN_forecast(y, window, output =False, title=''):
    y = pd.Series(y)
    fh = np.arange(1, window+1)
    regressor = KNeighborsRegressor()
    forecaster = make_reduction(regressor, strategy="recursive")
    param_grid = {"window_length": [7, 12, 15], "estimator__n_neighbors": np.arange(1, 5)}
    cv = SlidingWindowSplitter(window_length=int(len(y) * recommender_config.WINDOW_SPILLTER), fh = fh)
    gscv = ForecastingGridSearchCV(forecaster, strategy="refit", cv=cv, param_grid=param_grid)
    gscv.fit(y)
    y_pred = gscv.predict(fh)
    if output: plot_series([y, y_pred], labels=["y", "y_pred"], title=title)
    return y_pred.values

def SVM_forecast(y, window, output =False, title=''):
    y = pd.Series(y)
    fh = np.arange(1, window+1)
    regressor = SVR()
    forecaster = make_reduction(regressor, strategy="recursive")
    param_grid = {"window_length": [7, 12, 15],
                  "estimator__kernel": ['linear', 'poly', 'rbf', 'sigmoid'],
                  "estimator__gamma": [1e-3, 1e-4],
                  "estimator__C": [1, 10, 100, 1000]}
    cv = SlidingWindowSplitter(window_length=int(len(y) * recommender_config.WINDOW_SPILLTER), fh = fh)
    gscv = ForecastingGridSearchCV(forecaster, strategy="refit", cv=cv, param_grid=param_grid)
    gscv.fit(y)
    y_pred = gscv.predict(fh)
    if output: plot_series([y, y_pred], labels=["y", "y_pred"], title=title)
    return y_pred.values

def DT_forecast(y, window, output =False, title=''):
    y = pd.Series(y)
    fh = np.arange(1, window+1)
    regressor = DecisionTreeRegressor()
    forecaster = make_reduction(regressor, strategy="recursive")
    param_grid = {"window_length": [7, 12, 15]}
    cv = SlidingWindowSplitter(window_length=int(len(y) * recommender_config.WINDOW_SPILLTER), fh = fh)
    gscv = ForecastingGridSearchCV(forecaster, strategy="refit", cv=cv, param_grid=param_grid)
    gscv.fit(y)
    y_pred = gscv.predict(fh)
    if output: plot_series([y, y_pred], labels=["y", "y_pred"], title=title)
    return y_pred.values

def VPA_forecast(y, window, output =False, title=''):
    pre_step_idxs = range(len(y)-window, len(y))
    previous_usage = y[pre_step_idxs]

    y = pd.Series(y)
    y_pred = previous_usage
    y_pred_index = range(len(y), len(y)+window)
    y_pred = pd.Series(y_pred, index=y_pred_index)

    if output: plot_series([y, y_pred], labels=["y", "y_pred"], title=title)

    return y_pred.values

### Statistical tests
def detect_theta_periodicity(trace, theta_threshold):
    period_est, error_est = scan(trace)
    return np.var(error_est) >= theta_threshold

def detect_adf_stationarity(trace, stat_threshold):
    adf = sm.tsa.stattools.adfuller(trace)
    stationary = adf[1] < stat_threshold # if True reject null hypothesis --> stationary
    return stationary

def detect_trend(trace):
    return mk.original_test(trace)[0] != 'no trend'

def detect_change_point(trace, max_change_points):
    algo1 = rpt.Pelt(model="rbf").fit(trace)
    change_location = algo1.predict(pen=max_change_points)
    return len(change_location)>1

def perform_tests(y, stat_threshold= 0.01, theta_threshold = 0.001, max_change_points = 5):
    theta_test = detect_theta_periodicity(y, theta_threshold)
    adf_test = detect_adf_stationarity(y, stat_threshold)
    mk_test = detect_trend(y)
    cp_test = detect_change_point(y, max_change_points)
    return {"theta":theta_test, "adf":adf_test, "mk": mk_test, "pelt":cp_test}

ext: py

max_stars_repo_name: openshift/predictive-vpa-recommenders

max_issues_repo_name: openshift/predictive-vpa-recommenders

max_forks_repo_name: openshift/predictive-vpa-recommenders

reason: import numpy,import scipy,import statsmodels,from statsmodels


text:
import numpy as np
import pandas as pd
from typing import Dict

from cascade_at.core.log import get_loggers
from cascade_at.inputs.utilities.gbd_ids import make_age_intervals, make_time_intervals
from cascade_at.inputs import InputsError

LOG = get_loggers(__name__)

class CovariateInterpolationError(InputsError):
    """Raised when there is an issue with covariate interpolation."""
    pass

def values(interval):
    return interval.begin, interval.end, interval.data

def interval_weighting(intervals, lower, upper):
    """
    Compute a weighting function by finding the proportion
    within the dataframe df's lower and upper bounds.

    Note: intervals is of the form ((lower, upper, id), ...)
    """

    if len(intervals) == 1:
        return np.asarray([1])
    wts = np.ones(len(intervals))
    lower_limit, upper_limit = intervals[0], intervals[-1]
    wts[0] = (lower_limit[1] - lower) / np.diff(lower_limit[:2])
    wts[-1] = (upper - upper_limit[0]) / np.diff(upper_limit[:2])
    return wts

class CovariateInterpolator:
    def __init__(self,
                 covariate: pd.DataFrame,
                 population: pd.DataFrame):
        """
        Interpolates a covariate by population weighting.

        Parameters
        ----------
        covariate
            Data frame with covariate information
        population
            Data frame with population information
        """
        # Covariates must be sorted by both age_group_id and age_lower because age_lower is not unique to age_group_id
        indices = ['location_id', 'sex_id', 'year_id', 'age_group_id']
        sort_order = indices + ['age_lower']

        self.covariate = covariate.sort_values(by=sort_order)
        self.population = population.sort_values(by=sort_order)

        self.location_ids = self.covariate.location_id.unique()
        self.year_min = self.covariate.year_id.min()
        self.year_max = self.covariate.year_id.max() + 1

        self.age_intervals = make_age_intervals(df=self.covariate)
        self.time_intervals = make_time_intervals(df=self.covariate)

        self.dict_cov = dict(zip(
            map(tuple, self.covariate[indices].values.tolist()), self.covariate['mean_value'].values
        ))
        self.dict_pop = dict(zip(
            map(tuple, self.population[indices].values.tolist()), self.population['population'].values
        ))

    @staticmethod
    def _restrict_time(time, time_min, time_max):
        return max(min(time, time_max), time_min)

    def _weighting(self, age_lower, age_upper, time_lower, time_upper):
        if age_lower == age_upper:
            age_groups = sorted(map(values, self.age_intervals[age_lower]))
        else:
            age_groups = sorted(map(values, self.age_intervals[age_lower: age_upper]))

        if not age_groups:
            raise CovariateInterpolationError(
                f"There is no covariate age group for age lower {age_lower} and age upper {age_upper}."
            )
        age_group_ids = [a[-1] for a in age_groups]
        age_wts = interval_weighting(tuple(age_groups), age_lower, age_upper)

        # We are *not* linearly interpolating past the covariate time
        # ranges -- instead we carry over the values from the left
        # or rightmost time point.
        time_lower = self._restrict_time(time_lower, time_min=self.year_min, time_max=self.year_max)
        time_upper = self._restrict_time(time_upper, time_min=self.year_min, time_max=self.year_max)

        # This is to ensure that the time_lower can actually subset
        # an interval. For example, if time_lower = 2012 and time_upper = 2012,
        # but the max interval goes from 2011-2012, it will not be able
        # to select that interval until we decrease time_lower.
        # We don't have to do this on the leftmost end, however,
        # because that's already taken care of by _restrict_time,
        # and the leftmost point of the interval *is* the key for IntervalTrees.
        if not self.time_intervals.overlaps(time_lower):
            time_lower -= 1

        if time_lower == time_upper:
            time_groups = sorted(map(values, self.time_intervals[time_lower]))
        else:
            time_groups = sorted(map(values, self.time_intervals[time_lower: time_upper]))

        if not time_groups:
            raise CovariateInterpolationError(
                f"There is no covariate time group for time lower {time_lower} and time upper {time_upper}."
            )
        year_ids = [t[-1] for t in time_groups]
        time_wts = interval_weighting(tuple(time_groups), time_lower, time_upper)

        # The order of outer must agree with the covariate and population sort order
        wt = np.outer(time_wts, age_wts)
        return age_group_ids, year_ids, wt

    def interpolate(self, loc_id, sex_id, age_lower, age_upper, time_lower, time_upper):
        """
        Main interpolation function.
        """
        if loc_id not in self.location_ids:
            LOG.warning(f"Covariate is missing for location_id {loc_id},"
                        f"sex_id {sex_id} -- setting the value to None.")
            cov_value = None
        else:
            age_group_ids, year_ids, epoch_weights = self._weighting(
                age_lower=age_lower, age_upper=age_upper,
                time_lower=time_lower, time_upper=time_upper
            )
            shape = epoch_weights.shape
            # This loop indexing order matters, and must agree with the covariate and population sort order
            cov_value = np.asarray([self.dict_cov[(loc_id, sex_id, year_id, age_id)]
                                    for year_id in year_ids for age_id in age_group_ids]).reshape(shape)
            # This loop indexing order matters, and must agree with the covariate and population sort order
            pop_value = np.asarray([self.dict_pop[(loc_id, sex_id, year_id, age_id)]
                                    for year_id in year_ids for age_id in age_group_ids]).reshape(shape)

            weight = epoch_weights * pop_value
            cov_value = np.average(cov_value, weights=weight)
        return cov_value

def get_interpolated_covariate_values(data_df: pd.DataFrame,
                                      covariate_dict: Dict[str, pd.DataFrame],
                                      population_df: pd.DataFrame) -> pd.DataFrame:
    """
    Gets the unique age-time combinations from the data_df, and creates
    interpolated covariate values for each of these combinations by population-weighting
    the standard GBD age-years that span the non-standard combinations.

    Parameters
    ----------
    data_df
        A data frame with data observations in it
    covariate_dict
        A dictionary of covariate data frames with covariate names as keys
    population_df
        A data frame with population in it
    """
    data = data_df.copy()
    pop = population_df.copy()

    data_groups = data.groupby([
        'location_id', 'sex_id', 'age_lower', 'age_upper', 'time_lower', 'time_upper'
    ], as_index=False)

    cov_objects = {cov_name: CovariateInterpolator(covariate=raw_cov, population=pop)
                   for cov_name, raw_cov in covariate_dict.items()}
    num_groups = len(data_groups)
    for cov_id, cov_obj in cov_objects.items():
        LOG.info(f"Interpolating covariate {cov_id}.")
        for i, (k, v) in enumerate(data_groups):
            if i % 1000 == 0:
                LOG.info(f"Processed {i} of {num_groups} data groups.")
            [loc_id, sex_id, age_lower, age_upper, time_lower, time_upper] = k
            cov_value = cov_obj.interpolate(
                loc_id=loc_id, sex_id=sex_id,
                age_lower=age_lower, age_upper=age_upper,
                time_lower=time_lower, time_upper=time_upper
            )
            data.loc[v.index, cov_id] = cov_value
    return data

max_stars_repo_name: ihmeuw/cascade-at

max_issues_repo_name: ihmeuw/cascade

max_forks_repo_name: ihmeuw/cascade

reason: import numpy


text:
import os

import nibabel
import numpy as np

from nipype.interfaces.base import (
    BaseInterface, BaseInterfaceInputSpec, traits, File, TraitedSpec)
from nipype.interfaces.base import OutputMultiPath
from nipype.utils.filemanip import split_filename

from _utils import check_images

def compute_perfusion(in_file, ctl_scans=None,
                      tag_scans=None):
    """Compute the mean perfusion image from a functional ASL sequence. The
    ASL sequence is assumed aligned.

    Parameters
    ----------
    in_file : str
        The filename of the input functional ASL 4D image.

    ctl_scans : list of int or None, optional
        Indexes of control volumes.

    tag_scans : list of int or None, optional
        Indexes of tagged volumes, same length as ctl_scans.

    Returns
    -------
    out_file : str
        The filename of the output perfusion image.
    """
    image = nibabel.load(in_file)
    data = image.get_data()
    n_scans = data.shape[-1]
    if ctl_scans is None:
        ctl_scans = range(0, n_scans, 2)

    if tag_scans is None:
        tag_scans = range(1, n_scans, 2)

    if len(ctl_scans) != len(tag_scans):
        raise ValueError('{0} control scans, {1} tagged scans.'.format(
                         ctl_scans, tag_scans))

    ctl_data = data[..., ctl_scans]
    tag_data = data[..., tag_scans]
    perfusion_data = ctl_data - tag_data
    perfusion_data = np.mean(perfusion_data, axis=-1)
    image = nibabel.Nifti1Image(perfusion_data, image.get_affine(),
                                image.get_header())
    out_file = os.path.join(os.path.dirname(in_file),
                            'perfusion_' + os.path.basename(in_file))
    nibabel.save(image, out_file)
    return out_file

class QuantifyCBFInputSpec(BaseInterfaceInputSpec):
    perfusion_file = File(
        exists=True,
        mandatory=True,
        copyfile=True,
        desc='perfusion image filename')
    m0_file = File(
        exists=True,
        mandatory=True,
        copyfile=True,
        desc='M0 image filename')
    tr = traits.Float(
        mandatory=True,
        usedefault=True,
        desc='TR, in ms')
    t1_gm = traits.Float(
        1331.,
        usedefault=True,
        desc='Gray matter T1 value, in ms')

class QuantifyCBFOutputSpec(TraitedSpec):
    cbf_file = OutputMultiPath(
        traits.Either(traits.List(File(exists=True)), File(exists=True)),
        desc='The CBF filename')

class QuantifyCBF(BaseInterface):
    input_spec = QuantifyCBFInputSpec
    output_spec = QuantifyCBFOutputSpec

    def _run_interface(self, runtime):
        """Basic PASL quantification for QUIPPS II.
        CBF (ml/100g/min) = 6000 * brain_blood_coef * perfusion *
                            exp(TI / T1b) * (1 - exp(-TR / T1_tissue)) /
                            (2 * label_efficiency * M0 * TI1)
        """
        # Load images
        perfusion_image = nibabel.load(self.inputs.perfusion_file)
        perfusion_data = perfusion_image.get_data()
        perfusion_affine = perfusion_image.get_affine()
        m0_image = nibabel.load(self.inputs.m0_file)
        m0_data = m0_image.get_data()
        m0_affine = m0_image.get_affine()
        t1_data = self.inputs.t1_gm  # as recommended in Alsop 2014

        # Check shapes and affines  # TODO: replace with check_images
        if np.any(m0_affine != perfusion_affine):
            print np.max(np.abs(perfusion_affine - m0_affine))
            raise ValueError('affine for {0}: {1}, for {2}:'
                             ' {3}'.format(self.inputs.m0_file,
                                           m0_affine,
                                           self.inputs.perfusion_file,
                                           perfusion_affine))

        if m0_data.shape != perfusion_data.shape:
            raise ValueError('Perfusion and M0 have different shapes.'
                             '{0} of shape {1}, {2} of shape {3}. Resampling'
                             ' is needed.'.format(self.inputs.m0_file,
                                                  m0_data.shape,
                                                  self.inputs.perfusion_file,
                                                  perfusion_data.shape))

        # Compute the CBF
        m0_data = m0_data.astype(float)
        m0_data = m0_data / (1. - np.exp(- self.inputs.tr / t1_data))
        non_zero_m0 = np.abs(m0_data) > 1e-4
        brain_blood_coef = 0.9  # brain blood partition coefficient, in mL/g
        unit_scaling = 6000.  # from ml/g/s to ml/100g/min
        cbf_data = perfusion_data * unit_scaling
        cbf_data[non_zero_m0] /= m0_data[non_zero_m0]
        cbf_data[np.logical_not(non_zero_m0)] = np.nan
        cbf_data *= brain_blood_coef

        # Save the CBF image
        image = nibabel.Nifti1Image(cbf_data, perfusion_affine,
                                    perfusion_image.get_header())
        fname = self.inputs.perfusion_file
        _, base, _ = split_filename(fname)
        cbf_file = os.path.abspath('basic_cbf_' + base + '.nii')
        nibabel.save(image, cbf_file)
        return runtime

    def _list_outputs(self):
        outputs = self._outputs().get()
        fname = self.inputs.perfusion_file
        _, base, _ = split_filename(fname)
        outputs["cbf_file"] = os.path.abspath('basic_cbf_' + base + '.nii')
        return outputs


text:
# coding=utf-8
# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Hotel reviews from the TripAdvisor Dataset as preprocessed by Bao et al. (2018)"""

from __future__ import absolute_import, division, print_function

import os
import csv
import numpy as np

import pdb
import datasets

_CITATION = """\
@misc{bao2018deriving,
      title={Deriving Machine Attention from Human Rationales},
      author={Yujia Bao and Shiyu Chang and Mo Yu and Regina Barzilay},
      year={2018},
      eprint={1808.09367},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
}
"""

_DESCRIPTION = """\
This dataset consists of hotel reviews from TripAdvisor
"""

_URL = "http://web.tecnico.ulisboa.pt/~ist178550/hotel_location_proc.zip"

class HotelLocationDatasetConfig(datasets.BuilderConfig):
    """BuilderConfig for Hotel"""

    def __init__(self, **kwargs):
        """
        Args:
            **kwargs: keyword arguments forwarded to super.
        """
        super().__init__(**kwargs)

class HotelLocationDataset(datasets.GeneratorBasedBuilder):
    """Beer reviews from beeradvocate. Version preprocessed by McAuley et al. (2012)."""

    VERSION = datasets.Version("1.0.0")

    BUILDER_CONFIG_CLASS = HotelLocationDatasetConfig
    BUILDER_CONFIGS = [
        HotelLocationDatasetConfig(
            name="hotel_location_dataset_",
            description="Hotel reviews for Location aspect.",
        )
    ]

    def _info(self):
        return datasets.DatasetInfo(
            # This is the description that will appear on the datasets page.
            description=_DESCRIPTION,
            # This defines the different columns of the dataset and their types
            features=datasets.Features(
                {
                    "tokens": datasets.Value("string"),
                    # we have five scores (one for each aspect) normalized between 0 and 1
                    "scores": datasets.features.Sequence(
                        datasets.Value("float"), length=1
                    ),
                    "annotations": datasets.features.Sequence(
                        datasets.features.Sequence(
                            datasets.features.Sequence(datasets.Value("int32"))
                        )
                    ),
                }
            ),
            # If there's a common (input, target) tuple from the features,
            # specify them here. They'll be used if as_supervised=True in
            # builder.as_dataset.
            supervised_keys=None,
            # Homepage of the dataset for documentation
            homepage="https://www.cs.virginia.edu/~hw5x/dataset.html",
            citation=_CITATION,
        )

    def _split_generators(self, dl_manager):
        """Returns SplitGenerators."""

        # dl_manager is a datasets.download.DownloadManager that can be used to
        # download and extract URLs
        dl_dir = dl_manager.download_and_extract(_URL)
        data_dir = dl_dir
        filepaths = {
            "train": os.path.join(data_dir, "hotel_location/hotel_Location_train.csv"),
            "dev": os.path.join(data_dir, "hotel_location/hotel_Location_dev.csv"),
            "test": os.path.join(data_dir, "hotel_location/hotel_Location_test.csv"),
        }

        return [
            datasets.SplitGenerator(
                name=datasets.Split.TRAIN,
                gen_kwargs={
                    "filepath": filepaths["train"],
                    "split": "train",
                },
            ),
            datasets.SplitGenerator(
                name=datasets.Split.VALIDATION,
                gen_kwargs={
                    "filepath": filepaths["dev"],
                    "split": "dev",
                },
            ),
            datasets.SplitGenerator(
                name=datasets.Split.TEST,
                gen_kwargs={"filepath": filepaths["test"], "split": "test"},
            ),
        ]

    def _generate_examples(self, filepath, split):
        """Yields examples."""
        with open(filepath, "r", encoding="utf8") as f:
            if split == "train":
                f = csv.DictReader(f, delimiter=";")
            else:
                f = csv.DictReader(f, delimiter="\t")
            for id_, row in enumerate(f):
                annotations = []
                if split == "test":
                    tokens = row["text"]
                    scores = [float(row["label"])]
                    raw_annotations = np.array(
                        [int(s) for s in row["rationale"].split()]
                    )
                    a1 = raw_annotations > 0
                    a1_rshifted = np.roll(a1, 1)
                    starts = a1 & ~a1_rshifted
                    ends = ~a1 & a1_rshifted
                    if len(np.nonzero(starts)[0]) > 0:
                        for i in range(len(np.nonzero(starts)[0])):
                            annotations.append(
                                [
                                    np.nonzero(starts)[0][i].item(),
                                    np.nonzero(ends)[0][i].item(),
                                ]
                            )
                    else:
                        annotations.append([])

                    yield id_, {
                        "tokens": tokens,
                        "scores": scores,
                        "annotations": [annotations],
                    }

                else:
                    tokens = row["text"]
                    scores = [float(row["label"])]
                    yield id_, {
                        "tokens": tokens,
                        "scores": scores,
                        "annotations": [[[0]]],  # dummy value
                    }

max_stars_repo_name: deep-spin/spectra-rationalization

max_issues_repo_name: deep-spin/spectra-rationalization

max_forks_repo_name: deep-spin/spectra-rationalization


text:
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from auto_scan_test import PassAutoScanTest, SkipReasons
from program_config import TensorConfig, ProgramConfig
import numpy as np
import paddle.inference as paddle_infer
from functools import partial
from typing import Optional, List, Callable, Dict, Any, Set
import unittest

import hypothesis
from hypothesis import given, settings, seed, example, assume
import hypothesis.strategies as st

class TestScaleMatmulMkldnnFusePass(PassAutoScanTest):
    def is_program_valid(self, program_config: ProgramConfig) -> bool:
        return True

    def sample_program_config(self, draw):
        scale = draw(st.floats(min_value=0.01, max_value=2))
        bias = 0.0
        bias_after_scale = draw(st.booleans())
        transpose_X = draw(st.booleans())
        transpose_Y = draw(st.booleans())
        alpha = draw(st.floats(min_value=0.01, max_value=2))
        batch_size = draw(st.integers(min_value=1, max_value=4))
        channel = draw(st.integers(min_value=1, max_value=64))
        input_dim = draw(st.sampled_from([1, 32, 64]))

        def generate_input(attrs, type):
            if attrs[1]['transpose_X'] and attrs[1]['transpose_Y']:
                shape_x = [
                    attrs[2]['batch_size'], attrs[2]['channel'],
                    attrs[2]['input_dim'], 32
                ]
                shape_y = [
                    attrs[2]['batch_size'], attrs[2]['channel'], 64,
                    attrs[2]['input_dim']
                ]
            elif attrs[1]['transpose_X']:
                shape_x = [
                    attrs[2]['batch_size'], attrs[2]['channel'],
                    attrs[2]['input_dim'], 32
                ]
                shape_y = [
                    attrs[2]['batch_size'], attrs[2]['channel'],
                    attrs[2]['input_dim'], 64
                ]
            elif attrs[1]['transpose_Y']:
                shape_x = [
                    attrs[2]['batch_size'], attrs[2]['channel'], 32,
                    attrs[2]['input_dim']
                ]
                shape_y = [
                    attrs[2]['batch_size'], attrs[2]['channel'], 8,
                    attrs[2]['input_dim']
                ]
            else:
                shape_x = [
                    attrs[2]['batch_size'], attrs[2]['channel'], 32,
                    attrs[2]['input_dim']
                ]
                shape_y = [
                    attrs[2]['batch_size'], attrs[2]['channel'],
                    attrs[2]['input_dim'], 16
                ]

            if type == "x":
                return np.random.random(shape_x).astype(np.float32)
            else:
                return np.random.random(shape_y).astype(np.float32)

        attrs = [{
            "scale": scale,
            "bias": bias,
            "bias_after_scale": bias_after_scale
        }, {
            "transpose_X": transpose_X,
            "transpose_Y": transpose_Y,
            "alpha": alpha
        }, {
            'batch_size': batch_size,
            'channel': channel,
            'input_dim': input_dim
        }]

        ops_config = [{
            "op_type": "scale",
            "op_inputs": {
                "X": ["input_data1"]
            },
            "op_outputs": {
                "Out": ["scale_output"]
            },
            "op_attrs": {
                "scale": attrs[0]['scale'],
                "bias": attrs[0]['bias'],
                "bias_after_scale": attrs[0]['bias_after_scale']
            },
        }, {
            "op_type": "matmul",
            "op_inputs": {
                "X": ["scale_output"],
                "Y": ["input_data2"]
            },
            "op_outputs": {
                "Out": ["matmul_output"]
            },
            "op_attrs": {
                'transpose_X': attrs[1]['transpose_X'],
                'transpose_Y': attrs[1]['transpose_Y'],
                'alpha': attrs[1]['alpha'],
                "fused_reshape_X": [],
                "fused_reshape_Y": [],
                "fused_transpose_X": [],
                "fused_transpose_Y": [],
                "fused_reshape_Out": [],
                "fused_transpose_Out": []
            }
        }]

        ops = self.generate_op_config(ops_config)

        program_config = ProgramConfig(
            ops=ops,
            weights={},
            inputs={
                "input_data1":
                TensorConfig(data_gen=partial(generate_input, attrs, "x")),
                "input_data2":
                TensorConfig(data_gen=partial(generate_input, attrs, "y"))
            },
            outputs=["matmul_output"])

        return program_config

    def sample_predictor_configs(self, program_config):
        config = self.create_inference_config(use_mkldnn=True)
        yield config, ['matmul'], (1e-5, 1e-5)

    def test(self):
        self.run_and_statis(quant=False, passes=["scale_matmul_fuse_pass"])

if __name__ == "__main__":
    unittest.main()

max_stars_repo_name: zmxdream/Paddle

max_issues_repo_name: zmxdream/Paddle

max_forks_repo_name: zmxdream/Paddle


text:
import numpy as np
import tensorflow as tf
import tensorflow.keras as tfk
import tensorflow.keras.layers as tfkl

from drosoph_vae.layers.padded_conv1d_transposed import PaddedConv1dTransposed
from drosoph_vae.models.drosoph_vae import DrosophVAE
from drosoph_vae.models.utils import make_inference_net

class DrosophVAESkipConv(DrosophVAE):
    """
    About the Deconvolution: https://datascience.stackexchange.com/questions/6107/what-are-deconvolutional-layers

    Based on https://arxiv.org/pdf/1807.04863.pdf to avoid latent space collapse

    """
    def __init__(self, latent_dim, input_shape, batch_size,
                 n_conv_layers=None, n_start_filters=None, dropout_rate_temporal=0.2, loss_weight_reconstruction=1.0, loss_weight_kl=1.0):
        """
        Args:
        -----

        latent_dim              int, dimension of latent space
        input_shape             tuple, total input shape is: [batch_size, *input_shape]
        batch_size              int
        n_layers                int, number of dense layers.
                                output shape of the dense layers is linearly scaled.
        dropout_rate_temporal   float, in [0, 1). dropout rate for temporal blocks (conv layers).
        filters_conv_layer      list[int]. filter sizes for conv layers
        """
        # just a dummy init, we are only interested in defining the two nets.
        super(DrosophVAESkipConv, self).__init__(
            latent_dim=latent_dim,
            input_shape=input_shape,
            batch_size=batch_size,
            loss_weight_reconstruction=loss_weight_reconstruction,
            loss_weight_kl=loss_weight_kl,
            use_wavenet_temporal_layer=False,
            n_layers=1,
        )

        if n_start_filters is None:
            n_start_filters = input_shape[-1]

        if n_conv_layers is None:
            n_conv_layers = np.int(np.ceil((n_start_filters - latent_dim) / 2 + 1))

        self.latent_dim = latent_dim
        self._layer_sizes_inference  = np.linspace(n_start_filters, 2 * latent_dim, num=input_shape[-2] -1, dtype=np.int)
        # pseudo reverse as the inference network goes down to double the latent space, ask Semigh about this
        # the 2 * n_layers is to keep compression speed roughly the same

        # since the layers grow only one time-step per layer:
        self._layer_sizes_generative = np.linspace(latent_dim, n_start_filters, num=input_shape[-2] - 1, dtype=np.int)

        # TODO add MaxPooling
        self.inference_net = make_inference_net(tfk.Sequential([_convolutional_layer_(idx=i,
                                                                  filters=fs,
                                                                  kernel_size=2,
                                                                  padding='valid',
                                                                  name=f"inf_{i}",
                                                                  activation=tf.nn.leaky_relu)
                                                                for i, fs in enumerate(self._layer_sizes_inference)], name='inference_net'),
                                                input_shape, batch_size)

        self.generative_net = _skip_connection_model_(input_shape=self.latent_dim,
                                                      layer_sizes=self._layer_sizes_generative,
                                                      output_dim=input_shape[-1],
                                                      name='generative_net')

def _convolutional_layer_(idx, **kwargs):
    return tfk.Sequential([tfkl.Conv1D(**{**kwargs, 'name': f"{kwargs['name']}_block_{idx}_conv_0"}),
                           tfkl.Conv1D(**{**kwargs, 'name': f"{kwargs['name']}_block_{idx}_conv_1", 'padding': 'same'}),
                            tfkl.BatchNormalization(name=f"conv_block_{idx}_batch_norm")],
                          name=f"conv_block_{idx}")

class SkipConnectionLayer(tfkl.Layer):
    """
    Taken from https://arxiv.org/pdf/1512.03385.pdf, Deep Residual Learning for Image Recognition

    The batch normalisation prior to a convolution and before activation follows:
    S. Ioffe and C. Szegedy. Batch normalization:  Accelerating deepnetwork training by reducing internal covariate shift. InICML, 2015.

    This class only exists because I can't read too much output. Tensorflow, yeah! (or not)

    Roughtly equivalent (as it was this some time ago) "functional"-style:
    x = tfkl.BatchNormalization()(x)
    x = PaddedConv1dTransposed(n_filters=fs, activation=None, batch_norm=False)(x)
    x_skip = tf.reshape(tfkl.Dense(fs, activation=None)(input_layer), [-1, 1, x.shape[-1]])
    x = x + x_skip
    x = tfkl.BatchNormalization()(x)
    x = tfkl.Activation(tf.nn.leaky_relu)(x)
    """
    def __init__(self, n_filters_weight_layer, activation=tf.nn.leaky_relu, name=None):
        super(SkipConnectionLayer, self).__init__(name=name)
        self.bn_in = tfkl.BatchNormalization()
        self.weight_layer_0 = PaddedConv1dTransposed(n_filters=n_filters_weight_layer, activation=None, batch_norm=False)
        self.act_0 = tfkl.Activation(activation)
        self.weight_layer_1 = PaddedConv1dTransposed(n_filters=n_filters_weight_layer, activation=None, batch_norm=False, padding='same')

        self.bn_sk = tfkl.BatchNormalization()
        self.identity_layer = tfkl.Dense(n_filters_weight_layer, activation=None)
        self.reshape_layer = tfkl.Lambda(lambda x: tf.reshape(x, [-1, 1, n_filters_weight_layer]))
        self.act_1 = tfkl.Activation(activation)

    def call(self, x_and_skipped):
        x, x_skipped = x_and_skipped
        x = self.bn_in(x)
        x_skipped = self.bn_sk(x_skipped) # This is from another paper, which I forgot to bookmark, seemed sensible
        x_skipped = self.identity_layer(x_skipped)

        x = self.weight_layer_0(x)
        x = self.act_0(x)

        x = self.weight_layer_1(x)

        x = x + self.reshape_layer(x_skipped)

        x = self.act_1(x)

        return x

def _skip_connection_model_(input_shape, layer_sizes, output_dim, name):
    """
    """
    input_layer = tfkl.Input(shape=(input_shape,))
    x = tfkl.Lambda(lambda x: tf.reshape(x, [-1, 1, input_shape]))(input_layer)

    for i, fs in enumerate(layer_sizes):
        x = SkipConnectionLayer(fs)([x, input_layer])

    x = tfkl.TimeDistributed(tfkl.Dense(output_dim, activation=None))(x)

    return tfk.Model(inputs=[input_layer], outputs=[x])

max_stars_repo_name: samuelsmal/drosophVAE

max_issues_repo_name: samuelsmal/drosophVAE

max_forks_repo_name: samuelsmal/drosophVAE


text:
"""
    Author: Ot Prat
"""

import sys
import io
import json
import base64
from PIL import Image
import numpy as np

from PyQt5.QtWidgets import *
from PyQt5.QtCore import *

from pyqtgraph.Qt import QtCore, QtGui

import pyqtgraph as pg
from pyqtgraph.parametertree import Parameter, ParameterTree

class DragDropLabel(QLabel):
    """ Empty label on which you can drag a file to open it.
    """
    acceptedFormat = "json"
    droppedFile = pyqtSignal(str)

    def __init__(self, parent):
        super(DragDropLabel, self).__init__(parent)
        self.setAcceptDrops(True)

    def dragEnterEvent(self, event):
        """

        Parameters
        ----------
        event :

        Returns
        -------

        """
        if event.mimeData().hasUrls():
            event.accept()
        else:
            event.ignore()

    def dropEvent(self, event):
        """

        Parameters
        ----------
        event :

        Returns
        -------

        """
        for url in event.mimeData().urls():
            self.filename = url.toLocalFile()
            if list(self.filename.split("."))[-1] == self.acceptedFormat:
                self.droppedFile.emit(self.filename)
            else:
                pass

class JsonReader(QWidget):
    """ """

    def __init__(self):
        super().__init__()
        self.title = "Json metadata reader"
        self.left = 100
        self.top = 100
        self.width = 500
        self.height = 500
        self.initUI()

    def initUI(self):
        """ """
        self.setWindowTitle(self.title)
        self.setGeometry(self.left, self.top, self.width, self.height)

        self.getbtn = QPushButton("Select file")
        self.getbtn.clicked.connect(self.get_file)

        self.draglbl = DragDropLabel(self)
        self.draglbl.setText(
            "... or drop .{} file here".format(DragDropLabel.acceptedFormat.upper())
        )
        self.draglbl.setAlignment(QtCore.Qt.AlignCenter)
        self.draglbl.droppedFile.connect(self.open_file)

        self.layout = QGridLayout(self)
        self.layout.addWidget(self.getbtn, 0, 0)
        self.layout.addWidget(self.draglbl, 1, 0)
        self.layout.setColumnMinimumWidth(0, 500)

        self.setLayout(self.layout)
        self.show()

    def get_file(self):
        """ """
        self.path, _ = QFileDialog.getOpenFileName(filter="Json files (*.json)")

        if not self.path:
            pass
        else:
            self.open_file(self.path)

    def open_file(self, filename):
        """

        Parameters
        ----------
        filename :

        Returns
        -------

        """
        self.filename = filename

        with open(filename) as json_file:
            self.metadata_dict = json.load(json_file)

        self.display_tree_widget(self.metadata_dict)

    def display_tree_widget(self, metadata):
        """Display the parameter tree from the experiment metadata.

        Parameters
        ----------
        metadata :
            json metadata

        Returns
        -------

        """
        # Close previous tree widget
        self.close_tree()

        # Create a variable checking whether changes have been made to the parameter tree values
        self.has_changed = False

        # Create list with parameters for the tree
        self.parameters = self.create_parameters(self.fix_types(metadata))

        # Add parameters for Save buttons and Image options
        self.parameters.append(
            {
                "name": "Metadata Image Options",
                "type": "group",
                "children": [
                    {"name": "Append Image", "type": "action"},
                    {"name": "View Image", "type": "action"},
                ],
            }
        )

        self.parameters.append(
            {
                "name": "Save/Reset metadata",
                "type": "group",
                "children": [
                    {"name": "Save changes", "type": "action"},
                    {"name": "Reset changes", "type": "action"},
                ],
            }
        )

        # Create tree of Parameter objects
        self.p = Parameter.create(name="params", type="group", children=self.parameters)

        # Check if Image parameters already exist. If they don't,
        # add them and update Parameter object
        self.children_list = []
        for child in self.p.children():
            self.children_list.append(str(child).split("'")[1])

        # Save original state
        self.original_state = self.p.saveState()

        # Connect Save/Reset buttons to respective functions
        self.p.param("Save/Reset metadata", "Save changes").sigActivated.connect(
            self.save_treevals
        )
        self.p.param("Save/Reset metadata", "Reset changes").sigActivated.connect(
            self.reset
        )

        # Create ParameterTree widget
        self.tree = ParameterTree()
        self.tree.setParameters(self.p, showTop=False)
        self.tree.setWindowTitle("pyqtgraph example: Parameter Tree")

        # Display tree widget
        self.layout.addWidget(self.tree, 2, 0)

        # Send signal when any entry is changed
        self.p.sigTreeStateChanged.connect(self.change)

    def append_img(self):
        """Attach a .png image to the metadata file."""
        self.imagefile, _ = QFileDialog.getOpenFileName(filter="png images (*.png)")

        if not self.imagefile:
            pass
        else:
            with open(self.imagefile, "rb") as f:
                self.img = f.read()
            self.encoded_img = base64.b64encode(self.img)
            self.encoded_img = self.encoded_img.decode("utf8")
            self.p.param("Metadata Image").setValue(self.encoded_img)
            self.p.param("Metadata Image").setDefault(self.encoded_img)

    def display_img(self):
        """Show image appended to the metadata file"""
        try:
            self.layout.removeWidget(self.imglbl)
            self.imglbl.deleteLater()
            self.imglbl = None
        except AttributeError:
            pass

        if not self.p.param("Metadata Image").defaultValue():
            self.imglbl = QLabel("No image associated to this metadata")
            self.layout.addWidget(self.imglbl, 2, 1)

            try:
                self.layout.removeWidget(self.viewbtn)
                self.viewbtn.deleteLater()
                self.viewbtn = None
            except AttributeError:
                pass

        else:
            self.figstring = self.p.param("Metadata Image").defaultValue()
            self.figbytes = self.figstring.encode("utf8")
            self.figbytes = base64.b64decode(self.figbytes)

            self.viewbtn = QPushButton("Zoom image")
            self.viewbtn.clicked.connect(self.image_viewer)

            self.image = QtGui.QPixmap()
            self.image.loadFromData(self.figbytes)
            self.image = self.image.scaledToHeight(500)

            self.imglbl = QLabel()
            self.imglbl.setPixmap(self.image)
            self.layout.addWidget(self.imglbl, 0, 1, 3, 1)
            self.layout.addWidget(self.viewbtn, 3, 1)

    def image_viewer(self):
        """Open metadata image in the Image Viewer from PyQtGraph."""
        self.win = QtGui.QMainWindow()
        self.win.resize(800, 800)

        self.fignp = np.array(Image.open(io.BytesIO(self.figbytes)))
        self.fignp = np.swapaxes(self.fignp, 0, 1)
        self.imv = pg.ImageView()
        self.imv.setImage(self.fignp)

        self.win.setCentralWidget(self.imv)
        self.win.setWindowTitle("Metadata Image")
        self.win.show()

    def save_treevals(self):
        """Save current values of the parameter tree into a dictionary."""
        # Recover data from tree and store it in a dict
        self.treevals_dict = self.p.getValues()
        self.metadata_dict_mod = self.get_mod_dict(self.treevals_dict)

        # Nasty way to make new dict (with modified metadata) with same structure as the original one
        self.metadata_dict_mod.pop("Save/Reset metadata")
        self.metadata_dict_mod.pop("Metadata Image Options")
        self.metadata_dict_mod["stimulus"]["log"] = self.metadata_dict["stimulus"][
            "log"
        ]
        self.metadata_dict_mod["stimulus"]["display_params"]["pos"] = json.loads(
            self.metadata_dict_mod["stimulus"]["display_params"]["pos"]
        )
        self.metadata_dict_mod["stimulus"]["display_params"]["size"] = json.loads(
            self.metadata_dict_mod["stimulus"]["display_params"]["size"]
        )

        self.show_warning()

    def show_warning(self):
        """Upon saving, display a warning message
         to choose whether to create a new metadata file or replace the existing one.

        Parameters
        ----------

        Returns
        -------

        """
        if self.has_changed:
            self.msg = QMessageBox()
            self.msg.setIcon(QMessageBox.Warning)
            self.setWindowTitle("Saving Warning")
            self.msg.setText("Some parameters have changed")
            self.msg.setInformativeText(
                "Do you want to overwrite the original .json metadata file?"
            )
            self.msg.addButton("Create new file", QMessageBox.AcceptRole)
            self.msg.setStandardButtons(QMessageBox.Yes | QMessageBox.No)

            self.ret = self.msg.exec_()

            if self.ret == QMessageBox.Yes:
                self.overwrite_metadata_file(self.metadata_dict_mod)
            elif self.ret == QMessageBox.AcceptRole:
                self.create_metadata_file(self.metadata_dict_mod)
            else:
                pass
        else:
            self.msg2 = QMessageBox()
            self.msg2.setIcon(QMessageBox.Information)
            self.setWindowTitle("Saving Warning")
            self.msg2.setText("No changes have been made.")
            self.msg2.addButton("OK", QMessageBox.AcceptRole)

            self.ret = self.msg2.exec_()

    def overwrite_metadata_file(self, metadata_dict_mod):
        """

        Parameters
        ----------
        metadata_dict_mod :

        Returns
        -------

        """
        # Overwritte original metadata file
        with open(self.filename, "w") as file:
            json.dump(metadata_dict_mod, file)

    def create_metadata_file(self, metadata_dict_mod):
        """

        Parameters
        ----------
        metadata_dict_mod :

        Returns
        -------

        """
        # Overwritte original metadata file
        self.name, self.ext = self.filename.split(".")
        with open("{}_modified.{}".format(self.name, self.ext), "w") as file:
            json.dump(metadata_dict_mod, file)

    def reset(self):
        """Reset parameter tree values to the original state after loading."""
        self.p.restoreState(self.original_state, recursive=True)
        self.tree.setParameters(self.p, showTop=False)

    def fix_types(self, datadict):
        """Modify metadata dict so only accepted types are found.

        Parameters
        ----------
        datadict :

        Returns
        -------

        """
        param_dict = dict()
        for key, value in datadict.items():
            if isinstance(value, list):
                param_dict[key] = str(value)
            elif isinstance(value, dict):
                param_dict[key] = self.fix_types(value)
            else:
                param_dict[key] = value
        return param_dict

    def create_parameters(self, datadict):
        """Create list with parameters and Children to which the tree will be built from.

        Parameters
        ----------
        datadict :

        Returns
        -------

        """
        parameters = []
        for key, value in datadict.items():
            if key == "log":
                pass
            else:
                if isinstance(value, dict):
                    parameters.append(
                        {
                            "name": "{}".format(key),
                            "type": "group",
                            "children": self.create_parameters(value),
                        }
                    )
                else:
                    parameters.append(
                        {
                            "name": "{}".format(key),
                            "type": "{}".format(type(value).__name__),
                            "value": value,
                        }
                    )
        return parameters

    def get_mod_dict(self, treevals_dict):
        """Recursive function to convert into dict output of getValues function.

        Parameters
        ----------
        treevals_dict :

        Returns
        -------

        """
        metadata_dict_mod = dict()
        for key, value in treevals_dict.items():
            if value[0] is None:
                metadata_dict_mod[key] = dict(self.get_mod_dict(value[1]))
            else:
                metadata_dict_mod[key] = value[0]

        return metadata_dict_mod

    def change(self, param, changes):
        """

        Parameters
        ----------
        param :

        changes :

        Returns
        -------

        """
        print("tree changes:")
        for param, change, data in changes:
            path = self.p.childPath(param)
            if path is not None:
                childName = ".".join(path)
            else:
                childName = param.name()
            print("  parameter: %s" % childName)
            print("  change:    %s" % change)
            print("  data:      %s" % str(data))
            print("  ----------")

            if change == "activated":
                pass
            else:
                self.has_changed = True

    def close_tree(self):
        """ """
        try:
            self.layout.removeWidget(self.tree)
            self.tree.deleteLater()
            self.tree = None
        except AttributeError:
            pass

if __name__ == "__main__":
    app = QApplication(sys.argv)
    ex = JsonReader()
    sys.exit(app.exec_())

max_stars_repo_name: mark-dawn/stytra

max_issues_repo_name: mark-dawn/stytra

max_forks_repo_name: mark-dawn/stytra


text:
from load_macrodata import dta
import matplotlib.pyplot as plt
import pandas as pd

import statsmodels.api as sm

cycle, trend = sm.tsa.filters.hpfilter(dta.realgdp, 1600)
gdp_decomp = dta[['realgdp']].copy()
gdp_decomp["cycle"] = cycle
gdp_decomp["trend"] = trend

fig, ax = plt.subplots()
gdp_decomp[["realgdp", "trend"]]["2000-03-31":].plot(ax=ax,
                                                     fontsize=16)

max_stars_repo_name: CCHiggins/statsmodels

max_issues_repo_name: CCHiggins/statsmodels

max_forks_repo_name: CCHiggins/statsmodels

reason: import statsmodels


text:
import keras
from keras.models import Sequential
from keras.models import load_model
from keras.layers import Dense
from keras.optimizers import Adam,RMSprop
from sklearn.linear_model import LinearRegression
from sklearn.ensemble import RandomForestClassifier
import matplotlib.pyplot as plt
from keras.utils import to_categorical
import os
os.environ['KERAS_BACKEND' ] = 'tensorflow'
os.environ['MKL_THREADING_LAYER'] = 'GNU'
from keras.callbacks import TensorBoard
import pandas as pd
import numpy as np
from keras import optimizers
from keras.utils import to_categorical
from keras.models import Model
from keras.layers import Input, Dense, Dropout
from keras.layers.normalization import BatchNormalization
from keras.layers.core import Dropout, Activation
from sklearn.preprocessing import MinMaxScaler
import time
import numpy as np
import random
from collections import deque
class Agent:
	def __init__(self, state_size, is_eval=False, model_name=""):
		self.state_size = state_size # normalized previous days
		self.action_size = 3 # sit, buy, sell
		self.memory = deque(maxlen=1020)
		self.inventory = []
		self.model_name = model_name
		self.is_eval = is_eval
		self.gamma = 0.45
		self.epsilon = 1.0
		self.epsilon_min = 0.15
		self.epsilon_decay = 0.9999999
		self.model = load_model("models/" + model_name) if is_eval else self._model()

	def _model(self):
		batch_size = 666
		dr = 0.75

		visible = Input(shape=(self.state_size,))
		hidden1 = Dense(5, activation='sigmoid')(visible)
		hidden1 = Dense(5, activation='sigmoid')(hidden1)
		hidden1 = Dense(5, activation='sigmoid')(hidden1)
		hidden1 = Dense(5, activation='sigmoid')(hidden1)
		hidden1 = Dense(5, activation='sigmoid')(hidden1)
		hidden1 = Dense(self.action_size, activation="linear")(hidden1)
		hidden1 = Dropout(dr)(hidden1)

		hidden2 = Dense(self.action_size, activation="linear")(hidden1)
		merge0 = keras.layers.concatenate([hidden1,hidden2], axis=1)
		hidden3 = Dense(5, activation='sigmoid')(merge0)
		hidden3 = Dense(5, activation='sigmoid')(hidden3)
		hidden3 = Dense(5, activation='sigmoid')(hidden3)
		hidden3 = Dense(self.action_size, activation="relu")(hidden3)
		hidden3 = Dropout(dr)(hidden3)

		hidden4 = Dense(self.action_size, activation="relu")(hidden3)
		output =Dense(self.action_size, activation="sigmoid")(hidden4)
		model = Model(inputs=visible, outputs=output)
		model.compile(loss='mse',optimizer='rmsprop')
		return model

	def act(self, state):
		if not self.is_eval and random.random() <= self.epsilon:
			return random.randrange(self.action_size)
		action = self.model.predict(state)
		return np.argmax(action[0])

	def expReplay(self, batch_size):
		mini_batch = []
		l = len(self.memory)
		for i in range(l - batch_size + 1, l):
			mini_batch.append(self.memory[i])
		for state, action, reward, next_state, done in mini_batch:
			target = reward
			if not done:
				target = reward + self.gamma * np.amax(self.model.predict(next_state)[0])

			target_f = self.model.predict(state)
			target_f[0][action] = target
			self.model.fit(state, target_f, epochs=1, verbose=0,validation_split=.65)

		if self.epsilon > self.epsilon_min:
			self.epsilon *= self.epsilon_decay