text:
#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

import os
import six
import ast
import copy
import re
import shutil
import time

import numpy as np
import paddle.fluid as fluid
import paddle.distributed.fleet as fleet
import paddle
from .topo import Topology

def init_pretraining_params(exe,
                            pretraining_params_path,
                            main_program,
                            ):
    assert os.path.exists(pretraining_params_path
                          ), "[%s] cann't be found." % pretraining_params_path

    def existed_params(var):
        var_path = os.path.join(pretraining_params_path, var.name)
        var_exists = os.path.exists(var_path)
        if not isinstance(var, fluid.framework.Parameter):
            return False
        else:
            if var_exists:
                print('loading {} for {}'.format(var_path, var.name))
            else:
                print('do not detecting param {} for {}'.format(var.name, var_path))
        return var_exists

    fluid.io.load_vars(
        exe,
        pretraining_params_path,
        main_program=main_program,
        predicate=existed_params)
    print("Load pretraining parameters from {}.".format(pretraining_params_path))

def init_checkpoint(exe, init_checkpoint_path, main_program):
    assert os.path.exists(
        init_checkpoint_path), "[%s] cann't be found." % init_checkpoint_path

    def existed_persitables(var):
        if not fluid.io.is_persistable(var):
            return False
        return os.path.exists(os.path.join(init_checkpoint_path, var.name))

    fluid.io.load_vars(
        exe,
        init_checkpoint_path,
        main_program=main_program,
        predicate=existed_persitables)
    print("Load model from {}".format(init_checkpoint_path))

def checkpoint_rearrange(save_model_dir, init_checkpoint_path, index, num_pp, num_mp, steps, dp_rank):
    matchObj = re.search(r'.*saved_model_pp(\d+)mp(\d+).*', init_checkpoint_path, re.M|re.I)
    if matchObj:
        pre_num_pp = int(matchObj.group(1))
        pre_num_mp = int(matchObj.group(2))
    else:
        raise Exception('saved_model_pp(\d+)mp(\d+) must in init_checkpoint_path')

    if pre_num_pp < num_pp:
        raise Exception('current num_pp({}) must less than or equal to checkpoint num_pp({})'.format(num_pp, pre_num_pp))

    if pre_num_mp < num_mp:
        raise Exception('current num_mp({}) must equal to checkpoint num_mp({})'.format(num_mp, pre_num_mp))

    pre_topo = Topology(rank=index, world_size=pre_num_pp*pre_num_mp, dp=1, pp=pre_num_pp, sharding=1, mp=pre_num_mp)
    cur_topo = Topology(rank=index, world_size=num_pp*num_mp, dp=1, pp=num_pp, sharding=1, mp=num_mp)
    start_index = cur_topo.pp.world.index(index) * len(cur_topo.pp.world)
    pre_ranks = pre_topo.pp.world[start_index:start_index+len(cur_topo.pp.world)]
    dst_dir = os.path.join(save_model_dir, 'rank_' + str(index), 'step_' + str(steps))
    lock_file_name = os.path.join(save_model_dir, 'rank_' + str(index), 'step_' + str(steps) + '.lock')
    if dp_rank == 0:
        shutil.rmtree(dst_dir, ignore_errors=True)
        os.makedirs(dst_dir)
        for idx in pre_ranks:
            src_dir = os.path.join(init_checkpoint_path, 'rank_' + str(idx), 'step_' + str(steps))
            for dirpath, dirnames, filenames in os.walk(src_dir):
                for filename in filenames:
                    src_path = os.path.join(dirpath, filename)
                    shutil.copy(src_path, dst_dir)
        with open(lock_file_name, 'w') as f:
            pass
        time.sleep(3)
        os.remove(lock_file_name)
    else:
        while True:
            if not os.path.exists(lock_file_name):
                time.sleep(1)
            else:
                break

def save_checkpoint(exe, save_path, train_program, num_sharding=1, num_pp=1):
    if num_sharding > 1:
        fleet.meta_optimizers.sharding.utils.save_persistables(exe, save_path, train_program)
    elif num_pp > 1:
        fluid.io.save_persistables(exe, save_path, train_program._pipeline_opt['section_program'])
    else:
        fluid.io.save_persistables(exe, save_path, train_program)

def save_inference_model(dirname,
                         feeded_var_names,
                         fetch_var_names,
                         exe,
                         train_program,
                         num_sharding=1,
                         num_pp=1,
                         is_first=False,
                         is_last=False):
    if num_sharding > 1:
        raise Exception('num_sharding > 1 do not supported save_inference_model now.')
    elif num_pp > 1:
        op_role_key = fluid.core.op_proto_and_checker_maker.kOpRoleAttrName()
        op_role_forward = int(fluid.core.op_proto_and_checker_maker.OpRole.Forward)

        ops = train_program._pipeline_opt['section_program'].global_block().ops
        if is_first:
            fetch_var_names = []
            program_input_names = []
            last_idx = -1
            for idx, op in enumerate(ops):
                if op.type == 'partial_send' and op.attr(op_role_key) == op_role_forward:
                    fetch_var_names += op.input("X")
                    last_idx = max(last_idx, idx)
                elif op.type == 'read' and op.attr(op_role_key) == op_role_forward:
                    program_input_names += op.output("Out")

            for idx, op in enumerate(ops):
                if idx > last_idx:
                    break
                for input_name in program_input_names:
                    if input_name in op.input_arg_names and input_name not in feeded_var_names:
                        feeded_var_names.append(input_name)

        elif is_last:
            feeded_var_names = []
            program_input_names = []
            last_idx = -1
            for idx, op in enumerate(ops):
                if op.type == 'partial_recv' and op.attr(op_role_key) == op_role_forward:
                    feeded_var_names += op.output("Out")
                elif op.type == 'read' and op.attr(op_role_key) == op_role_forward:
                    program_input_names += op.output("Out")

                for output_name in fetch_var_names:
                    if output_name in op.output_arg_names:
                        last_idx = max(last_idx, idx)

            for idx, op in enumerate(ops):
                if idx > last_idx:
                    break
                for input_name in program_input_names:
                    if input_name in op.input_arg_names and input_name not in feeded_var_names:
                        feeded_var_names.append(input_name)

        else:
            feeded_var_names = []
            fetch_var_names = []
            program_input_names = []
            last_idx = -1
            for idx, op in enumerate(ops):
                if op.type == 'partial_recv' and op.attr(op_role_key) == op_role_forward:
                    feeded_var_names += op.output("Out")
                elif op.type == 'partial_send' and op.attr(op_role_key) == op_role_forward:
                    fetch_var_names += op.input("X")
                    last_idx = max(last_idx, idx)
                elif op.type == 'read' and op.attr(op_role_key) == op_role_forward:
                    program_input_names += op.output("Out")

            for idx, op in enumerate(ops):
                if idx > last_idx:
                    break
                for input_name in program_input_names:
                    if input_name in op.input_arg_names and input_name not in feeded_var_names:
                        feeded_var_names.append(input_name)

        feeded_vars = [train_program.global_block().vars[var_name] for var_name in feeded_var_names]
        fetch_vars = [train_program.global_block().vars[var_name] for var_name in fetch_var_names]

        paddle.static.save_inference_model(dirname, feeded_vars, fetch_vars, exe,
            program=train_program._pipeline_opt['section_program'])
    else:
        feeded_vars = [train_program.global_block().vars[var_name] for var_name in feeded_var_names]
        fetch_vars = [train_program.global_block().vars[var_name] for var_name in fetch_var_names]
        paddle.static.save_inference_model(dirname, feeded_vars, fetch_vars, exe,
            program=train_program)

max_stars_repo_name: sneaxiy/FleetX

max_issues_repo_name: sneaxiy/FleetX

max_forks_repo_name: sneaxiy/FleetX


text:
import os
import numpy as np
import torch
from torch import nn

from aw_nas import utils
from aw_nas.final.cnn_trainer import CNNFinalTrainer #, _warmup_update_lr
from aw_nas.utils.common_utils import nullcontext
from aw_nas.utils.exception import expect

def _warmup_update_lr(optimizer, epoch, init_lr, warmup_epochs, warmup_ratio=0.0):
    k = (1 - epoch / warmup_epochs) * (1 - warmup_ratio)
    lr = init_lr * (1 - k)
    for param_group in optimizer.param_groups:
        param_group["lr"] = lr
    return lr

class WrapperFinalTrainer(CNNFinalTrainer):  # pylint: disable=too-many-instance-attributes
    NAME = "wrapper_final_trainer"

    def __init__(
            self,
            model,
            dataset,
            device,
            gpus,
            objective,  # pylint: disable=dangerous-default-value
            multiprocess=False,
            epochs=600,
            batch_size=96,
            optimizer_type="SGD",
            optimizer_kwargs=None,
            learning_rate=0.025,
            momentum=0.9,
            freeze_base_net=False,
            base_net_lr=1e-4,
            warmup_epochs=0,
            warmup_steps=0,
            warmup_ratio=0.,
            optimizer_scheduler={
                "type": "CosineAnnealingLR",
                "T_max": 600,
                "eta_min": 0.001
            },
            weight_decay=3e-4,
            no_bias_decay=False,
            grad_clip=5.0,
            auxiliary_head=False,
            auxiliary_weight=0.4,
            add_regularization=False,
            save_as_state_dict=False,
            workers_per_queue=2,
            eval_every=10,
            eval_batch_size=1,
            eval_no_grad=True,
            eval_dir=None,
            calib_bn_setup=False,
            seed=None,
            schedule_cfg=None):

        self.freeze_base_net = freeze_base_net
        self.base_net_lr = base_net_lr
        super(WrapperFinalTrainer,
              self).__init__(model, dataset, device, gpus, objective,
                             multiprocess, epochs, batch_size, optimizer_type,
                             optimizer_kwargs, learning_rate, momentum,
                             warmup_epochs, optimizer_scheduler, weight_decay,
                             no_bias_decay, grad_clip, auxiliary_head,
                             auxiliary_weight, add_regularization,
                             save_as_state_dict, workers_per_queue,
                             eval_no_grad, eval_every, eval_batch_size, calib_bn_setup, seed, schedule_cfg)

        self._criterion = self.objective._criterion
        self._acc_func = self.objective.get_acc
        self._perf_func = self.objective.get_perfs

        self.warmup_steps = warmup_steps
        self.warmup_ratio = warmup_ratio

    def _init_optimizer(self):
        optim_cls = getattr(torch.optim, self.optimizer_type)
        optim_kwargs = {
            "lr": self.learning_rate,
            "momentum": self.momentum,
            "weight_decay": self.weight_decay
        }
        backbone = self.model.backbone
        neck = self.model.neck
        head = self.model.head
        if not self.freeze_base_net:
            params = [{"params": self.model.parameters()}]
        else:
            params = [{
                "params": backbone.parameters(),
                "lr": self.base_net_lr
            }, {
                "params": head.parameters()
            }]
            if neck is not None:
                params += [{"params": neck.parameters()}]
        optim_kwargs.update(self.optimizer_kwargs or {})
        optimizer = optim_cls(params, **optim_kwargs)

        return optimizer

    def evaluate_split(self, split):
        # if len(self.gpus) >= 2:
        #     self._forward_once_for_flops(self.model)
        rank = os.environ.get("LOCAL_RANK")
        if rank is not None and rank != '0':
            return 0., 0.
        assert split in {"train", "test"}
        if split == "test":
            queue = self.valid_queue
        else:
            queue = self.train_queue
        acc, obj, perfs = self.infer_epoch(queue, self.parallel_model,
                                           self._criterion, self.device)
        self.logger.info(
            "acc %f ; obj %f ; performance: %s", acc, obj,
            "; ".join(["{}: {:.3f}".format(n, v) for n, v in perfs.items()]))
        return acc, obj

    @classmethod
    def supported_data_types(cls):
        return ["image"]

    def train_epoch(self, train_queue, model, criterion, optimizer, device,
                    epoch):
        expect(self._is_setup, "trainer.setup should be called first")
        top1 = utils.AverageMeter()
        top5 = utils.AverageMeter()
        losses_obj = utils.OrderedStats()
        model.train()
        self.objective.set_mode("train")

        for step, (inputs, targets) in enumerate(train_queue):
            cur_step = step + len(train_queue) * (epoch - 1)
            if 0 <= cur_step <= self.warmup_steps:
                lr = _warmup_update_lr(optimizer, cur_step,
                                       self.learning_rate, self.warmup_steps, self.warmup_ratio)
                if cur_step % self.report_every == 0:
                    self.logger.info("Step {} LR: {}".format(step, lr))

            inputs = inputs.to(self.device)
            optimizer.zero_grad()
            predictions = model.forward(inputs)
            losses = criterion(inputs, predictions, targets, model)
            loss = sum(losses.values())
            loss.backward()

            #for p in optimizer.param_groups[0]['params']:
            #    if p.grad is not None:
            #        print(p.grad.sum())

            if isinstance(self.grad_clip, (int, float)) and self.grad_clip > 0:
                nn.utils.clip_grad_norm_(model.parameters(), self.grad_clip)
            optimizer.step()
            prec1, prec5 = self._acc_func(inputs, predictions, targets, model)
            n = inputs.size(0)
            losses_obj.update(losses)
            top1.update(prec1.item(), n)
            top5.update(prec5.item(), n)

            if step % self.report_every == 0:
                self.logger.info("train %03d %.2f%%; %.2f%%; %s",
                                 step, top1.avg, top5.avg, "; ".join(
                                     ["{}: {:.3f}".format(perf_n, v)
                                      for perf_n, v in losses_obj.avgs().items()]))
        return top1.avg, sum(losses_obj.avgs().values())

    def infer_epoch(self, valid_queue, model, criterion, device):
        expect(self._is_setup, "trainer.setup should be called first")
        top1 = utils.AverageMeter()
        top5 = utils.AverageMeter()
        objective_perfs = utils.OrderedStats()
        losses_obj = utils.OrderedStats()
        all_perfs = []
        model.eval()
        self.objective.set_mode("eval")

        context = torch.no_grad if self.eval_no_grad else nullcontext
        with context():
            for step, (inputs, targets) in enumerate(valid_queue):
                inputs = inputs.to(device)
                #targets = targets.to(device)

                predictions = model(inputs)
                losses = criterion(inputs, predictions, targets, model)
                prec1, prec5 = self._acc_func(inputs, predictions, targets,
                                              model)
                perfs = self._perf_func(inputs, predictions, targets, model)
                all_perfs.append(perfs)
                n = inputs.size(0)
                objective_perfs.update(dict(zip(self._perf_names, perfs)), n=n)
                losses_obj.update(losses, n=n)
                top1.update(prec1.item(), n)
                top5.update(prec5.item(), n)

                if step % self.report_every == 0:
                    self.logger.info(
                        "valid %03d; %s, %s", step, ";".join(
                            ["%s: %.3f" % l for l in losses.items()]),
                        "; ".join([
                            "{}: {:.3f}".format(perf_n, v) for perf_n, v in
                            list(objective_perfs.avgs().items())]))
        all_perfs = list(zip(*all_perfs))
        obj_perfs = {
            k: self.objective.aggregate_fn(k, False)(v)
            for k, v in zip(self._perf_names, all_perfs)
        }
        return top1.avg, sum(losses_obj.avgs().values()), obj_perfs

ext: py

reason: import numpy


text:
# SPDX-FileCopyrightText: 2021 Division of Intelligent Medical Systems, DKFZ
# SPDX-FileCopyrightText: 2021 Janek Groehl
# SPDX-License-Identifier: MIT

import unittest
from simpa.utils import Tags
from simpa.utils.settings import Settings
from simpa.core.simulation import simulate
import numpy as np
from simpa_tests.test_utils import create_test_structure_parameters
import os
from simpa import ModelBasedVolumeCreationAdapter
from simpa.core.simulation_modules.optical_simulation_module.optical_forward_model_test_adapter import \
    OpticalForwardModelTestAdapter
from simpa.core.simulation_modules.acoustic_forward_module.acoustic_forward_model_test_adapter import \
    AcousticForwardModelTestAdapter
from simpa.core.device_digital_twins import RSOMExplorerP50

class TestPipeline(unittest.TestCase):

    def setUp(self):

        self.VOLUME_WIDTH_IN_MM = 4
        self.VOLUME_HEIGHT_IN_MM = 3
        self.SPACING = 0.25
        self.RANDOM_SEED = 4711

    def test_pipeline(self):
        # Seed the numpy random configuration prior to creating the settings file in
        # order to ensure that the same volume
        # is generated with the same random seed every time.

        np.random.seed(self.RANDOM_SEED)

        settings = {
            # These parameters set the general propeties of the simulated volume
            Tags.RANDOM_SEED: self.RANDOM_SEED,
            Tags.VOLUME_NAME: "TestName_"+str(self.RANDOM_SEED),
            Tags.SIMULATION_PATH: ".",
            Tags.SPACING_MM: self.SPACING,
            Tags.DIM_VOLUME_Z_MM: self.VOLUME_HEIGHT_IN_MM,
            Tags.DIM_VOLUME_X_MM: self.VOLUME_WIDTH_IN_MM,
            Tags.DIM_VOLUME_Y_MM: self.VOLUME_WIDTH_IN_MM,

            # The following parameters set the optical forward model
            Tags.WAVELENGTHS: [800],

            # The following parameters tell the script that we do not want any extra
            # modelling steps
            # Add the volume_creation_module to be simulated to the tissue
        }

        settings = Settings(settings)
        settings.set_volume_creation_settings(
            {
                Tags.STRUCTURES: create_test_structure_parameters()
            }
        )
        settings.set_optical_settings({
            Tags.OPTICAL_MODEL_NUMBER_PHOTONS: 1e7,
            Tags.OPTICAL_MODEL: Tags.OPTICAL_MODEL_TEST,
            Tags.ILLUMINATION_TYPE: Tags.ILLUMINATION_TYPE_PENCIL,
            Tags.LASER_PULSE_ENERGY_IN_MILLIJOULE: 50
        })
        settings.set_acoustic_settings({
                Tags.OPTICAL_MODEL_NUMBER_PHOTONS: 1e7,
                Tags.OPTICAL_MODEL: Tags.OPTICAL_MODEL_TEST,
                Tags.ILLUMINATION_TYPE: Tags.ILLUMINATION_TYPE_PENCIL,
                Tags.LASER_PULSE_ENERGY_IN_MILLIJOULE: 50
        })

        simulation_pipeline = [
            ModelBasedVolumeCreationAdapter(settings),
            OpticalForwardModelTestAdapter(settings),
            AcousticForwardModelTestAdapter(settings),
        ]

        simulate(simulation_pipeline, settings, RSOMExplorerP50(0.1, 1, 1))

        if (os.path.exists(settings[Tags.SIMPA_OUTPUT_PATH]) and
                os.path.isfile(settings[Tags.SIMPA_OUTPUT_PATH])):
            # Delete the created file
            os.remove(settings[Tags.SIMPA_OUTPUT_PATH])

max_stars_repo_name: IMSY-DKFZ/simpa

max_issues_repo_name: jgroehl/simpa

max_forks_repo_name: IMSY-DKFZ/simpa


text:
"""Unit test for DataReader (public methods only)"""
import unittest
import numpy as np
import os
from dicom_data_preprocess import parsing
from dicom_data_preprocess.reader import DataReader

__author__ = 'Christine Hsu'

class TestReader(unittest.TestCase):
	@classmethod
	def setUpClass(TestReader):
		TestReader.download_data_path = 'tests/data/sample-batchset/'
		TestReader.data_basepath = 'tests/data/output_data/'
		TestReader.logs_path = 'tests/logs/',
		TestReader.plots_path = 'tests/plots/'
		TestReader.contour_type = 'i-contours'
		TestReader.save_plot = False

		TestReader.dicoms_basepath = os.path.join(TestReader.download_data_path, 'dicoms')
		TestReader.contours_basepath = os.path.join(TestReader.download_data_path, 'contourfiles')
		TestReader.link_filepath = os.path.join(TestReader.download_data_path, 'link.csv')

		link_tuples = DataReader._read_link(TestReader, TestReader.link_filepath)
		TestReader.sample_tuples = DataReader._assemble_link(TestReader, link_tuples)

	def test_load_samples(self):
		print('\nTesting the loading of eight assembled samples...')

		reader = DataReader(download_data_path=TestReader.download_data_path,
			data_basepath=TestReader.data_basepath,
			logs_path=TestReader.logs_path,
			plots_path=TestReader.plots_path,
			contour_type=TestReader.contour_type,
			save_plot=TestReader.save_plot)

		images, masks, metadata = reader.load_samples(TestReader.sample_tuples)
		self.assertTrue(isinstance(images, list))
		self.assertTrue(isinstance(masks, list))
		self.assertTrue(isinstance(metadata, list))
		self.assertTrue(isinstance(images[0], np.ndarray))
		self.assertEqual(masks[0].dtype, np.bool)
		self.assertTrue(isinstance(metadata[0], str))

		reader.plot_samples(images, masks, metadata, 'test_load_samples.jpg')

if __name__ == "__main__":
	unittest.main()


text:
import numpy as np
import automatic_speech_recognition as asr

def test_standardize():
    features = np.random.normal(loc=2, size=[20, 10])
    standardized = asr.features.FeaturesExtractor.standardize(features)
    assert standardized.shape == (20, 10)
    assert np.isclose(standardized.mean(), 0)
    assert np.isclose(standardized.std(), 1)

def test_align():
    X = asr.features.FeaturesExtractor.align([np.ones([50, 10]),
                                              np.zeros([70, 10]),
                                              np.ones([60, 10])], default=9)
    assert X.shape == (3, 70, 10)
    assert np.all(X[2, 60:, :] == 9)


text:
""" Test ``yatsm changemap ...``
"""
from click.testing import CliRunner
import numpy as np

from yatsm.cli.main import cli

# flake8: ignore = E241

# TRUTH VALUES
first_change_YYYYDOY = np.array([
    [  -9999,   -9999, 2003288,   -9999,   -9999],
    [  -9999,   -9999,   -9999,   -9999,   -9999],
    [  -9999,   -9999, 2000152, 2001178, 2001146],
    [2001162,   -9999, 2001234, 2001234, 2001298],
    [2001234, 2001234, 2001194, 2001170, 2001266]], dtype=np.int32)
first_change_YYYYMMDD = np.array([
    [   -9999,    -9999, 20031015,    -9999,    -9999],
    [   -9999,    -9999,    -9999,    -9999,    -9999],
    [   -9999,    -9999, 20000531, 20010627, 20010526],
    [20010611,    -9999, 20010822, 20010822, 20011025],
    [20010822, 20010822, 20010713, 20010619, 20010923]], dtype=np.int32)

last_change_YYYYDOY = np.array([
    [  -9999,   -9999, 2003288,   -9999,   -9999],
    [  -9999,   -9999,   -9999,   -9999,   -9999],
    [  -9999,   -9999, 2000152, 2003192, 2010275],
    [2002229,   -9999, 2011182, 2001234, 2001298],
    [2001234, 2001234, 2010275, 2010275, 2010275]], dtype=np.int32)

num_change = np.array([
    [-9999, -9999,     1, -9999, -9999],
    [-9999, -9999, -9999, -9999, -9999],
    [-9999, -9999,     1,     2,     2],
    [    2, -9999,     2,     1,     1],
    [    1,     1,     2,     3,     3]], dtype=np.int32)

def test_changemap_first_pass_1(example_results, tmpdir, read_image):
    """ Make changemap of first change
    """
    image = tmpdir.join('first.gtif').strpath
    runner = CliRunner()
    result = runner.invoke(
        cli,
        ['-v', 'changemap',
         '--root', example_results['root'],
         '--result', example_results['results_dir'],
         '--image', example_results['example_img'],
         'first', '2000-01-01', '2020-01-01', image
         ]
    )
    assert result.exit_code == 0
    np.testing.assert_equal(read_image(image)[0, ...], first_change_YYYYDOY)

def test_changemap_first_pass_2(example_results, tmpdir, read_image):
    """ Make changemap of first change, output as YYYYMMDD
    """
    image = tmpdir.join('first.gtif').strpath
    runner = CliRunner()
    result = runner.invoke(
        cli,
        ['-v', 'changemap',
         '--root', example_results['root'],
         '--result', example_results['results_dir'],
         '--image', example_results['example_img'],
         '--out_date', '%Y%m%d',
         'first', '2000-01-01', '2020-01-01', image
         ]
    )
    assert result.exit_code == 0
    np.testing.assert_equal(read_image(image)[0, ...], first_change_YYYYMMDD)

def test_changemap_first_pass_3(example_results, tmpdir, read_image):
    """ Make changemap of first change, but with all NDV
    """
    image = tmpdir.join('first.gtif').strpath
    runner = CliRunner()
    result = runner.invoke(
        cli,
        ['-v', 'changemap',
         '--root', example_results['root'],
         '--result', example_results['results_dir'],
         '--image', example_results['example_img'],
         '--out_date', '%Y%m%d',
         '--ndv', '0',
         'first', '1914-07-28', '1918-11-11', image
         ]
    )
    assert result.exit_code == 0
    np.testing.assert_equal(read_image(image)[0, ...],
                            np.zeros((5, 5), dtype=np.int16))

def test_changemap_last_pass_1(example_results, tmpdir, read_image):
    """ Make changemap of last change
    """
    image = tmpdir.join('last.gtif').strpath
    runner = CliRunner()
    result = runner.invoke(
        cli,
        ['-v', 'changemap',
         '--root', example_results['root'],
         '--result', example_results['results_dir'],
         '--image', example_results['example_img'],
         'last', '2000-01-01', '2020-01-01', image
         ]
    )
    assert result.exit_code == 0
    np.testing.assert_equal(read_image(image)[0, ...], last_change_YYYYDOY)

def test_changemap_num_pass_1(example_results, tmpdir, read_image):
    """ Make changemap of number of changes
    """
    image = tmpdir.join('num.gtif').strpath
    runner = CliRunner()
    result = runner.invoke(
        cli,
        ['-v', 'changemap',
         '--root', example_results['root'],
         '--result', example_results['results_dir'],
         '--image', example_results['example_img'],
         'num', '2000-01-01', '2020-01-01', image
         ]
    )
    assert result.exit_code == 0
    np.testing.assert_equal(read_image(image)[0, ...], num_change)

max_stars_repo_name: TedrosGitHub/TSA-yatsm

max_issues_repo_name: TedrosGitHub/TSA-yatsm

max_forks_repo_name: TedrosGitHub/TSA-yatsm


text:
import csv
import os.path
import argparse

import numpy as np
import tensorflow as tf
import multiprocessing

def seq2onehot(seq):
    """Create 26-dim embedding"""
    chars = ['-', 'D', 'G', 'U', 'L', 'N', 'T', 'K', 'H', 'Y', 'W', 'C', 'P',
             'V', 'S', 'O', 'I', 'E', 'F', 'X', 'Q', 'A', 'B', 'Z', 'R', 'M']
    vocab_size = len(chars)
    vocab_embed = dict(zip(chars, range(vocab_size)))

    # Convert vocab to one-hot
    vocab_one_hot = np.zeros((vocab_size, vocab_size), int)
    for _, val in vocab_embed.items():
        vocab_one_hot[val, val] = 1

    embed_x = [vocab_embed[v] for v in seq]
    seqs_x = np.array([vocab_one_hot[j, :] for j in embed_x])

    return seqs_x

def load_list(fname):
    """
    Load PDB chains
    """
    pdb_chain_list = []
    fRead = open(fname, 'r')
    for line in fRead:
        pdb_chain_list.append(line.strip())
    fRead.close()

    return pdb_chain_list

def load_GO_annot(filename):
    """ Load GO annotations """
    onts = ['molecular_function', 'biological_process', 'cellular_component']
    prot2annot = {}
    goterms = {ont: [] for ont in onts}
    gonames = {ont: [] for ont in onts}
    with open(filename, mode='r') as tsvfile:
        reader = csv.reader(tsvfile, delimiter='\t')

        # molecular function
        next(reader, None)  # skip the headers
        goterms[onts[0]] = next(reader)
        next(reader, None)  # skip the headers
        gonames[onts[0]] = next(reader)

        # biological process
        next(reader, None)  # skip the headers
        goterms[onts[1]] = next(reader)
        next(reader, None)  # skip the headers
        gonames[onts[1]] = next(reader)

        # cellular component
        next(reader, None)  # skip the headers
        goterms[onts[2]] = next(reader)
        next(reader, None)  # skip the headers
        gonames[onts[2]] = next(reader)

        next(reader, None)  # skip the headers
        for row in reader:
            prot, prot_goterms = row[0], row[1:]
            prot2annot[prot] = {ont: [] for ont in onts}
            for i in range(3):
                goterm_indices = [goterms[onts[i]].index(goterm) for goterm in prot_goterms[i].split(',') if goterm != '']
                prot2annot[prot][onts[i]] = np.zeros(len(goterms[onts[i]]), dtype=np.int64)
                prot2annot[prot][onts[i]][goterm_indices] = 1.0
    return prot2annot, goterms, gonames

def load_EC_annot(filename):
    """ Load EC annotations """
    prot2annot = {}
    ec_numbers = []
    with open(filename, mode='r') as tsvfile:
        reader = csv.reader(tsvfile, delimiter='\t')

        # molecular function
        next(reader, None)  # skip the headers
        ec_numbers = next(reader)
        next(reader, None)  # skip the headers
        for row in reader:
            prot, prot_ec_numbers = row[0], row[1]
            ec_indices = [ec_numbers.index(ec_num) for ec_num in prot_ec_numbers.split(',')]
            prot2annot[prot] = np.zeros(len(ec_numbers), dtype=np.int64)
            prot2annot[prot][ec_indices] = 1.0
    return prot2annot, ec_numbers

class GenerateTFRecord(object):
    def __init__(self, prot_list, prot2annot, ec, npz_dir, tfrecord_fn, num_shards=30):
        self.prot_list = prot_list
        self.prot2annot = prot2annot
        self.ec = ec
        self.npz_dir = npz_dir
        self.tfrecord_fn = tfrecord_fn
        self.num_shards = num_shards

        shard_size = len(prot_list)//num_shards
        indices = [(i*(shard_size), (i+1)*(shard_size)) for i in range(0, num_shards)]
        indices[-1] = (indices[-1][0], len(prot_list))
        self.indices = indices

    def _bytes_feature(self, value):
        """Returns a bytes_list from a string / byte."""
        if not isinstance(value, list):
            value = [value]
        return tf.train.Feature(bytes_list=tf.train.BytesList(value=value))

    def _float_feature(self, value):
        """Returns a float_list from a float / double."""
        return tf.train.Feature(float_list=tf.train.FloatList(value=value))

    def _dtype_feature(self):
        return lambda array: tf.train.Feature(int64_list=tf.train.Int64List(value=array))

    def _int_feature(self, value):
        return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))

    def _serialize_example(self, prot_id, sequence, ca_dist_matrix, cb_dist_matrix):
        labels = self._dtype_feature()

        d_feature = {}
        # load appropriate tf.train.Featur class depending on dtype
        d_feature['prot_id'] = self._bytes_feature(prot_id.encode())
        d_feature['seq_1hot'] = self._float_feature(seq2onehot(sequence).reshape(-1))
        d_feature['L'] = self._int_feature(len(sequence))

        if self.ec:
            d_feature['ec_labels'] = labels(self.prot2annot[prot_id])
        else:
            d_feature['mf_labels'] = labels(self.prot2annot[prot_id]['molecular_function'])
            d_feature['bp_labels'] = labels(self.prot2annot[prot_id]['biological_process'])
            d_feature['cc_labels'] = labels(self.prot2annot[prot_id]['cellular_component'])

        d_feature['ca_dist_matrix'] = self._float_feature(ca_dist_matrix.reshape(-1))
        d_feature['cb_dist_matrix'] = self._float_feature(cb_dist_matrix.reshape(-1))

        example = tf.train.Example(features=tf.train.Features(feature=d_feature))
        return example.SerializeToString()

    def _convert_numpy_folder(self, idx):
        tfrecord_fn = self.tfrecord_fn + '_%0.2d-of-%0.2d.tfrecords' % (idx, self.num_shards)
        # writer = tf.python_io.TFRecordWriter(tfrecord_fn)
        writer = tf.io.TFRecordWriter(tfrecord_fn)
        print ("### Serializing %d examples into %s" % (len(self.prot_list), tfrecord_fn))

        tmp_prot_list = self.prot_list[self.indices[idx][0]:self.indices[idx][1]]

        for i, prot in enumerate(tmp_prot_list):
            if i % 500 == 0:
                print ("### Iter = %d/%d" % (i, len(tmp_prot_list)))
            pdb_file = self.npz_dir + '/' + prot + '.npz'
            if os.path.isfile(pdb_file):
                cmap = np.load(pdb_file)
                sequence = str(cmap['seqres'])
                ca_dist_matrix = cmap['C_alpha']
                cb_dist_matrix = cmap['C_beta']

                example = self._serialize_example(prot, sequence, ca_dist_matrix, cb_dist_matrix)
                writer.write(example)
            else:
                print (pdb_file)
        print ("Writing {} done!".format(tfrecord_fn))

    def run(self, num_threads):
        pool = multiprocessing.Pool(processes=num_threads)
        shards = [idx for idx in range(0, self.num_shards)]
        pool.map(self._convert_numpy_folder, shards)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('-annot', type=str, default='./data/nrPDB-GO_2020.06.18_annot.tsv', help="Input file (*.tsv) with preprocessed annotations.")
    parser.add_argument('-ec', help="Use EC annotations.", action="store_true")
    parser.add_argument('-prot_list', type=str, default='./data/nrPDB-GO_2019.06.18_train.txt',
                        help="Input file (*.txt) with a set of protein IDs with distMAps in npz_dir.")
    parser.add_argument('-npz_dir', type=str, default='./data/annot_pdb_chains_npz/',
                        help="Directory with distance maps saved in *.npz format to be loaded.")
    parser.add_argument('-num_threads', type=int, default=20, help="Number of threads (CPUs) to use in the computation.")
    parser.add_argument('-num_shards', type=int, default=20, help="Number of tfrecord files per protein set.")
    parser.add_argument('-tfr_prefix', type=str, default='/mnt/ceph/users/vgligorijevic/ContactMaps/TFRecords/PDB_GO_train',
                        help="Directory with tfrecord files for model training.")
    args = parser.parse_args()

    prot_list = load_list(args.prot_list)
    if args.ec:
        prot2annot, _ = load_EC_annot(args.annot)
    else:
        prot2annot, _, _ = load_GO_annot(args.annot)

    tfr = GenerateTFRecord(prot_list, prot2annot, args.ec, args.npz_dir, args.tfr_prefix, num_shards=args.num_shards)
    tfr.run(num_threads=args.num_threads)

max_stars_repo_name: SoliareofAstora/DeepFRI

max_issues_repo_name: kadinaj/DeepFRI

max_forks_repo_name: kadinaj/DeepFRI


text:
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""
Export mindir.
"""
import json
from os.path import join
import argparse
from warnings import warn
from hparams import hparams, hparams_debug_string
from mindspore import context, Tensor
from mindspore.train.serialization import load_checkpoint, load_param_into_net, export
from wavenet_vocoder import WaveNet
from wavenet_vocoder.util import is_mulaw_quantize, is_scalar_input
import numpy as np
from src.loss import PredictNet

parser = argparse.ArgumentParser(description='TTS training')
parser.add_argument('--preset', type=str, default='', help='Path of preset parameters (json).')
parser.add_argument('--checkpoint_dir', type=str, default='./checkpoints_test',
                    help='Directory where to save model checkpoints [default: checkpoints].')
parser.add_argument('--speaker_id', type=str, default='',
                    help=' Use specific speaker of data in case for multi-speaker datasets.')
parser.add_argument('--pretrain_ckpt', type=str, default='', help='Pretrained checkpoint path')
parser.add_argument('--platform', type=str, default='GPU', help='Running device')
args = parser.parse_args()

if __name__ == '__main__':

    target = args.platform
    context.set_context(mode=context.GRAPH_MODE, device_target=target, save_graphs=False)

    speaker_id = int(args.speaker_id) if args.speaker_id != '' else None
    if args.preset is not None:
        with open(args.preset) as f:
            hparams.parse_json(f.read())

    assert hparams.name == "wavenet_vocoder"
    print(hparams_debug_string())

    fs = hparams.sample_rate
    output_json_path = join(args.checkpoint_dir, "hparams.json")
    with open(output_json_path, "w") as f:
        json.dump(hparams.values(), f, indent=2)

    if is_mulaw_quantize(hparams.input_type):
        if hparams.out_channels != hparams.quantize_channels:
            raise RuntimeError(
                "out_channels must equal to quantize_chennels if input_type is 'mulaw-quantize'")
    if hparams.upsample_conditional_features and hparams.cin_channels < 0:
        s = "Upsample conv layers were specified while local conditioning disabled. "
        s += "Notice that upsample conv layers will never be used."
        warn(s)

    upsample_params = hparams.upsample_params
    upsample_params["cin_channels"] = hparams.cin_channels
    upsample_params["cin_pad"] = hparams.cin_pad
    model = WaveNet(
        out_channels=hparams.out_channels,
        layers=hparams.layers,
        stacks=hparams.stacks,
        residual_channels=hparams.residual_channels,
        gate_channels=hparams.gate_channels,
        skip_out_channels=hparams.skip_out_channels,
        cin_channels=hparams.cin_channels,
        gin_channels=hparams.gin_channels,
        n_speakers=hparams.n_speakers,
        dropout=hparams.dropout,
        kernel_size=hparams.kernel_size,
        cin_pad=hparams.cin_pad,
        upsample_conditional_features=hparams.upsample_conditional_features,
        upsample_params=upsample_params,
        scalar_input=is_scalar_input(hparams.input_type),
        output_distribution=hparams.output_distribution,
    )

    Net = PredictNet(model)
    Net.set_train(False)
    if target != "Ascend":
        receptive_field = model.receptive_field
        print("Receptive field (samples / ms): {} / {}".format(receptive_field, receptive_field / fs * 1000))
    param_dict = load_checkpoint(args.pretrain_ckpt)
    load_param_into_net(model, param_dict)
    print('Successfully loading the pre-trained model')

    if is_mulaw_quantize(hparams.input_type):
        x = np.array(np.random.random((2, 256, 10240)), dtype=np.float32)
        c = np.array(np.random.random((2, 80, 44)), dtype=np.float32)
    else:
        x = np.array(np.random.random((2, 1, 4096)), dtype=np.float32)
        c = np.array(np.random.random((2, 80, 20)), dtype=np.float32)
    g = np.array([0, 0], dtype=np.int64)

    export(Net, Tensor(x), Tensor(c), Tensor(g), file_name="WaveNet", file_format='MINDIR')

max_stars_repo_name: leelige/mindspore

max_issues_repo_name: leelige/mindspore

max_forks_repo_name: leelige/mindspore


text:
import numpy as np

a = np.array([[1,2,3], [4,5,6], [7,8,9], [10, 11, 12]])
print(a)

nums = [1, 2, 3, 4]
print(nums[:2])


text:
import ci_reduce.xmatch.gaia as gaia_xmatch
import astropy.io.fits as fits
import os
import healpy
import numpy as np
from astropy.coordinates import SkyCoord
from astropy import units as u
import matplotlib.pyplot as plt
from astropy import wcs
from .amm_2dhist import amm_2dhist
from scipy.ndimage import gaussian_filter
from .center_contrast import center_contrast
import ci_reduce.common as common
import copy
#import fitsio # for testing

def downselected_star_sample(cat, n_desi_max):
    assert(len(np.unique(cat['camera'])) == 1)

    try:
        _cat = cat[cat['DQ_FLAGS'] == 0]
    except:
        _cat = cat[cat['dq_flags'] == 0]

    try:
        flux = _cat['APER_SUM_BKGSUB_3']
    except:
        flux = _cat['aper_sum_bkgsub_3']

    sind = np.argsort(-1.0*flux)

    n = len(sind)

    result = _cat[sind[0:(min(n_desi_max, n))]]

    return result

def gaia_cat_for_exp(racen, deccen):

    nside = 32
    ipix = healpy.pixelfunc.get_all_neighbours(nside, racen, phi=deccen,
                                               lonlat=True)

    # remove dummy -1 values from ipix

    ipix = ipix[ipix >= 0]

    # probably also want to include the neighbors to the neighbors
    ipix_all = []
    for _ipix in ipix:
        ipix_all.append(healpy.pixelfunc.get_all_neighbours(nside,
                                                            _ipix))

    ipix = np.unique(ipix_all)

    ipix = ipix[ipix >= 0]

    ra_pixcenters, dec_pixcenters = healpy.pixelfunc.pix2ang(nside, ipix,
                                                              lonlat=True)

    gaia = gaia_xmatch.read_gaia_cat(ra_pixcenters, dec_pixcenters)
    return gaia

def kentools_center(catalog, skyra, skydec, extname='GUIDE0', arcmin_max=3.5,
                    gaia=None):

    # cat needs to have fields xcentroid and ycentroid
    # skyra, skydec are the initial guesses of the
    # actual center of the field of view; these need to be within
    # arcmin_max of the true FOV center for this routine to succeed

    #assert(os.path.exists(fname_cat))

    # output needs to include, at a minimum:
    #     xshift_best
    #     yshift_best
    #     contrast
    #     extname
    # probably also want
    #     expid retrieved from the catalog table

    #fname_reduced = fname_cat.replace('_catalog', '_reduced')

    #assert(os.path.exists(fname_reduced))

    #h = fits.getheader(fname_reduced, extname=extname)

    #expid = h['EXPID']

    #cat = fits.getdata(fname_cat)
    cat = copy.deepcopy(catalog)
 # should just entirely rename racen, deccen to skyr, skydec ...
    racen = skyra
    deccen = skydec

    # this apparently happened for the CI in some cases...
    if isinstance(racen, str) or isinstance(deccen, str):
        return None

    if gaia is None:
        gaia = gaia_cat_for_exp(racen, deccen)

    g = SkyCoord(gaia['ra']*u.deg, gaia['dec']*u.deg)
    c = SkyCoord(racen*u.deg, deccen*u.deg)

    dangle = g.separation(c)

    gaia = gaia[dangle.degree < 2]
    g = SkyCoord(gaia['ra']*u.deg, gaia['dec']*u.deg)

    cat = cat[cat['camera'] == extname]

    assert(len(cat) > 0)

    n_desi_max = 150

    if len(cat) > n_desi_max:
        cat = downselected_star_sample(cat, n_desi_max)

    par = common.ci_misc_params()
    fname_wcs_templates = os.environ['GFA_REDUCE_ETC'] + '/' + par['headers_dummy_filename']

    assert(os.path.exists(fname_wcs_templates))

    h = fits.getheader(fname_wcs_templates, extname=extname)

    assert(h['EXTNAME'] == extname)

    astrom = wcs.WCS(h)
    astrom.wcs.crval = [racen, deccen]

    x_gaia_guess, y_gaia_guess = astrom.wcs_world2pix(gaia['ra'], gaia['dec'],
                                                      0)
    dx_all = np.array([], dtype='float64')
    dy_all = np.array([], dtype='float64')

    cat = cat[np.argsort(cat['dec'])] # not really necessary

    for i in range(len(cat)):
        print(i+1, ' of ', len(cat))
        c = SkyCoord(cat[i]['ra']*u.deg, cat[i]['dec']*u.deg)
        dangle = c.separation(g)
        w = (np.where(dangle.arcminute < arcmin_max))[0]
        print(len(w), len(g), len(gaia))
        if len(w) == 0:
            continue
        dx = cat[i]['xcentroid'] - x_gaia_guess[w]
        dy = cat[i]['ycentroid'] - y_gaia_guess[w]
        dx_all = np.concatenate((dx_all, dx))
        dy_all = np.concatenate((dy_all, dy))

    assert(len(dx_all) == len(dy_all))
    #print(np.min(dy_all), np.max(dy_all))

    axlim = max(np.round(arcmin_max*60.0/0.214), 1000.0)

    #print(axlim)

    dx = 1.0
    dy = 1.0
    nx = ny = 2*axlim
    counts, x_edges_left, y_edges_left = amm_2dhist(-1.0*axlim, -1.0*axlim,
                                                    nx, ny, dx, dy,
                                                    dx_all, dy_all)

    counts = counts.astype(float)

    # 7.5 value is tailored to the CI -- revisit for GFAs !!
    fwhm_pix = 4.7
    sigma_pix = fwhm_pix/(2*np.sqrt(2*np.log(2)))
    smth = gaussian_filter(counts, sigma_pix, mode='constant')

    counts_shape = counts.shape
    sidelen = counts_shape[0]
    #plt.imshow(smth, cmap='gray_r')
    #plt.show()
    indmax = np.argmax(smth)

    indx = (indmax // sidelen).astype(int)
    indy = (indmax % sidelen).astype(int)

    #print(indmax, indx, indy, counts_shape, x_edges_left[indx],
    #      y_edges_left[indy])
    #print(x_edges_left[indx], y_edges_left[indy], ' ~~~~~~~~~~~~~~~~')

    xshift_best = x_edges_left[indx] + 0.5*dx
    yshift_best = y_edges_left[indy] + 0.5*dy

# assumes dx = dy = 1 !!
    ycen_grid, xcen_grid = np.meshgrid(x_edges_left[0:(len(x_edges_left)-1)], y_edges_left[0:(len(y_edges_left)-1)])

    d = np.sqrt(np.power(xcen_grid - xshift_best, 2) + np.power(ycen_grid - yshift_best, 2))

    #fitsio.write('/global/cscratch1/sd/ameisner/smth.fits', smth)
    #fitsio.write('/global/cscratch1/sd/ameisner/d.fits', d)

    r_max = 4*4.7 # roughly 4 asec radius for the CI

    sind = np.argsort(np.ravel(smth))

    val_90 = np.ravel(smth)[sind[int(np.round(0.925*len(sind)))]]

    high = [(d < r_max) & ((smth == np.max(smth)) | (smth > val_90))]

    assert(np.sum(high) > 0)

    print(xshift_best, yshift_best)

    wt = np.sum(high*smth)
    xshift_best = np.sum(high*xcen_grid*smth)/wt
    yshift_best = np.sum(high*ycen_grid*smth)/wt

    print(xshift_best, yshift_best)
    contrast = center_contrast(smth)

    #print(xcen_grid.shape)
    #print(counts.shape, smth.shape)

    result = {'xshift_best': xshift_best,
              'yshift_best': yshift_best,
              'contrast': contrast,
              'extname': extname,
              'astr_guess': astrom}

    return result

def _test(extname='CIC', arcmin_max=3.5):
    fname_cat = '/project/projectdirs/desi/users/ameisner/CI/reduced/v0001/20190403/ci-00003697/ci-00003697_catalog.fits'

    gaia = kentools_center(fname_cat, extname=extname, arcmin_max=arcmin_max)

    return gaia

def _test_gfa():
    fname_cat = '/project/projectdirs/desi/users/ameisner/GFA/reduced/v0000/20191116/00028537/gfa-00028537_catalog.fits'

    result = kentools_center(fname_cat, extname='GUIDE2')

    return result

def __test_gfa():
    fname_cat = '/project/projectdirs/desi/users/ameisner/GFA/reduced/v0000/20191116/00028537/gfa-00028537_catalog.fits'

    fname_raw = '/project/projectdirs/desi/spectro/data/20191116/00028537/gfa-00028537.fits.fz'

    h = fits.getheader(fname_raw, extname='GFA')

    cat = fits.getdata(fname_cat)

    result = kentools_center(cat, h['SKYRA'], h['SKYDEC'], extname='GUIDE2')

    return result

def _loop(indstart, nproc):
    tab = fits.getdata('/global/homes/a/ameisner/ci/pro/ci_quality_summary.fits')
    # tab = tab[tab['GOOD'] == 1]

    # shuffle
    np.random.seed(seed=99)
    sind = np.argsort(np.random.rand(len(tab)))

    tab = tab[sind]

    results = []

    indend = min(indstart + nproc, len(tab))
    for i in range(indstart, indend):
        t = tab[i]
        fname_cat = t['FNAME'].replace('_psf-a', '_catalog')
        extname = t['EXTNAME']
        print('WORKING ON: ', i, '   ', fname_cat, '   ', extname)
        result = kentools_center(fname_cat, extname=extname, arcmin_max=3.5)
        results.append(result)

    # then write out the results to a pickle file
    # need to construct the output file name first
    outname = 'center_' + str(indstart).zfill(5) + '_' + str(indend-1).zfill(5) + '.pkl'
    assert(not os.path.exists(outname))
    import pickle
    pickle.dump(results, open(outname, 'wb'))

reason: import numpy,from scipy,import astropy,from astropy


text:
# STUDENTS: DON'T RUN THIS FILE. WE HAVE CREATED CLOZE FOR YOU.

import collections
import numpy as np
import numpy.random as random
import sys

BLANK = "<blank>"  # special token for blanks
SHORT_THRES = 10  # length threshold for a sentence to be considered short and thus be removed
LONG_THRES = 30  # length threshold for a sentence to be considered long and thus has max number of blanks
MAX_BLANK_FREQENCY_RANK = 500  # maximum frequency rank of the word type to be used to create blank
STOP_LIST = [".", ",", "(", ")", "!", "#", "%", "@-@", """, "-", ";", ":"]

if len(sys.argv) < 4:
  sys.stderr.write("usage: python create_cloze.py [test_file] [train_file] [max_blanks_per_sentence]\n")
  sys.exit(1)
else:
  test_file = open(sys.argv[1], 'r')
  train_file = open(sys.argv[2], 'r')
  cloze_question = open(sys.argv[1] + ".cloze", 'w')
  cloze_answer = open(sys.argv[1] + ".answer", 'w')
  max_blanks_per_sent = int(sys.argv[3])

# first pass: collect count
token_cnt = collections.Counter()
for line in train_file:
  tokens = line.split()
  for token in tokens:
    token_cnt[token] += 1

wordlist = token_cnt.most_common(n=MAX_BLANK_FREQENCY_RANK)
wordlist = [pair[0] for pair in wordlist]

for line in test_file:
  tokens = line.strip().split()
  if len(tokens) < SHORT_THRES:
    continue
  candidate_indexes = []
  for idx, token in enumerate(tokens):
    if token in wordlist and token not in STOP_LIST:
      candidate_indexes.append(idx)

  blank_indexes = []
  for blank in range(int(min(np.ceil(max_blanks_per_sent * len(candidate_indexes) / LONG_THRES), 3))):
    idx = random.randint(len(candidate_indexes))
    blank_indexes.append(candidate_indexes[idx])
    del candidate_indexes[idx]
  blank_indexes = sorted(blank_indexes)

  answers = []
  for idx in blank_indexes:
    answers.append(tokens[idx])
    tokens[idx] = BLANK
  cloze_question.write(" ".join(tokens) + "\n")
  cloze_answer.write(" ".join(answers) + "\n")

test_file.close()
cloze_question.close()
cloze_answer.close()


text:
"""This module implements a class that..."""
from __future__ import print_function, unicode_literals
from __future__ import division

import logging

import numpy
from skimage.measure import label
from typing import Dict, List, Tuple

from muscima.inference_engine_constants import InferenceEngineConstants as _CONST

__version__ = "0.0.1"
__author__ = "Jan Hajic jr."

##############################################################################

def connected_components2bboxes(labels):
    # type: (List[List[int]]) -> Dict[int,Tuple[int, int, int, int]]
    """Returns a dictionary of bounding boxes (upper left c., lower right c.)
    for each label.

    >>> labels = [[0, 0, 1, 1], [2, 0, 0, 1], [2, 0, 0, 0], [0, 0, 3, 3]]
    >>> bboxes = connected_components2bboxes(labels)
    >>> bboxes[0]
    [0, 0, 4, 4]
    >>> bboxes[1]
    [0, 2, 2, 4]
    >>> bboxes[2]
    [1, 0, 3, 1]
    >>> bboxes[3]
    [3, 2, 4, 4]

    :param labels: The output of cv2.connectedComponents().

    :returns: A dict indexed by labels. The values are quadruplets
        (xmin, ymin, xmax, ymax) so that the component with the given label
        lies exactly within labels[xmin:xmax, ymin:ymax].
    """
    bboxes = {}
    for x, row in enumerate(labels):  # type: int, List[int]
        for y, l in enumerate(row):  # type: int, int
            if l not in bboxes:
                bboxes[l] = [x, y, x + 1, y + 1]
            else:
                box = bboxes[l]
                if x < box[0]:
                    box[0] = x
                elif x + 1 > box[2]:
                    box[2] = x + 1
                if y < box[1]:
                    box[1] = y
                elif y + 1 > box[3]:
                    box[3] = y + 1
    return bboxes

def compute_connected_components(image):
    # type: (numpy.ndarray) -> (int, numpy.ndarray, Dict[int,Tuple[int, int, int, int]])
    labels = label(image, background=0)
    cc = int(labels.max())
    bboxes = connected_components2bboxes(labels)
    return cc, labels, bboxes

def resolve_notehead_wrt_staffline(notehead, staffline_or_ledger_line):
    from muscima.cropobject import CropObject
    # type: (CropObject, CropObject) -> int
    """Resolves the relative vertical position of the notehead with respect
    to the given staff_line or ledger_line object. Returns -1 if notehead
    is *below* staffline, 0 if notehead is *on* staffline, and 1 if notehead
    is *above* staffline."""
    ll = staffline_or_ledger_line

    # Determining whether the notehead is on a ledger
    # line or in the adjacent temp staffspace.
    # This uses a magic number, ON_STAFFLINE_RATIO_THRESHOLD.
    output_position = 0

    ### DEBUG!!!
    dtop, dbottom = 1, 1

    # Weird situation with notehead vertically *inside* bbox
    # of ledger line (could happen with slanted LLs and very small
    # noteheads).
    if ll.top <= notehead.top <= notehead.bottom <= ll.bottom:
        output_position = 0

    # No vertical overlap between LL and notehead
    elif ll.top > notehead.bottom:
        output_position = 1
    elif notehead.top > ll.bottom:
        output_position = -1

    # Complicated situations: overlap
    else:
        # Notehead "around" ledger line.
        if notehead.top < ll.top <= ll.bottom < notehead.bottom:
            dtop = ll.top - notehead.top
            dbottom = notehead.bottom - ll.bottom

            if min(dtop, dbottom) / max(dtop, dbottom) \
                    < _CONST.ON_STAFFLINE_RATIO_TRHESHOLD:
                if dtop > dbottom:
                    output_position = 1
                else:
                    output_position = -1

        # Notehead interlaced with ledger line, notehead on top
        elif notehead.top < ll.top <= notehead.bottom <= ll.bottom:
            # dtop = closest_ll.top - notehead.top
            # dbottom = max(notehead.bottom - closest_ll.top, 1)
            # if float(dbottom) / float(dtop) \
            #         < InferenceEngineConstants.ON_STAFFLINE_RATIO_TRHESHOLD:
            output_position = 1

        # Notehead interlaced with ledger line, ledger line on top
        elif ll.top <= notehead.top <= ll.bottom < notehead.bottom:
            # dtop = max(closest_ll.bottom - notehead.top, 1)
            # dbottom = notehead.bottom - closest_ll.bottom
            # if float(dtop) / float(dbottom) \
            #         < InferenceEngineConstants.ON_STAFFLINE_RATIO_TRHESHOLD:
            output_position = -1

        else:
            logging.warning('Strange notehead {0} vs. ledger line {1}'
                            ' situation: bbox notehead {2}, LL {3}.'
                            ' Note that the output position is unusable;'
                            ' please re-do this attachment manually.'
                            ''.format(notehead.uid, ll.uid,
                                      notehead.bounding_box,
                                      ll.bounding_box))
    return output_position

def is_notehead_on_line(notehead, line_obj):
    from muscima.cropobject import CropObject
    # type: (CropObject, CropObject) -> bool
    """Check whether given notehead is positioned on the line object."""
    if line_obj.clsname not in _CONST.STAFFLINE_LIKE_CROPOBJECT_CLSNAMES:
        raise ValueError('Cannot resolve relative position of notehead'
                         ' {0} to non-staffline-like object {1}'
                         ''.format(notehead.uid, line_obj.uid))

    position = resolve_notehead_wrt_staffline(notehead, line_obj)
    return position == 0

max_stars_repo_name: hajicj/muscima

max_issues_repo_name: hajicj/muscima

max_forks_repo_name: hajicj/muscima


text:
import os

import networkx as nx
from peartree.paths import get_representative_feed, load_feed_as_graph
from peartree.convert import convert_to_digraph

def fixture(filename):
    return os.path.join(os.path.dirname(__file__), 'fixtures', filename)

def test_convert_multidigraph_to_digraph():
    path = fixture('samtrans-2017-11-28.zip')
    feed = get_representative_feed(path)

    # Shorter amount of time to speed up the test
    start = 7 * 60 * 60
    end = 8 * 60 * 60
    Gmdg = load_feed_as_graph(feed, start, end, name='foobar')

    # Run conversaion operation
    Gdg = convert_to_digraph(Gmdg)

    assert isinstance(Gdg, nx.DiGraph)
    assert len(Gdg.edges()) == len(Gmdg.edges())
    assert len(Gdg.nodes()) == len(Gmdg.nodes())

max_stars_repo_name: yiyange/peartree

max_issues_repo_name: yiyange/peartree

max_forks_repo_name: yiyange/peartree

reason: import networkx


text:
import numpy as np

from PIL import Image
from os.path import join
from tqdm.auto import tqdm

import torch
import torchvision
import random

from data.matterport_dataset import MatterportDataset
from data.abstract_dataset import Abstract_DataModule

class Matterport_Single_Scene_DataModule(Abstract_DataModule):
    def __init__(self,
                 root_path: str,
                 batch_size: int = 32,
                 num_workers: int = 1,
                 transform_rgb=None,
                 transform_label=None,
                 transform_uv=None,
                 resize_size=(256, 256),
                 pyramid_levels=5,
                 min_pyramid_depth=0.25,
                 min_pyramid_height=32,
                 scene=None,
                 min_images=1000,
                 max_images=-1,
                 shuffle: bool = False,
                 sampler_mode: str = "random",
                 index_repeat: int = 1,
                 split: list = [0.8, 0.2],
                 split_mode: str = "skip",
                 region_index=0,
                 verbose: bool = False):

        Abstract_DataModule.__init__(self,
                                     dataset=Matterport_Single_House_Dataset,
                                     root_path=join(root_path, "v1/scans"),
                                     transform_rgb=transform_rgb,
                                     transform_label=transform_label,
                                     transform_uv=transform_uv,
                                     resize_size=resize_size,
                                     use_scene_filter=True,
                                     scene=scene,
                                     min_images=min_images,
                                     max_images=max_images,
                                     verbose=verbose,
                                     batch_size=batch_size,
                                     num_workers=num_workers,
                                     shuffle=shuffle,
                                     sampler_mode=sampler_mode,
                                     index_repeat=index_repeat,
                                     split=split,
                                     split_mode=split_mode)

        self.min_pyramid_depth = min_pyramid_depth
        self.min_pyramid_height = min_pyramid_height
        self.pyramid_levels = pyramid_levels
        self.region_index = region_index

    def after_create_dataset(self, d, root_path):
        if isinstance(d, MatterportDataset):
            d.set_uv_pyramid_mode(self.min_pyramid_depth, self.min_pyramid_height)
            d.set_pyramid_levels(self.pyramid_levels)
            d.region_index = self.region_index
            d.create_data()

class Matterport_Single_House_Dataset(MatterportDataset):

    def __init__(self,
                 root_path,
                 scene=None,
                 min_images=1000,
                 max_images=-1,
                 transform_rgb=None,
                 transform_label=None,
                 transform_uv=None,
                 resize_size=(256, 256),
                 pyramid_levels=5,
                 min_pyramid_depth=0.25,
                 min_pyramid_height=32,
                 region_index=0,
                 verbose=False):

        self.input_scene = scene
        self.min_images = min_images
        self.max_images = max_images

        MatterportDataset.__init__(self,
                                root_path=root_path,
                                transform_rgb=transform_rgb,
                                transform_label=transform_label,
                                transform_uv=transform_uv,
                                resize_size=resize_size,
                                min_pyramid_depth=min_pyramid_depth,
                                min_pyramid_height=min_pyramid_height,
                                pyramid_levels=pyramid_levels,
                                region_index=region_index,
                                verbose=verbose)

    def create_data(self):
        self.scene_dict = self.parse_scenes()[-1]
        self.rgb_images, self.extrinsics, self.intrinsics, self.intrinsic_image_sizes, self.depth_images, self.uv_maps, self.angle_maps, self.size, self.scene = self.get_scene(self.input_scene, self.min_images, self.max_images)

        print(f"Using scene: {self.scene}. Input was: {self.input_scene}")

        # use this to finally set the self.uvs_npy, self.angle_npy and self.rendered_depth attributes correctly to the state of the chosen scene
        self.get_depth(join(self.root_path, self.scene))
        self.get_uvs(join(self.root_path, self.scene))
        self.get_angles(join(self.root_path, self.scene))

    def get_scene(self, scene, min_images, max_images):
        items = self.get_scene_items(scene)
        if self.in_range(min_images, max_images, items):
            return self.parse_scene(scene)
        else:
            return self.find_house(min_images, max_images)

    def get_scene_items(self, scene):
        if scene is None:
            return None
        elif scene not in self.scene_dict:
            return 0
        else:
            return self.scene_dict[scene]["items"]

    def in_range(self, min, max, value):
        return (value is not None) and (min == -1 or value >= min) and (max == -1 or value <= max)

    def parse_scene(self, scene):
        h = self.scene_dict[scene]
        return h["color"], h["extrinsics"], h["intrinsics"], h["image_size"], h["depth"], h["uv_map"], h["angle_map"], len(h["color"]), scene

    def find_house(self, min_images, max_images):
        max = -1
        min = -1
        scenes = [s for s in self.scene_dict.keys()]
        random.shuffle(scenes)
        if self.verbose:
            scenes = tqdm(scenes)
            print(f"Searching for a house with more than {min_images} images")
        for h in scenes:
            size = self.get_scene_items(h)
            if max == -1 or size > max:
                max = size
            if min == -1 or size < min:
                min = size
            if self.in_range(min_images, max_images, size):
                if self.verbose:
                    print(f"Using scene '{h}' which has {size} images")
                return self.parse_scene(h)
        raise ValueError(f"No scene found with {min_images} <= i <= {max_images} images. Min/Max available: {min}/{max}")

max_stars_repo_name: lukasHoel/stylemesh

max_issues_repo_name: lukasHoel/stylemesh

max_forks_repo_name: lukasHoel/stylemesh


text:
import os
import pandas as pd
import numpy as np
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error
import matplotlib.pyplot as plt

a=os.path.join(os.path.dirname(__file__),"train.csv")
#Reading file and removing missing data for all columns
data=pd.read_csv(os.path.join(os.path.dirname(__file__),"train.csv"))
data.dropna(axis=0)

features=["LotArea","OverallQual","OverallCond","YearBuilt"]
y=data.SalePrice.values
x=data[features].values

trainx,testx,trainy,testy=train_test_split(x,y,random_state=0)

regress=LinearRegression()
regress.fit(trainx,trainy)
predicted=regress.predict(testx)

error=mean_squared_error(testy,predicted)
print("Error",error)
print("t0:",regress.intercept_)
print("coeff",regress.coef_)

final_prediction=regress.intercept_+(x.dot(np.array(regress.coef_)))
with open("pre_result.txt","w") as f:
    for k,m in zip(final_prediction,y.reshape(-1,1)):
        f.write(str(int(k))+" "+str(m))
        f.write("\n")
    f.close()

myin=regress.predict(np.array([8246,5,8,1968]).reshape(1,4))
print("Prediction",myin)
print("--------------------------------------------------------------")
plt.title("House Price Prediction")
plt.xlabel("Year Built")
plt.ylabel("Price")
plt.scatter(x[:,3],y,label="Values",color="blue")
plt.plot([min(x[:,3]),max(x[:,3])],[min(final_prediction),max(final_prediction)],label="Regression Line",color="red")
plt.show()

max_stars_repo_name: iSriBalaji/MachineLearning

max_issues_repo_name: iSriBalaji/MachineLearning

max_forks_repo_name: iSriBalaji/MachineLearning


text:
from flask import Flask,render_template,request
import re
import jieba
from sklearn.decomposition import PCA
from typing import List
from sklearn.metrics.pairwise import cosine_similarity
from operator import itemgetter
from gensim import models
import numpy as np
import pandas as pd

app = Flask(__name__)
# 结巴分词
def cut(string):
    return list(jieba.cut(string))
# 拆分成列表
def sentenceslist(article):
    setences = re.sub('([。！？\?])([^”’])', r"\1 \2", article)  # 单字符断句符
    setences = re.sub('(\.{6})([^”’])', r"\1 \2", setences)  # 英文省略号
    setences = re.sub('(\…{2})([^”’])', r"\1  \2", setences)  # 中文省略号
    setences = re.sub('([。！？\?][”’])([^，。！？\?])', r'\1 \2', setences)
    sentences_list = setences.split(' ')
    sentence_list_list = []
    for sentence in sentences_list:
        sentence = cut(sentence)
        sentence_list_list.append(sentence)
    return sentences_list,sentence_list_list
# PCA与句子向量
def sentence_to_vec(setencelist, a = 0.0001):
    model = models.Word2Vec.load("./model/wiki_corpus.model")
    sentence_set = []
    j = 0
    for sentence in setencelist:
        vs = np.zeros((250,))  # add all word2vec values into one vector for the sentence
        sentence_length = len(sentence)
        for word in sentence:
            if word in model:
                w_v = model[word]
                p_w = model.wv.vocab[word].count
                a_value = a / (a + p_w/459358208)  # smooth inverse frequency, SIF
                vs = np.add(vs, np.multiply(a_value, w_v))
            else:
                continue
        vs = np.divide(vs, sentence_length)
        sentence_set.append(vs)  # add to our existing re-calculated set of sentences
#         print(vs)
    # calculate PCA of this sentence set
#     sentence_set = np.mat
# rix(sentence_set).astype(np.float)
    pca = PCA()
    pca.fit(np.array(sentence_set))
    u = pca.components_[0]  # the PCA vector
    u = np.multiply(u, np.transpose(u))  # u x uT

    # pad the vector?  (occurs if we have less sentences than embeddings_size)
    if len(u) < 250:
        for i in range(250 - len(u)):
            u = np.append(u, 0)  # add needed extension for multiplication below

    # resulting sentence vectors, vs = vs -u x uT x vs
    sentence_vecs = []
    all_vs = np.zeros((250,))
    for vs in sentence_set:
        sub = np.multiply(u,vs)
        fin_vs = np.subtract(vs, sub)
        sentence_vecs.append(fin_vs)
        all_vs = np.add(all_vs,fin_vs)
    sentence_vecs.append(all_vs)
    return sentence_vecs
# 生成摘要
def sentences_sort(sentences_list,sentence_vecs):
    score_dict = {}
    for i in range(len(sentence_vecs)-1):
        vs  = sentence_vecs[i]
        all_vs = sentence_vecs[-1]
        a = []
        a.append(vs)
        a.append(all_vs)
        sim = cosine_similarity(a)
        score = sim[1,0]
        score_dict[i] = score
    score_list_sort = sorted(score_dict.items(),key=itemgetter(1),reverse=True)
    score_list_cut = score_list_sort[:10]
    sort_list = []
    for i in score_list_cut:
        sort_list += [i[0]]
    sort_list.sort()
    sentence_str = ''
    for i in sort_list:
        print(sentences_list[i])
        sentence_str += sentences_list[i]
    return sentence_str

@app.route("/")
def index():
    return render_template('zy.html')

@app.route("/zy",methods=["POST","GET"])
def zy():
    if request.method == 'GET':
        return render_template('zy.html')

    if request.method == 'POST':
        article = request.form.get('article')
        sentences_list, sentencelist = sentenceslist(article)
        sentence_vecs = sentence_to_vec(sentencelist)
        sentence_sort = sentences_sort(sentences_list, sentence_vecs)
        return render_template('zy.html',z_y = sentence_sort)

if __name__ == '__main__':

    app.run(debug=True)


text:
import os
import openpyxl
import copy
import shutil
from openpyxl.drawing.image import Image
import zipfile
import matplotlib.pyplot as plt
import numpy as np

path_bath = "库存感知测试结果汇总_最终"
path_src = "C:/Users/liang-jiashun/Desktop" + "/" + path_bath
path_sample = "C:/Users/liang-jiashun/Desktop/Excel模板"
path_sample_chart = path_sample + "/" + "chart.xlsx"
stdtemp = []

def main():
    files = os.listdir(path_src)
    for file in files:
        if not os.path.isdir(path_src + "/" + file):
            # print(file)
            result, data1, data2, data3, name = get_data(path_src + "/" + file)
            # write_excel(result, file, data1, data2, data3, name)
            # print(result)
    print(stdtemp)
    plt.figure()
    for i in stdtemp:
        plt.scatter(range(3), i)
    plt.show()
    # add_image(path_src, path_src + "/test/" + "image")

def write_excel(result, file, data1, data2, data3, name):
    path_chart = path_src + "/test/image/" + "chart_" + file
    shutil.copy(path_sample_chart, path_chart)
    wb = openpyxl.load_workbook(filename=path_chart)
    ws = wb.worksheets[1]

    wb.worksheets[0].cell(row=2, column=2).value = name
    for i in range(len(data1)):
        wb.worksheets[2].cell(row=2+i, column=1).value = data1[i]
        wb.worksheets[2].cell(row=2 + i, column=2).value = data2[i]
        wb.worksheets[2].cell(row=2 + i, column=3).value = data3[i]

    for i in range(len(result[0])):
        ws.cell(row=2+i, column=1).value = len(result[0]) - i
    for i, temp in enumerate(result):
        for ii, data in enumerate(temp):
            ws.cell(row=2 + ii, column=2+i).value = data
    wb.save(filename=path_chart)

def get_data(path):
    wb = openpyxl.load_workbook(filename=path, data_only=True)
    ws = wb.worksheets[1]
    y_start = 4
    x_start = 13
    result = []
    data1 = []
    data2 = []
    data3 = []
    temp = []
    for i in range(1000):
        if ws.cell(row=i + y_start, column=x_start).value == None:
            break
        aa = int(ws.cell(row=i + y_start, column=x_start).value)
        bb = int(ws.cell(row=i + y_start, column=x_start + 1).value)
        cc = int(ws.cell(row=i + y_start, column=x_start + 2).value)
        data1.append(aa)
        data2.append(bb)
        data3.append(cc)
        if int(aa) == 0:
            result.append(copy.deepcopy(temp))
            temp.clear()
            continue
        temp.append(aa+bb-cc)
    result = data_analysis(result, data1, data2, data3, 2)
    return result, data1, data2, data3, wb.worksheets[0].cell(row=17, column=2).value

def data_analysis(result, data1, data2, data3, flag):
    aa = [0, 0, 0]
    aa[0] = np.std(result[0][0:5])
    aa[1] = np.std(result[0][6:10])
    aa[2] = np.std(result[0][10:])
    for i in range(len(result)):
        aa[0] = (np.std(result[i][0:5])+aa[0])/2
        aa[1] = (np.std(result[i][6:10])+aa[1])/2
        aa[2] = (np.std(result[i][10:])+aa[2])/2
    stdtemp.append(copy.deepcopy(aa))
    print(aa)
    # temp = np.array(result).T
    # print(aa)
    # plt.figure()
    # plt.scatter(range(3), aa)
    # plt.show()
    # exit()

def add_image(path_src, path_file):
    name_file = os.listdir(path_file)

    path_image = "C:/Users/liang-jiashun/Desktop/Excel模板/image"
    shutil.rmtree(path_image)
    os.makedirs(path_image)
    files = os.listdir(path_src)  # 文件遍历
    for file in files:
        if file != "test":
            shutil.copy(path_src+"/"+file, path_image)

    index = 0
    files = os.listdir(path_image)           # 文件遍历
    for file in files:
        if not os.path.isdir(path_image + "/" + file):
            wb = openpyxl.load_workbook(filename=path_file + "/" +name_file[index])
            ws = wb.worksheets[0]

            file_name = os.path.basename(path_image + "/" + file)  # 获取文件名
            new_name = str(file_name.split('.')[0]) + ".zip"  # 新的文件名，命名为：xxx.zip
            dir_path = os.path.dirname(path_image + "/" + file)  # 获取文件所在目录
            new_path = dir_path+"/"+new_name  # 新的文件路径
            os.rename(path_image + "/" + file, new_path)

            file_zip = zipfile.ZipFile(new_path, 'r')
            zipdir = path_image+"/"+str(file.split('.')[0])  # 获取文件所在目录
            for files in file_zip.namelist():
                file_zip.extract(files, zipdir)  # 解压到指定文件目录
            file_zip.close()
            os.remove(new_path)

            pic_dir = 'xl'+"/"+'media'  # excel变成压缩包后，再解压，图片在media目录
            pic_path = path_image+"/"+str(file.split('.')[0])+"/"+pic_dir
            image = pic_path + "/" +"image1.jpeg"

            print(image)
            img1 = Image(image)
            img1.width, img1.height = 73*2, 73*2
            ws.add_image(img1,'A5')
            wb.save(filename=path_file + "/" +name_file[index])
            index += 1
    wb.close()

if __name__ == '__main__':
    main()

max_stars_repo_path: commodity forecast/Excel/CreatChart.py

max_issues_repo_path: commodity forecast/Excel/CreatChart.py

max_forks_repo_path: commodity forecast/Excel/CreatChart.py


text:
import tensorflow as tf
import math
import predict_unlabel
import scipy.io as sio
label = predict_unlabel.predict_models()
import numpy as np
import cnn_indices
data_cnn = cnn_indices.read_data_sets()
import rnn_indices
data_rnn = rnn_indices.read_data_sets()
import julei

import new_class_sample_update

batch_size = 2000
num_input = 17  # MNIST data input (img shape: 28*28)
timesteps = 6
logits_cnn = np.zeros((1, 9), dtype=np.float64)
prob_cnn = np.zeros((1, 9), dtype=np.float64)
logits_rnn = np.zeros((1, 9), dtype=np.float64)
prob_rnn = np.zeros((1, 9), dtype=np.float64)

for index1 in range((data_cnn.unlabel._num_examples // batch_size) + 1):
    batch_x1 = data_cnn.unlabel.next_batch_test(batch_size)
    batch_x1 = batch_x1.transpose((0, 3, 1, 2))
    logits_cnn_med, prob_cnn_med = label.cnn.predict1(batch_x1)
    logits_cnn = np.concatenate((logits_cnn, logits_cnn_med), axis=0)
    prob_cnn = np.concatenate((prob_cnn, prob_cnn_med), axis=0)

for index2 in range((data_rnn.unlabel._num_examples // batch_size) + 1):
    batch_x2 = data_rnn.unlabel.next_batch_test(batch_size)
    if index2 == (data_rnn.unlabel._num_examples // batch_size):
        batch_x2 = batch_x2.reshape(((data_rnn.unlabel._num_examples % batch_size), timesteps, num_input))
    else:
        batch_x2 = batch_x2.reshape((batch_size, timesteps, num_input))
    logits_rnn_med, prob_rnn_med = label.rnn.predict2(batch_x2)
    logits_rnn = np.concatenate((logits_rnn, logits_rnn_med), axis=0)
    prob_rnn = np.concatenate((prob_rnn, prob_rnn_med), axis=0)

prob_cnn = prob_cnn[1:]
prob_rnn = prob_rnn[1:]
logits_rnn = logits_rnn[1:]
logits_cnn = logits_cnn[1:]
# find same predicted label of two networks
label_cnn = np.argmax(prob_cnn, axis=1) + 1
label_rnn = np.argmax(prob_rnn, axis=1) + 1
same_indices = np.where(label_cnn == label_rnn)
big_dif = np.where(label_cnn != label_rnn)

batch_cnn_center = data_cnn.train.next_batch_test(789)
batch_cnn_center = batch_cnn_center.transpose((0, 3, 1, 2))
logits_cnn_center, prob_center_cnn = label.cnn.predict1(batch_cnn_center)
label_center_cnn = np.argmax(prob_center_cnn, axis=1) + 1
center_indices1 = []
juleicenter1 = {}
for i in range(9):
    indices = [j for j, x in enumerate(label_center_cnn.tolist()) if x == i+1]
    np.random.shuffle(indices)
    juleicenter1[i] = indices[:1]
    center_indices1 += juleicenter1[i]
cnn_center_data = logits_cnn_center[center_indices1]
logits_cnn = np.concatenate((cnn_center_data, logits_cnn), axis=0)

batch_rnn_center = data_rnn.train.next_batch_test(789)
batch_rnn_center = batch_rnn_center.reshape((789, timesteps, num_input))
logits_rnn_center, prob_center_rnn = label.rnn.predict2(batch_rnn_center)
label_center_rnn = np.argmax(prob_center_rnn, axis=1) + 1
center_indices2 = []
juleicenter2 = {}
for i in range(9):
    indices = [j for j, x in enumerate(label_center_rnn.tolist()) if x == i+1]
    np.random.shuffle(indices)
    juleicenter2[i] = indices[:1]
    center_indices2 += juleicenter2[i]
rnn_center_data = logits_rnn_center[center_indices2]
logits_rnn = np.concatenate((rnn_center_data, logits_rnn), axis=0)

norm_rnn = np.zeros((logits_rnn.shape[0], logits_rnn.shape[1]), dtype=np.float32)
norm_cnn = np.zeros((logits_cnn.shape[0], logits_cnn.shape[1]), dtype=np.float32)
max_cnn = np.amax(logits_cnn, axis=1)
min_cnn = np.amin(logits_cnn, axis=1)
substract_cnn = [x-y for x, y in zip(max_cnn, min_cnn)]
max_rnn = np.amax(logits_rnn, axis=1)
min_rnn = np.amin(logits_rnn, axis=1)
substract_rnn = [x-y for x, y in zip(max_rnn, min_rnn)]
for i in range(logits_cnn.shape[0]):
    for j in range(logits_cnn.shape[1]):
        norm_cnn[i][j] = (logits_cnn[i][j] - min_cnn[i]) / substract_cnn[i]
        norm_rnn[i][j] = (logits_rnn[i][j] - min_rnn[i]) / substract_rnn[i]
cnn_center = norm_cnn[:9]
cnn_unlabeled = norm_cnn[9:]
rnn_center = norm_rnn[:9]
rnn_unlabeled = norm_rnn[9:]

def update_cnn():
    max_diff_cnn = [x-y for x, y in zip(rnn_unlabeled, cnn_unlabeled)]
    max_dcpe_cnn = np.argmax(max_diff_cnn, axis=1) + 1
    remove_same_cnn = (big_dif[0]).tolist()
    np.random.shuffle(remove_same_cnn)
    same_indexall_cnn = (same_indices[0]).tolist()
    np.random.shuffle(same_indexall_cnn)
    print("kmeans and cnn same samples begin")
    kmeannetcnnsame_indices = julei.predict(cnn_unlabeled[same_indexall_cnn], cnn_center,
                                            label_cnn[same_indexall_cnn], same_indexall_cnn)
    same_cnn_index = new_class_sample_update.update1(kmeannetcnnsame_indices, label_cnn, same_indexall_cnn)
    # same_cnn_index = new_class_sample_update.update(same_indexall_cnn, label_cnn)

    print("kmeans and cnn dcpe samples begin")
    #
    kmeannetcnndcpe_indices = julei.predict(cnn_unlabeled[remove_same_cnn], cnn_center,
                                            max_dcpe_cnn[remove_same_cnn], remove_same_cnn)
    dcpe_cnn_index = new_class_sample_update.update2(kmeannetcnndcpe_indices, max_dcpe_cnn, remove_same_cnn)
    # dcpe_cnn_index = new_class_sample_update.update(remove_same_cnn, max_dcpe_cnn)
    cnn_index_all = same_cnn_index + dcpe_cnn_index
    cnn_label_all = np.concatenate((label_cnn[same_cnn_index], max_dcpe_cnn[dcpe_cnn_index]), axis=0)

    labeled_sets_cnn = np.load('/home/asdf/Documents/juyan/paper/paviac/mdcpe/newmdcpe/data/cnn/labeled_index.npy')
    unlabeled_sets_cnn = np.load('/home/asdf/Documents/juyan/paper/paviac/mdcpe/newmdcpe/data/cnn/unlabeled_index.npy')
    mat_gt_cnn = sio.loadmat("/home/asdf/Documents/juyan/paper/paviac/mdcpe/newmdcpe/data/cnn/Pavia_gt.mat")
    GT_cnn = mat_gt_cnn['pavia_gt']
    cnn_index_bigmap = unlabeled_sets_cnn[cnn_index_all]
    update_labeled_sets_cnn = np.concatenate((labeled_sets_cnn, cnn_index_bigmap), axis=0)
    gt_col_cnn = GT_cnn.reshape((GT_cnn.shape[0]*GT_cnn.shape[1], ))
    cnn_index_bigmap = cnn_index_bigmap.tolist()
    gt_col_cnn[cnn_index_bigmap] = cnn_label_all
    real_gt_cnn = gt_col_cnn.reshape((GT_cnn.shape[0], GT_cnn.shape[1]))
    # unlabeled_sets_cnn = list(set(unlabeled_sets_cnn.tolist()).difference(set(cnn_index_bigmap)))
    np.random.shuffle(update_labeled_sets_cnn)
    np.save('/home/asdf/Documents/juyan/paper/paviac/mdcpe/newmdcpe/data/cnn/unlabeled_index.npy', unlabeled_sets_cnn)
    np.save('/home/asdf/Documents/juyan/paper/paviac/mdcpe/newmdcpe/data/cnn/labeled_index.npy', update_labeled_sets_cnn)
    sio.savemat('/home/asdf/Documents/juyan/paper/paviac/mdcpe/newmdcpe/data/cnn/Pavia_gt.mat', {'pavia_gt': real_gt_cnn})
    return ()

def update_rnn():
    max_diff_rnn = [x - y for x, y in zip(cnn_unlabeled, rnn_unlabeled)]
    max_dcpe_rnn = np.argmax(max_diff_rnn, axis=1) + 1
    remove_same_rnn = (big_dif[0]).tolist()
    np.random.shuffle(remove_same_rnn)
    same_indexall_rnn = (same_indices[0]).tolist()
    np.random.shuffle(same_indexall_rnn)
    print("kmeans and rnn same samples begin")
    kmeannetrnnsame_indices = julei.predict(rnn_unlabeled[same_indexall_rnn], rnn_center,
                                         label_rnn[same_indexall_rnn], same_indexall_rnn)
    same_rnn_index = new_class_sample_update.update1(kmeannetrnnsame_indices, label_rnn, same_indexall_rnn)
    # same_rnn_index = new_class_sample_update.update(same_indexall_rnn, label_rnn)
    print("kmeans and rnn dcpe samples begin")
    kmeannetrnndcpe_indices = julei.predict(rnn_unlabeled[remove_same_rnn], rnn_center,
                                             max_dcpe_rnn[remove_same_rnn], remove_same_rnn)
    dcpe_rnn_index = new_class_sample_update.update2(kmeannetrnndcpe_indices, max_dcpe_rnn, remove_same_rnn)
    # dcpe_rnn_index = new_class_sample_update.update(remove_same_rnn, max_dcpe_rnn)
    rnn_index_all = same_rnn_index + dcpe_rnn_index
    rnn_label_all = np.concatenate((label_rnn[same_rnn_index], max_dcpe_rnn[dcpe_rnn_index]), axis=0)

    labeled_sets_rnn = np.load('/home/asdf/Documents/juyan/paper/paviac/mdcpe/newmdcpe/data/rnn/labeled_index.npy')
    unlabeled_sets_rnn = np.load('/home/asdf/Documents/juyan/paper/paviac/mdcpe/newmdcpe/data/rnn/unlabeled_index.npy')
    mat_gt_rnn = sio.loadmat("/home/asdf/Documents/juyan/paper/paviac/mdcpe/newmdcpe/data/rnn/Pavia_gt.mat")
    GT_rnn = mat_gt_rnn['pavia_gt']
    rnn_index_bigmap = unlabeled_sets_rnn[rnn_index_all]
    update_labeled_sets_rnn = np.concatenate((labeled_sets_rnn, rnn_index_bigmap), axis=0)
    gt_col_rnn = GT_rnn.reshape((GT_rnn.shape[0] * GT_rnn.shape[1],))
    gt_col_rnn[rnn_index_bigmap] = rnn_label_all
    real_gt_rnn = gt_col_rnn.reshape((GT_rnn.shape[0], GT_rnn.shape[1]))
    # unlabeled_sets_rnn = list(set(unlabeled_sets_rnn.tolist()).difference(set(rnn_index_bigmap)))
    np.random.shuffle(update_labeled_sets_rnn)
    np.save('/home/asdf/Documents/juyan/paper/paviac/mdcpe/newmdcpe/data/rnn/unlabeled_index.npy', unlabeled_sets_rnn)
    np.save('/home/asdf/Documents/juyan/paper/paviac/mdcpe/newmdcpe/data/rnn/labeled_index.npy', update_labeled_sets_rnn)
    sio.savemat('/home/asdf/Documents/juyan/paper/paviac/mdcpe/newmdcpe/data/rnn/Pavia_gt.mat', {'pavia_gt': real_gt_rnn})
    return ()

update_cnn()
update_rnn()

reason: import numpy,import scipy


text:
#!/usr/bin/env python3

'''
Training script for finite, constant-sum extensive-form games.
'''

import argparse
from collections import defaultdict
import json
from multiprocessing import Pool
import numpy as np
import os
import os.path
import pandas
from tensorboardX import SummaryWriter
import traceback
import yaml

from extensive_form.games import build_game
from extensive_form.algorithms import build_algorithm
from extensive_form.nash_conv import nash_conv
from grid_search import grid_search

def print_error(error):
    traceback.print_exception(type(error), error, error.__traceback__, limit=5)

def parse_args():
    parser = argparse.ArgumentParser("Training script for finite, constant-sum extensive-form games")

    parser.add_argument("-f", "--config-file", default=None, type=str, action="append",
                        help="If specified, use config options from this file.")
    parser.add_argument("-o", "--output-path", type=str, default="results/debug",
                        help="directory in which we should save results")
    parser.add_argument("-n", "--num-cpus", type=int, default=4,
                        help="the number of parallel worker processes to launch")

    return parser.parse_args()

def load_configs(args):
    if args.config_file:
        experiments = dict()

        for config_file in args.config_file:
            with open(config_file) as f:
                experiments.update(yaml.load(f, Loader=yaml.FullLoader))

    else:
        experiments = {
            "optimistic_nash_q_decoy_deep_sea": {
                "game": "decoy_deep_sea",
                "game_config": {
                    "decoy_games": 20,
                    "decoy_size": 20,
                    "decoy_payoff": 1.,
                    "target_size": 20,
                    "target_payoff": 1.,
                    "target_penalty": 0.0,
                    "adversary_payoff": 1.0
                },
                "alg": "strategic_ulcb",
                "alg_config": {
                    "iteration_episodes": 10,
                    "beta": 1.0,
                },
                "iterations": 200,
                "eval_interval": 10,
                "num_games": 1,
                "num_runs": 1,
            }
        }

    return experiments

def make_unique_dir(path, tag):
    sub_path = os.path.join(path, tag)
    idx = 0

    while os.path.exists(sub_path):
        idx += 1
        sub_path = os.path.join(path, tag + "_" + str(idx))

    os.makedirs(sub_path)
    return sub_path

def run_trail(path, experiment, game, game_idx, run_idx, seed):
    path = os.path.join(path, f"game_{game_idx}_run_{run_idx}")
    os.makedirs(path)

    # Initialize tensorboard logging
    writer = SummaryWriter(path, flush_secs=30)

    # Reseed numpy for parallel execution
    np.random.seed(seed)

    # Build environment and learning algorithm
    learner = build_algorithm(experiment["alg"],
                              game.build_env(),
                              experiment.get("alg_config", {}))

    results = defaultdict(list)
    statistics = defaultdict(list)
    total_samples = 0
    total_episodes = 0

    for iteration in range(experiment["iterations"]):
        samples, episodes, stats = learner.train()
        total_samples += samples
        total_episodes += episodes

        # Accumulate statistics between evaluation rounds
        for key, value in stats.items():
            statistics[key].append(value)

        if iteration != 0 and iteration % experiment["eval_interval"] == 0:

            # Compute NashConv end eval statistic
            row_value, column_value, nash = nash_conv(game, learner)

            results["iterations"].append(iteration)
            results["samples"].append(total_samples)
            results["episodes"].append(total_episodes)
            results["row_value"].append(row_value)
            results["column_value"].append(column_value)
            results["nash_conv"].append(nash)

            # Compute mean episode statistics
            for key, value in statistics.items():
                results["stats/" + key].append(np.mean(value))

            statistics = defaultdict(list)

            # Log to tensorboard
            for key, values in results.items():
                if not key.startswith("stats/"):
                    key = "loss/" + key

                writer.add_scalar(key, values[-1], global_step=results["samples"][-1])

    # Build and save data frame
    results_file = os.path.join(path, "results.csv")

    dataframe = pandas.DataFrame(results)
    dataframe.to_csv(results_file)

    # Close tensorboard writer
    writer.close()

def run_experiment(output_path, name, experiment, pool):

    # Make results directory
    path = make_unique_dir(output_path, name)

    # Save configuration
    with open(os.path.join(path, "config.json"), 'w') as config_file:
        json.dump({name: experiment}, config_file)

    # Launch trials
    trials = []
    seed = 0

    for game_idx in range(experiment["num_games"]):

        # Reseed to ensure games are identical across experiments
        np.random.seed(game_idx)

        # Build new (randomly generated) game
        game = build_game(experiment["game"], experiment.get("game_config", {}))

        for run_idx in range(experiment["num_runs"]):
            print(f"{name} - game: {game_idx}, run: {run_idx}")
            trials.append(pool.apply_async(run_trail, (path, experiment, game, game_idx, run_idx, seed), error_callback=print_error))
            seed += 1

    return trials

def main(args):
    experiments = load_configs(args)
    pool = Pool(args.num_cpus)

    trials = []

    for name, experiment in experiments.items():
        variations = grid_search(name, experiment)

        if variations is not None:
            output_path = make_unique_dir(args.output_path, name)

            with open(os.path.join(output_path, "config.json"), 'w') as config_file:
                json.dump({name: experiment}, config_file)

            for var_name, var_experiment in variations.items():
                trials += run_experiment(output_path, var_name, var_experiment, pool)
        else:
            trials += run_experiment(args.output_path, name, experiment, pool)

    for trial in trials:
        trial.wait()

if __name__ == '__main__':
    args = parse_args()
    main(args)


text:
import io
import socket
import struct
import time
import picamera
import picamera.array
import cv2
import numpy as np

client_socket = socket.socket()
client_socket.connect(('10.42.0.1', 8000))
#client_socket.connect(('192.168.1.152', 8000))

height = 240
width = 320
#height = 480
#width = 640

# Accept a single connection and make a file-like object out of it
connection = client_socket.makefile('wb')
try:
    #connection.write(struct.pack("<LLL", height, width, 3))
    client_socket.sendall(struct.pack("<LLL", height, width, 3))

    with picamera.PiCamera() as camera:
        camera.resolution = (width, height)
        camera.framerate = 10
        camera.awb_mode = "sunlight"
        rawCapture = picamera.array.PiRGBArray(camera, size=(width, height))
        # Start a preview and let the camera warm up
        #camera.start_preview()
        time.sleep(0.1)

        #start = time.time()
        #conn_write = connection.write

        # Indentify color red
        #lower = np.array([3, 150, 90], dtype = "uint8")
        #upper = np.array([12, 255, 255], dtype = "uint8")

        # Identify blue sponge
        lower_obj = np.array([100, 0, 0], dtype = "uint8")
        upper_obj = np.array([120, 255, 255], dtype = "uint8")

        # Indentify laser
        lower_laser = np.array([19, 0, 255], dtype = "uint8")
        upper_laser = np.array([255, 255, 255], dtype = "uint8")

        kernel = np.ones((5,5),np.uint8)

        center_x_obj = center_y_obj = center_x_laser = center_y_laser = 0

        print("Sending images")

        for frame in camera.capture_continuous(rawCapture, format='bgr', use_video_port=True):
            # Grab the raw NumPy array representing the image
            image = frame.array
            #print("size: {0}".format(image.size))

            hsv = cv2.cvtColor(image, cv2.COLOR_BGR2HSV)

            blurred = cv2.GaussianBlur(hsv, (5, 5), 0)

            mask_obj = cv2.inRange(blurred, lower_obj, upper_obj)
            mask_laser = cv2.inRange(hsv, lower_laser, upper_laser)
            #output = cv2.bitwise_and(image, image, mask=mask)

            mask_obj = cv2.erode(mask_obj, None, iterations=2)
            mask_obj = cv2.dilate(mask_obj, None, iterations=2)
            #mask = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kernel)
            #closing = cv2.morphologyEx(opening, cv2.MORPH_CLOSE, kernel)

            cnts_obj = cv2.findContours(mask_obj, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)[-2]
            cnts_laser = cv2.findContours(mask_laser, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)[-2]

            if len(cnts_obj) > 0:
                c_obj = max(cnts_obj, key=cv2.contourArea)

                rect = cv2.minAreaRect(c_obj)
                center_x_obj, center_y_obj = rect[0] #rect = ((center_x,center_y),(width,height),angle)
                center_x_obj, center_y_obj = int(center_x_obj), int(center_y_obj)
                box = cv2.boxPoints(rect)
                box = np.int0(box)
                cv2.drawContours(image, [box], 0, (0,255,0), 2)
                cv2.circle(image, (center_x_obj, center_y_obj), 3, (255,255,255), -1)

            if len(cnts_laser) > 0:
                c_laser = max(cnts_laser, key=cv2.contourArea)

                (x, y), radius = cv2.minEnclosingCircle(c_laser)
                (center_x_laser, center_y_laser) = (int(x), int(y))
                radius = int(radius)
                cv2.circle(image, (center_x_laser, center_y_laser), radius+5, (255,0,0), -1)

            print("obj=({0},{1})\t laser=({2},{3})".format(center_x_obj, center_y_obj, center_x_laser, center_y_laser))

            client_socket.sendall(image.data)
            #connection.flush()

            #print("Image sent")

            #finish = time.time()
            #print("%.1f FPS" % float(1/(finish-start)))
            #start = finish

            rawCapture.truncate(0)
except (KeyboardInterrupt):
    pass
finally:
    #connection.close()
    client_socket.close()

max_stars_repo_name: danielmundi/laser-tracking

max_issues_repo_name: danielmundi/laser-tracking

max_forks_repo_name: danielmundi/laser-tracking


text:
import numpy as np
from ...util.math import on_finite, linspc_, logspc_, linrng_, logrng_, flattengrid
from ...util.plot import bin_centres_to_edges
from ...util.distributions import sample_kde
import logging

logging.getLogger(__name__).addHandler(logging.NullHandler())
logger = logging.getLogger(__name__)

class DensityGrid(object):
    def __init__(
        self, x, y, extent=None, bins=50, logx=False, logy=False, coverage_scale=1.2
    ):
        # limits
        self.logx = logx
        self.logy = logy
        self.bins = bins
        self.coverage_scale = coverage_scale

        if extent is None:
            self.xmin, self.xmax, self.ymin, self.ymax = self.extent_from_xy(x, y)
        else:
            self.xmin, self.xmax, self.ymin, self.ymax = extent

        self.xstep = self.get_xstep()
        self.ystep = self.get_ystep()

        if self.logx:
            assert self.xmin > 0.0
            assert (self.xmin / self.xstep) > 0.0
        if self.logy:
            assert self.ymin > 0.0
            assert (self.ymin / self.ystep) > 0.0

        self.calculate_grid()

    def calculate_grid(self):
        self.get_centre_grid()
        self.get_edge_grid()

    def get_ystep(self):
        if self.logy:
            return (self.ymax / self.ymin) / self.bins
        else:
            return (self.ymax - self.ymin) / self.bins

    def get_xstep(self):
        if self.logx:
            return (self.xmax / self.xmin) / self.bins
        else:
            return (self.xmax - self.xmin) / self.bins

    def extent_from_xy(self, x, y, coverage_scale=None):
        cov = coverage_scale or self.coverage_scale
        expand_grid = (cov - 1.0) / 2
        return [
            *[linrng_, logrng_][self.logx](x, exp=expand_grid),
            *[linrng_, logrng_][self.logy](y, exp=expand_grid),
        ]

    def get_xrange(self):
        return self.xmin, self.xmax

    def get_yrange(self):
        return self.ymin, self.ymax

    def get_extent(self):
        return [*self.get_xrange(), *self.get_yrange()]

    def get_range(self):
        return [[*self.get_xrange()], [*self.get_yrange()]]

    def update_grid_centre_ticks(self):
        if self.logx:
            self.grid_xc = logspc_(self.xmin, self.xmax, 1.0, self.bins)
        else:
            self.grid_xc = linspc_(self.xmin, self.xmax, self.xstep, self.bins)

        if self.logy:
            self.grid_yc = logspc_(self.ymin, self.ymax, 1.0, self.bins)
        else:
            self.grid_yc = linspc_(self.ymin, self.ymax, self.ystep, self.bins)

    def update_grid_edge_ticks(self):
        self.update_grid_centre_ticks()
        if self.logx:
            self.grid_xe = np.exp(bin_centres_to_edges(np.log(np.sort(self.grid_xc))))
        else:
            self.grid_xe = bin_centres_to_edges(np.sort(self.grid_xc))

        if self.logy:
            self.grid_ye = np.exp(bin_centres_to_edges(np.log(np.sort(self.grid_yc))))
        else:
            self.grid_ye = bin_centres_to_edges(np.sort(self.grid_yc))

    def get_centre_grid(self):
        self.update_grid_centre_ticks()
        self.grid_xci, self.grid_yci = np.meshgrid(self.grid_xc, self.grid_yc)

    def get_edge_grid(self):
        self.update_grid_edge_ticks()
        self.grid_xei, self.grid_yei = np.meshgrid(self.grid_xe, self.grid_ye)

    def get_hex_extent(self):
        if self.logx:
            xex = [np.log(self.xmin / self.xstep), np.log(self.xmax * self.xstep)]
        else:
            xex = [self.xmin - self.xstep, self.xmax + self.xstep]

        if self.logy:
            yex = [np.log(self.ymin / self.ystep), np.log(self.ymax * self.ystep)]
        else:
            yex = [self.ymin - self.ystep, self.ymax + self.ystep]
        return xex + yex

    def kdefrom(self, xy, xtransform=lambda x: x, ytransform=lambda x: x):
        assert np.isfinite(self.grid_xc).all() and np.isfinite(self.grid_yc).all()
        arr = xy.copy()
        # generate x grid over range spanned by log(x)
        arr[:, 0] = xtransform(xy[:, 0])
        # generate y grid over range spanned by log(y)
        arr[:, 1] = ytransform(xy[:, 1])
        zi = sample_kde(
            arr,
            flattengrid(
                np.meshgrid(xtransform(self.grid_xc), ytransform(self.grid_yc))
            ),
        )
        zi = zi.reshape(self.grid_xci.shape)
        return zi

max_stars_repo_name: JustinGOSSES/pyrolite

max_issues_repo_name: JustinGOSSES/pyrolite

max_forks_repo_name: JustinGOSSES/pyrolite