text:
# Copyright (c) 2021 Graphcore Ltd. All rights reserved.
# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.
# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This file has been modified by Graphcore Ltd.
import os
from pathlib import Path

import numpy as np
import pytest
import tensorflow as tf
from tensorflow.python import ipu
from transformers import BertConfig
from transformers.models.bert.modeling_tf_bert import TFBertForPreTraining
from transformers.modeling_tf_utils import shape_list

from model.convert_bert_model import convert_tf_bert_model, post_process_bert_input_layer
from model.ipu_pretraining_model import gather_positions, IpuTFBertForPreTraining
from data_utils.wikipedia.load_wikipedia_data import get_pretraining_dataset
from model.losses import mlm_loss, nsp_loss
from model.pipeline_stage_names import PIPELINE_ALLOCATE_PREVIOUS, PIPELINE_NAMES
from utilities.assign_pipeline_stages import PipelineStagesAssigner
from utilities.ipu_utils import set_random_seeds

def hf_mlm_compute_loss(labels, logits):
    """
    Adapted to our dataset with labels with non-masked-tokens marker=0.
    """
    loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(
        from_logits=True, reduction=tf.keras.losses.Reduction.NONE
    )
    masked_lm_active_loss = tf.not_equal(tf.reshape(tensor=labels["mlm_long_labels"], shape=(-1,)), 0)
    masked_lm_reduced_logits = tf.boolean_mask(
        tensor=tf.reshape(tensor=logits[0], shape=(-1, shape_list(logits[0])[2])),
        mask=masked_lm_active_loss,
    )
    masked_lm_labels = tf.boolean_mask(
        tensor=tf.reshape(tensor=labels["mlm_long_labels"], shape=(-1,)), mask=masked_lm_active_loss
    )
    masked_lm_loss = loss_fn(y_true=masked_lm_labels, y_pred=masked_lm_reduced_logits)
    masked_lm_loss = tf.reduce_mean(input_tensor=masked_lm_loss)
    return masked_lm_loss

def hf_nsp_compute_loss(labels, logits):
    loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(
        from_logits=True, reduction=tf.keras.losses.Reduction.NONE
    )
    next_sentence_active_loss = tf.not_equal(tf.reshape(tensor=labels["next_sentence_label"], shape=(-1,)), -100)
    next_sentence_reduced_logits = tf.boolean_mask(
        tensor=tf.reshape(tensor=logits[1], shape=(-1, 2)), mask=next_sentence_active_loss
    )
    next_sentence_label = tf.boolean_mask(
        tensor=tf.reshape(tensor=labels["next_sentence_label"], shape=(-1,)), mask=next_sentence_active_loss
    )
    next_sentence_loss = loss_fn(y_true=next_sentence_label, y_pred=next_sentence_reduced_logits)
    next_sentence_loss = tf.reduce_mean(input_tensor=next_sentence_loss)
    return next_sentence_loss

def get_path():
    path = os.path.dirname(os.path.realpath(__file__))
    return os.path.dirname(path)

def load_dataset(micro_batch_size,
                 dataset_dir,
                 seq_length=128):
    dataset, _ = get_pretraining_dataset(micro_batch_size=micro_batch_size,
                                         dataset_dir=dataset_dir,
                                         max_seq_length=seq_length,
                                         max_predictions_per_seq=20,
                                         distributed_worker_count=1,
                                         seed=42,
                                         data_type=tf.float16,
                                         test=True)
    iterator = iter(dataset)
    sample = next(iterator)
    return sample

class TestConvertHFToFunctionalBertPretraining:

    @classmethod
    def setup_class(cls):
        cls.micro_batch_size = 2
        cls.seq_length = 128
        cls.max_predictions_per_seq = 20
        cls.sample_inputs, cls.sample_labels = load_dataset(
            micro_batch_size=cls.micro_batch_size,
            dataset_dir=Path(get_path()).joinpath("data_utils").joinpath("wikipedia"),
            seq_length=cls.seq_length,
        )

        cfg = ipu.config.IPUConfig()
        cfg.device_connection.type = ipu.config.DeviceConnectionType.ON_DEMAND
        cfg.configure_ipu_system()
        cls.strategy = ipu.ipu_strategy.IPUStrategy()
        with cls.strategy.scope():

            config = BertConfig(
                vocab_size=30528,
                num_hidden_layers=4,
                num_attention_heads=4,
                hidden_size=256,
                intermediate_size=768,
                max_predictions_per_seq=cls.max_predictions_per_seq,
            )

            set_random_seeds(seed=42)
            cls.orig_model = TFBertForPreTraining(config)
            cls.orig_outputs = cls.orig_model(cls.sample_inputs)
            cls.orig_logits = gather_positions(
                cls.orig_outputs.prediction_logits,
                cls.sample_inputs['masked_lm_positions']
            )

            set_random_seeds(seed=42)
            ipu_subclass_model = IpuTFBertForPreTraining(config)
            cls.sub_outputs = ipu_subclass_model(cls.sample_inputs)

            cls.functional_model = convert_tf_bert_model(
                ipu_subclass_model,
                cls.sample_inputs,
                post_process_bert_input_layer,
                replace_layers=True,
                use_outlining=True,
                embedding_serialization_factor=2
            )
            cls.functional_model.compile(loss={'mlm___cls': mlm_loss, 'nsp___cls': nsp_loss})
            cls.func_outputs = cls.functional_model(cls.sample_inputs)

    def test_get_pretraining_dataset(self):
        assert set(self.sample_inputs.keys()) == {
            'input_ids',
            'token_type_ids',
            'attention_mask',
            'masked_lm_positions'
        }
        for key, val in self.sample_inputs.items():
            if key == 'masked_lm_positions':
                assert tuple(val.shape) == (self.micro_batch_size, self.max_predictions_per_seq)
            else:
                assert tuple(val.shape) == (self.micro_batch_size, self.seq_length)

        assert len(self.sample_labels) == 3
        assert self.sample_labels[0].shape.as_list() == [self.micro_batch_size, self.max_predictions_per_seq]
        assert self.sample_labels[1].shape.as_list() == [self.micro_batch_size, 1]
        assert self.sample_labels[2].shape.as_list() == [self.micro_batch_size, self.seq_length]

    def test_inference_outputs(self):
        np.testing.assert_almost_equal(
            self.orig_logits.numpy(),
            self.sub_outputs.prediction_logits.numpy(),
            decimal=5
        )
        np.testing.assert_almost_equal(
            self.sub_outputs.prediction_logits.numpy(),
            self.func_outputs[0].numpy(),
            decimal=5
        )
        np.testing.assert_almost_equal(
            self.sub_outputs.seq_relationship_logits.numpy(),
            self.func_outputs[1].numpy(),
            decimal=5
        )

    def test_inference_losses(self):
        labels = {
            "mlm_labels": self.sample_labels[0],
            "next_sentence_label": self.sample_labels[1],
            "mlm_long_labels": self.sample_labels[2]
        }

        orig_mlm_loss = hf_mlm_compute_loss(labels, self.orig_outputs)
        orig_nsp_loss = hf_nsp_compute_loss(labels, self.orig_outputs)
        orig_loss = orig_mlm_loss + orig_nsp_loss

        func_mlm_loss = self.functional_model.loss['mlm___cls'](labels["mlm_labels"], self.func_outputs[0])
        func_nsp_loss = self.functional_model.loss['nsp___cls'](labels["next_sentence_label"], self.func_outputs[1])
        func_loss = func_mlm_loss + func_nsp_loss

        np.testing.assert_almost_equal(orig_mlm_loss, func_mlm_loss, decimal=5)
        np.testing.assert_almost_equal(orig_nsp_loss, func_nsp_loss, decimal=5)
        np.testing.assert_almost_equal(orig_loss, func_loss, decimal=5)

class TestAssignPipelineStages:
    @classmethod
    def setup_class(cls):
        bert_config = BertConfig(max_predictions_per_seq=20)
        model = IpuTFBertForPreTraining(config=bert_config)
        sample_inputs, _ = load_dataset(
            micro_batch_size=2,
            dataset_dir=Path(get_path()).joinpath("data_utils").joinpath("wikipedia"),
            seq_length=128,
        )
        cfg = ipu.config.IPUConfig()
        cfg.device_connection.type = ipu.config.DeviceConnectionType.ON_DEMAND
        cfg.configure_ipu_system()
        strategy = ipu.ipu_strategy.IPUStrategy()
        with strategy.scope():
            cls.functional_model = convert_tf_bert_model(
                model,
                sample_inputs,
                post_process_bert_input_layer,
                replace_layers=True,
                use_outlining=True
            )

    @pytest.mark.parametrize(
        "pipeline_stages, expected_stages",
        [([["emb"],
           ["hid", "hid", "hid", "hid"],
           ["hid", "hid", "hid", "hid"],
           ["hid", "hid", "hid", "hid", "enc_out"],
           ["pool", "heads"]
           ],
          # The first zeros correspond to lambda layers (tf.ops) that process the input before the embeddings. The last
          # elements 4, 3, 4, 4 of the list correspond to the pooler, the layer to reduce the logits, and the two heads,
          # respectively, which are listed in this order in the model list of assignments.
          [0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 3, 4, 4]),
         ([["emb"],
           ["hid", "hid"],
           ["hid", "hid"],
           ["hid", "hid"],
           ["hid", "hid"],
           ["hid", "hid"],
           ["hid", "hid", "enc_out"],
           ["pool", "heads"]
           ],
          # As above, the first zeros correspond to lambda layers (tf.ops), while the last 7, 6, 7, 7 elements
          # correspond to the pooler, reduce logits layer and the heads, in the order stored in the model.
          [0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 6, 7, 7]),
         ])
    def test_pipeline(self, pipeline_stages, expected_stages):
        assignments = self.functional_model.get_pipeline_stage_assignment()
        pipeline_assigner = PipelineStagesAssigner(PIPELINE_ALLOCATE_PREVIOUS, PIPELINE_NAMES)
        assignments = pipeline_assigner.assign_pipeline_stages(assignments, pipeline_stages)
        self.functional_model.set_pipeline_stage_assignment(assignments)
        assert [a.pipeline_stage for a in assignments] == expected_stages


text:
import graphy as G
import numpy as np
import time, sys, os
from sacred import Experiment
from __builtin__ import False

ex = Experiment('Deep VAE')

@ex.config
def config():

    # optimization:
    n_reporting = 10 #epochs between reporting
    px = 'logistic'
    pad_x = 0

    # datatype
    problem = 'cifar10'
    n_batch = 16 # Minibatch size
    if problem == 'mnist':
        shape_x = (1,28,28)
        px = 'bernoulli'
        pad_x = 2
        n_h = 64
        n_z = 32
    if problem == 'cifar10':
        shape_x = (3,32,32)
        n_h = 160
        n_z = 32
    if problem == 'svhn':
        shape_x = (3,32,32)
        n_reporting = 1
        n_h = 160
        n_z = 32
    if problem == 'lfw':
        shape_x = (3,64,48)
        n_h = 160
        n_z = 32

    n_h1 = n_h
    n_h2 = n_h

    # dataset
    n_train = 0

    # model
    model_type = 'cvae1'

    if model_type == 'cvae1':
        depths = [2,2]

        margs = {
            'shape_x': shape_x,
            'depths': depths,
            'n_h1': n_h1,
            'n_h2': n_h2,
            'n_z': n_z,
            'prior': 'diag',
            'posterior': 'down_diag',
            'px': px,
            'nl': 'elu',
            'kernel_x': (5,5),
            'kernel_h': (3,3),
            'kl_min': 0.25,
            'optim': 'adamax',
            'alpha': 0.002,
            'beta1': 0.1,
            'pad_x': pad_x,
            'weightsharing': False,
            'depth_ar': 1,
            'downsample_type': 'nn'
        }

    if model_type == 'simplecvae1':
        depths = [2,2,2]
        widths = [32,64,128]

        margs = {
            'shape_x': shape_x,
            'depths': depths,
            'widths': widths,
            'n_z': n_z,
            'prior': 'diag',
            'posterior': 'down_diag',
            'px': px,
            'nl': 'elu',
            'kernel_x': (5,5),
            'kernel_h': (3,3),
            'kl_min': 0.25,
            'optim': 'adamax',
            'alpha': 0.002,
            'beta1': 0.1,
            'pad_x': pad_x,
            'weightsharing': False
        }

    # model loading/saving
    save_model = True
    load_model_path = None
    load_model_complete = True # Whether loaded parameters are complete

    # Estimate the marginal likelihood
    est_marglik = 0.
    est_marglik_data = 'valid'

def init_logs():
    global logpath, logdir
    # Create log directory
    logdir = str(time.time())
    logpath = os.environ['ML_LOG_PATH']+'/'+logdir+'/'
    print 'Logpath: '+logpath
    os.makedirs(logpath)
    # Log stdout messages to file
    sys.stdout = G.misc.logger.Logger(logpath+"log.txt")
    # Clone local source to logdir
    os.system("rsync -au --include '*/' --include '*.py' --exclude '*' . "+logpath+"source")
    with open(logpath+"source/run.sh", 'w') as f:
        f.write("python "+" ".join(sys.argv)+"\n")
    os.chmod(logpath+"source/run.sh", 0700)

@ex.capture
def construct_model(data_init, model_type, margs, load_model_path, load_model_complete, n_batch):
    import models
    margs['data_init'] = data_init
    if model_type == 'fcvae1':
        model = models.fcvae(**margs)
    if model_type == 'cvae1':
        model = models.cvae1(**margs)
    if model_type == 'simplecvae1':
        import simplemodel
        model = simplemodel.simplecvae1(**margs)

    if load_model_path != None:
        print 'Loading existing model at '+load_model_path
        _w = G.ndict.np_loadz(load_model_path+'/weights.ndict.tar.gz')

        G.ndict.set_value(model.w, _w, load_model_complete)
        G.ndict.set_value(model.w_avg, _w, load_model_complete)

    return model

@ex.capture
def get_data(problem, n_train, n_batch):

    if problem == 'cifar10':
        # Load data
        data_train, data_valid = G.misc.data.cifar10(False)
    if problem == 'svhn':
        # Load data
        data_train, data_valid = G.misc.data.svhn(False, True)
    elif problem == 'mnist':
        # Load data
        validset = False
        if validset:
            data_train, data_valid, data_test = G.misc.data.mnist_binarized(validset, False)
        else:
            data_train, data_valid = G.misc.data.mnist_binarized(validset, False)
        data_train['x'] = data_train['x'].reshape((-1,1,28,28))
        data_valid['x'] = data_valid['x'].reshape((-1,1,28,28))
    elif problem == 'lfw':
        data_train = G.misc.data.lfw(False,True)
        data_valid = G.ndict.getRows(data_train, 0, 1000)

    data_init = {'x':data_train['x'][:n_batch]}

    if n_train > 0:
        data_train = G.ndict.getRows(data_train, 0, n_train)
        data_valid = G.ndict.getRows(data_valid, 0, n_train)

    return data_train, data_valid, data_init

@ex.automain
def train(shape_x, problem, n_batch, n_train, n_reporting, save_model, est_marglik, est_marglik_data, margs):

    global logpath

    # Initialize logs
    init_logs()

    # Get data
    data_train, data_valid, data_init = get_data()

    # Construct model
    model = construct_model(data_init)

    # Estimate the marginal likelihood
    if est_marglik > 0:
        if est_marglik_data == 'valid':
            data = data_valid
        elif est_marglik_data == 'train':
            data = data_train
        # Correction since model's actual cost is divided by this factor
        correctionfactor = - (np.prod(shape_x) * np.log(2.))
        obj_test = []
        for i in range(est_marglik):
            cost = model.eval(data, n_batch=n_batch, randomorder=False)['cost'] * correctionfactor
            obj_test.append(cost)
            _obj = np.vstack(obj_test)
            _max = np.max(_obj, axis=0)
            _est = np.log(np.exp(_obj - _max).mean(axis=0)) + _max
            if i%1 == 0:
                print 'Estimate of logp(x) after', i+1, 'samples:', _est.mean() / correctionfactor
        raise Exception()
        sys.exit()

    # Report
    cost_best = [None]
    eps_fixed = model.eps({'n_batch':100})
    def report(epoch, dt, cost):
        if np.isnan(cost):
            raise Exception('NaN detected!!')

        results_valid = model.eval(data_valid, n_batch=n_batch)
        for i in results_valid: results_valid[i] = results_valid[i].mean()

        _w = G.ndict.get_value(model.w_avg)
        G.ndict.np_savez(_w, logpath+'weights')

        if cost_best[0] is None or results_valid['cost'] < cost_best[0]:
            cost_best[0] = results_valid['cost']
            if save_model:
                G.ndict.np_savez(_w, logpath+'weights_best')

        if True:
            # Write all results to file
            with open(logpath+"results.txt", "a") as log:
                if epoch == 0:
                    log.write("Epoch "+" ".join(map(str, results_valid.keys())) + "\n")
                log.write(str(epoch)+" "+" ".join(map(str, results_valid.values())) + "\n")

        if True:
            eps = model.eps({'n_batch':100})
            image = model.decode(eps)
            G.graphics.save_raster(image, logpath+'sample_'+str(epoch)+'.png')
            image = model.decode(eps_fixed)
            G.graphics.save_raster(image, logpath+'sample_fixed1_'+str(epoch)+'.png')

            #eps_fixed_copy = G.ndict.clone(eps_fixed)
            #for i in range(len(eps_fixed)):
            #    eps_fixed_copy['']

        if epoch == 0:
            print 'logdir:', 't:', 'Epoch:', 'Train cost:', 'Valid cost:', 'Best:', 'log(stdev) of p(x|z):'

        logsd_x = 0.
        if 'logsd_x' in model.w_avg:
            logsd_x = model.w_avg['logsd_x'].get_value()
        print logdir, '%.2f'%dt, epoch, '%.5f'%cost, '%.5f'%results_valid['cost'], '%.5f'%cost_best[0], logsd_x

    print 'Training'

    for epoch in xrange(1000000):
        t0 = time.time()

        result = model.train(data_train, n_batch=n_batch)

        if epoch <= 10 or epoch%n_reporting == 0:
            report(epoch, time.time()-t0, cost=np.mean(result))

ext: py

max_stars_repo_name: christopherhesse/iaf

max_issues_repo_name: christopherhesse/iaf

max_forks_repo_name: christopherhesse/iaf

reason: import numpy


text:
import os
import random
import shutil
from collections import OrderedDict

import numpy as np
import torch
import torch.nn as nn
from imgaug import imgaug as ia
from termcolor import colored
from torch.autograd import Variable

####
def convert_pytorch_checkpoint(net_state_dict):
    variable_name_list = list(net_state_dict.keys())
    is_in_parallel_mode = all(v.split(".")[0] == "module" for v in variable_name_list)
    if is_in_parallel_mode:
        colored_word = colored("WARNING", color="red", attrs=["bold"])
        print(
            (
                "%s: Detect checkpoint saved in data-parallel mode."
                " Converting saved model to single GPU mode." % colored_word
            ).rjust(80)
        )
        net_state_dict = {
            ".".join(k.split(".")[1:]): v for k, v in net_state_dict.items()
        }
    return net_state_dict

####
def check_manual_seed(seed):
    """ If manual seed is not specified, choose a
    random one and communicate it to the user.

    Args:
        seed: seed to check

    """
    seed = seed or random.randint(1, 10000)
    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    # ia.random.seed(seed)

    print("Using manual seed: {seed}".format(seed=seed))
    return

####
def check_log_dir(log_dir):
    """Check if log directory exists.

    Args:
        log_dir: path to logs

    """
    if os.path.isdir(log_dir):
        colored_word = colored("WARNING", color="red", attrs=["bold", "blink"])
        print("%s: %s exist!" % (colored_word, colored(log_dir, attrs=["underline"])))
        while True:
            print("Select Action: d (delete) / q (quit)", end="")
            key = input()
            if key == "d":
                shutil.rmtree(log_dir)
                break
            elif key == "q":
                exit()
            else:
                color_word = colored("ERR", color="red")
                print("---[%s] Unrecognize Characters!" % colored_word)
    return

def get_model_summary(
    model, input_size, batch_size=-1, device=torch.device("cpu"), dtypes=None
):
    """Reusable utility layers such as pool or upsample will also get printed, but their printed values will
    be corresponding to the last call.

    """
    if dtypes == None:
        dtypes = [torch.FloatTensor] * len(input_size)

    summary_str = ""

    def register_hook(module):
        def hook(module, input, output):
            class_name = str(module.__class__).split(".")[-1].split("'")[0]
            module_idx = len(summary)

            m_key = module.name if module.name != "" else "%s" % class_name

            summary[m_key] = OrderedDict()
            summary[m_key]["input_shape"] = list(input[0].size())
            summary[m_key]["input_shape"][0] = batch_size
            if isinstance(output, (list, tuple)):
                summary[m_key]["output_shape"] = [
                    [-1] + list(o.size())[1:] for o in output
                ]
            elif isinstance(output, dict):
                summary[m_key]["output_shape"] = [
                    [-1] + list(o.size())[1:] for o in output.values()
                ]
            elif isinstance(output, torch.Tensor):
                summary[m_key]["output_shape"] = list(output.size())
                summary[m_key]["output_shape"][0] = batch_size

            params = 0
            if hasattr(module, "weight") and hasattr(module.weight, "size"):
                params += torch.prod(torch.LongTensor(list(module.weight.size())))
                summary[m_key]["trainable"] = module.weight.requires_grad
            if hasattr(module, "bias") and hasattr(module.bias, "size"):
                params += torch.prod(torch.LongTensor(list(module.bias.size())))
            summary[m_key]["nb_params"] = params

        if len(list(module.children())) == 0:
            hooks.append(module.register_forward_hook(hook))

    # multiple inputs to the network
    if isinstance(input_size, tuple):
        input_size = [input_size]

    # batch_size of 2 for batchnorm
    x = [
        torch.rand(2, *in_size).type(dtype).to(device=device)
        for in_size, dtype in zip(input_size, dtypes)
    ]

    # create properties
    summary = OrderedDict()
    hooks = []

    # create layer name according to hierachy names
    for name, module in model.named_modules():
        module.name = name

    # register hook
    model.apply(register_hook)

    # make a forward pass
    model(*x)

    # aligning name to the left
    max_name_length = len(max(summary.keys(), key=len))
    summary = [(k.ljust(max_name_length), v) for k, v in summary.items()]
    summary = OrderedDict(summary)

    # remove these hooks
    for h in hooks:
        h.remove()

    header_line = "{}  {:>25} {:>15}".format(
        "Layer Name".center(max_name_length), "Output Shape", "Param #"
    )
    summary_str += "".join("-" for _ in range(len(header_line))) + "\n"
    summary_str += header_line + "\n"
    summary_str += "".join("=" for _ in range(len(header_line))) + "\n"
    total_params = 0
    total_output = 0
    trainable_params = 0
    for layer in summary:
        # input_shape, output_shape, trainable, nb_params
        line_new = "{:>20}  {:>25} {:>15}".format(
            layer,
            str(summary[layer]["output_shape"]),
            "{0:,}".format(summary[layer]["nb_params"]),
        )
        total_params += summary[layer]["nb_params"]

        total_output += np.prod(summary[layer]["output_shape"])
        if "trainable" in summary[layer]:
            if summary[layer]["trainable"] == True:
                trainable_params += summary[layer]["nb_params"]
        summary_str += line_new + "\n"

    # assume 4 bytes/number (float on cuda).
    total_input_size = abs(
        np.prod(sum(input_size, ())) * batch_size * 4.0 / (1024 ** 2.0)
    )
    total_output_size = abs(
        2.0 * total_output * 4.0 / (1024 ** 2.0)
    )  # x2 for gradients
    total_params_size = abs(total_params * 4.0 / (1024 ** 2.0))
    total_size = total_params_size + total_output_size + total_input_size

    summary_str += "".join("=" for _ in range(len(header_line))) + "\n"
    summary_str += "Total params: {0:,}".format(total_params) + "\n"
    summary_str += "Trainable params: {0:,}".format(trainable_params) + "\n"
    summary_str += (
        "Non-trainable params: {0:,}".format(total_params - trainable_params) + "\n"
    )
    summary_str += "".join("-" for _ in range(len(header_line))) + "\n"
    summary_str += "Input size (MB): %0.2f" % total_input_size + "\n"
    summary_str += "Forward/backward pass size (MB): %0.2f" % total_output_size + "\n"
    summary_str += "Params size (MB): %0.2f" % total_params_size + "\n"
    summary_str += "Estimated Total Size (MB): %0.2f" % total_size + "\n"
    summary_str += "".join("-" for _ in range(len(header_line))) + "\n"
    return summary_str


text:
import matplotlib.pyplot as plt
from eventio import IACTFile
import numpy as np
from argparse import ArgumentParser

parser = ArgumentParser()
parser.add_argument('inputfile')
parser.add_argument('-o', '--output-file')
parser.add_argument('-b', '--n-bins', default=500, type=int)
parser.add_argument('-e', '--event', default=1)

masses = {
    'Electrons': 0.000511,
    'Muons': 0.105658,
    'Pions': 0.139570,
}

def main():
    args = parser.parse_args()
    f = IACTFile(args.inputfile)
    it = iter(f)

    radius = f.telescope_positions['r'][0] / 100

    for i in range(args.event):
        event = next(it)

    photons = event.photon_bunches[0]
    emitter = event.emitter[0]

    fig, ax = plt.subplots(1, 1, figsize=(8, 8))

    image = np.zeros((args.n_bins, args.n_bins, 3))

    ax.set_title('R: Electrons, G: Muons, B: Pions')

    for i, (particle, mass) in enumerate(masses.items()):
        mask = np.isclose(emitter['mass'], mass, rtol=1e-4)

        hist, edges, edges = np.histogram2d(
            photons['x'][mask] / 100,
            photons['y'][mask] / 100,
            bins=args.n_bins,
            range=[[-radius, radius], [-radius, radius]],
        )
        hist = np.log10(1 + hist)
        image[:, :, i] = hist.T[::-1, :]

    image /= image.max()
    plt.imshow(image, extent=[-radius, radius, -radius, radius])
    plt.xlabel('x / m')
    plt.ylabel('y / m')
    plt.tight_layout()

    if args.output_file is None:
        plt.show()
    else:
        plt.savefig(args.output_file)

if __name__ == '__main__':
    main()

max_stars_repo_name: mblnk/pyeventio

max_issues_repo_name: mblnk/pyeventio

max_forks_repo_name: mblnk/pyeventio


text:
#
# Copyright John Reid 2009
#

"""
Some matrices to use as examples for structural basis for gapped PWMs.
"""

import numpy
from itertools import imap, ifilter

cebp_M00912_seqs = """
TTGC-TAAACT
TTGC-GCAAGA
TTGC-TCAATT
TTTC-TCAATA
TTGCTTGAATA
TTGC-CCAACT
"""

mef2_M00941_seqs = """
GGTTATTTTTAA-
GGTTATAATTAA-
AGTTATTTTTAA-
GGTTATTTTTAG-
TGCTAT-TTTAAA
GGCTAT-TTTAAG
ACCTATATTTAG-
AGCTAT-TTTAGG
GGCTATATTTAT-
"""

def parse_site_strings(s):
    """
    Take a string such as:

    GGTTATTTTTAA-
    GGTTATAATTAA-
    AGTTATTTTTAA-
    GGTTATTTTTAG-
    TGCTAT-TTTAAA
    GGCTAT-TTTAAG
    ACCTATATTTAG-
    AGCTAT-TTTAGG
    GGCTATATTTAT-

    and parse it to produce a list of sites.
    """
    return filter(None, imap(str.strip, s.split('\n')))

def char_to_base(c):
    if 'a' == c or 'A' == c:
        return 0
    if 'c' == c or 'C' == c:
        return 1
    if 'g' == c or 'G' == c:
        return 2
    if 't' == c or 'T' == c:
        return 3
    if '-' == c:
        return 4
    raise RuntimeError('Unknown base: %s' % c)

def seqs_to_freqs(seqs):
    if 0 == len(seqs):
        raise RuntimeError('No sequences given')
    K = len(seqs[0])
    counts = numpy.zeros((K, 5))
    for seq in seqs:
        if K != len(seq):
            raise RuntimeError('Not all sequences are the same length')
        for k, b in enumerate(imap(char_to_base, seq)):
            counts[k, b] += 1
    gaps = 1. - counts[:,4] / len(seqs)
    base_counts = counts[:,:4]
    freqs = (base_counts.T / base_counts.sum(axis=1)).T
    return freqs, gaps

def complete_gaps_to_list(gaps):
    "Converts format from array to number for each base to list of tuples (position, gap freq)."
    for i, g in enumerate(gaps):
        if g < 1.:
            yield i, g

if '__main__' == __name__:
    from information_content import MultipleGappedPWM, BG, ic
    for tag, seqs in [
        ('CEBP', cebp_M00912_seqs),
        ('MEF2', mef2_M00941_seqs),
    ]:
        freqs, gaps = seqs_to_freqs(parse_site_strings(seqs))
        pwm = MultipleGappedPWM(freqs, complete_gaps_to_list(gaps))
        logo = pwm.logo()
        logo.save('%s-logo.png' % tag)
        IC = ic(pwm, BG(pwm.K)) / numpy.log(2.)
        print '%s: IC = %f bits' % (tag, IC)

max_stars_repo_name: JohnReid/biopsy

max_issues_repo_name: JohnReid/biopsy

max_forks_repo_name: JohnReid/biopsy


text:
#Amtrak Recursive ROute Writer (ARROW)
#cont- does not write initial .npz file, relies on existing partials

def main(newdata=False,
         cont=False,
         newredund=False,
         arrive=True):
    import json
    import numpy as np
    import os
    import route_builder
    import glob
    import find_redundancy

    local = 'F:/Python34/America_By_Train/'
    rb = local+'route_builder/'
    direc = 'C:/Users/Owner/Documents/GitHub/capecchi.github.io/posts/AmericaByTrain/'

    if newdata or not os.path.isfile(local+'endpts.npz'):

        with open(direc+'amtrak.geojson') as f:
            data = json.load(f)
        feats = data['features']
        index = np.arange(len(feats))
        strt = []
        end = []
        for i in index:
            cc = feats[i]['geometry']['coordinates']
            strt.append(cc[0])
            end.append(cc[-1])

        #NEED route GPS endpoints to look for
        fcoords = local
        #fraarcid
        stpaulid = 182592 #keep east pt
        stpaul_iarr_cid = 182614 #mark eastern segment as redundant so we only search west
        portland_cid = 266301 #block southern route to Portland
        seattleid = 241310 #keep south pt
        laid = 211793 #keep south pt
        palmspringsid = 263261 #keep west pt
        neworleansid_end = 178659 #keep east pt NOTE does not connect to neworleans_start
        neworleansid_start = 243859 #keep south or east pt
        phillyid = 204870 #keep north pt
        dcid = 164103 #keep south pt
        chicagoid = 253079 #keep north pt

        eb_block = np.array([],dtype=int)
        cs_block = np.array([],dtype=int)
        sl_block = np.array([],dtype=int)
        cr_block = np.array([],dtype=int)
        cl_block = np.array([],dtype=int)

        for i in index:
            cid = feats[i]['properties']['FRAARCID']
            coords = feats[i]['geometry']['coordinates']
            c1 = coords[0]
            c2 = coords[-1]
            if cid == stpaulid:
                if c1[0] > c2[0]: stpaul = c1
                else: stpaul = c2
            if cid == stpaul_iarr_cid or cid == portland_cid:
                eb_block = np.append(eb_block,i)
            if cid == seattleid:
                if c1[1] < c2[1]: seattle = c1
                else: seattle = c2
            if cid == laid:
                if c1[1] < c2[1]: la = c1
                else: la = c2
            if cid == seattleid or cid == portland_cid or cid == 189128\
               or cid == 244148 or cid == 254149:
                cs_block = np.append(cs_block,i)
            if cid == palmspringsid:
                if c1[0] < c2[0]: palmsprings = c1
                else: palmsprings = c2
            if cid == neworleansid_end:
                if c1[0] > c2[0]: neworleans_end = c1
                else: neworleans_end = c2
            if cid == 263258 or cid == 266284 or cid == 178673:
                sl_block = np.append(sl_block,i)
            if cid == neworleansid_start:
                if c1[0] > c2[0]: neworleans_start = c1
                else: neworleans_start = c2
            if cid == phillyid:
                if c1[1] > c2[1]: philly = c1
                else: philly = c2
            if cid == 243812 or cid == 204623 or cid == 169919 or cid == 169921\
               or cid == 125491 or cid == 164053 or cid == 275062 or cid == 261822:
                cr_block = np.append(cr_block,i)
            if cid == dcid:
                if c1[1] < c2[1]: dc = c1
                else: dc = c2
            if cid == chicagoid:
                if c1[1] > c2[1]: chicago = c1
                else: chicago = c2
            if cid == 252822 or cid == 164114 or cid == 252939 or cid == 152297\
               or cid == 197933 or cid == 197961 or cid == 192650 or cid == 192649\
               or cid == 253070 or cid == 256677 or cid == 193489 or cid == 266257\
               or cid == 266676:
                cl_block = np.append(cl_block,i)
        cid = [feats[i]['properties']['FRAARCID'] for i in index]

        if newredund:
            #Identify redundant track segments
            fraarcid = [feats[i]['properties']['FRAARCID'] for i in index]
            iredund = np.array([],dtype=int)
            np.save(local+'redundant',iredund)
            redundant = find_redundancy.main(index,strt,end,fraarcid,local)

        #SAVE STUFF
        np.savez(local+'endpts',index=index,strt=strt,end=end,cid=cid,
                 stpaul=stpaul,seattle=seattle,la=la,palmsprings=palmsprings,
                 neworleans_end=neworleans_end,neworleans_start=neworleans_start,
                 philly=philly,dc=dc,chicago=chicago,eb_block=eb_block,
                 cs_block=cs_block,sl_block=sl_block,cr_block=cr_block,cl_block=cl_block)
        print('saved endpts arrays and city GPS coords')

    else:
        f=np.load(local+'endpts.npz')
        index = f['index']
        strt = f['strt']
        end = f['end']
        cid = f['cid']
        stpaul = f['stpaul']
        eb_block = f['eb_block']
        seattle = f['seattle']
        la = f['la']
        cs_block = f['cs_block']
        palmsprings = f['palmsprings']
        neworleans_end = f['neworleans_end']
        sl_block = f['sl_block']
        neworleans_start = f['neworleans_start']
        philly = f['philly']
        cr_block = f['cr_block']
        dc = f['dc']
        chicago = f['chicago']
        cl_block = f['cl_block']

    #EMPIRE BUILDER
    if 1:
        print('finding EMPIRE BUILDER routes')
        ptA = [stpaul]
        iredund = np.load(local+'redundant.npy')
        #for i in eb_block: iredund = np.append(iredund,i)
        iarr = np.array([],dtype=int)
        if not cont: np.savez(rb+'partial',ptA=ptA,iarr=iarr)
        partials = glob.glob(rb+'*.npz')
        while len(partials) > 0:
            level = 0
            with np.load(partials[0]) as f:
                ptA = f['ptA']
                iarr = f['iarr']
            os.remove(partials[0])
            route_builder.main(ptA,iarr,seattle,rb+'empire_builder',level,\
                               iredund,arrive=arrive)
            partials = glob.glob(rb+'*.npz')

    #COAST STARLIGHT
    if 0:
        print('finding COAST STARLIGHT routes')
        ptA = [seattle]
        ptB = la
        iredund = np.load(local+'redundant.npy')
        for i in cs_block:
            iredund = np.append(iredund,i)
        iarr = np.array([],dtype=int)
        if not cont: np.savez(rb+'partial',ptA=ptA,iarr=iarr)
        partials = glob.glob(rb+'*.npz')
        while len(partials) > 0:
            level = 0
            with np.load(partials[0]) as f:
                ptA = f['ptA']
                iarr = f['iarr']
            os.remove(partials[0])
            route_builder.main(ptA,iarr,ptB,rb+'coast_starlight',level,\
                               iredund,arrive=arrive)
            partials = glob.glob(rb+'*.npz')

    #SUNSET LIMITED
    if 0:
        print('finding SUNSET LIMITED routes')
        ptA = [palmsprings]
        ptB = neworleans_end
        iredund = np.load(local+'redundant.npy')
        for i in sl_block:
            iredund = np.append(iredund,i)
        iarr = np.array([],dtype=int)
        if not cont: np.savez(rb+'partial',ptA=ptA,iarr=iarr)
        partials = glob.glob(rb+'*.npz')
        while len(partials) > 0:
            level = 0
            with np.load(partials[0]) as f:
                ptA = f['ptA']
                iarr = f['iarr']
            os.remove(partials[0])
            route_builder.main(ptA,iarr,ptB,rb+'sunset_limited',\
                               level,iredund,arrive=arrive)
            partials = glob.glob(rb+'*.npz')

    #CRESCENT
    if 0:
        print('finding CRESCENT routes')
        ptA = [neworleans_start]
        ptB = philly
        iredund = np.load(local+'redundant.npy')
        for i in cr_block:
            iredund = np.append(iredund,i)
        iarr = np.array([],dtype=int)
        if not cont: np.savez(rb+'partial',ptA=ptA,iarr=iarr)
        partials = glob.glob(rb+'*.npz')
        while len(partials) > 0:
            level = 0
            with np.load(partials[0]) as f:
                ptA = f['ptA']
                iarr = f['iarr']
            os.remove(partials[0])
            route_builder.main(ptA,iarr,ptB,rb+'crescent',level,iredund,arrive=arrive)
            partials = glob.glob(rb+'*.npz')

    #CAPITOL LIMITED
    if 0:
        print('finding CAPITOL LIMITED routes')
        ptA = [dc]
        ptB = chicago
        iredund = np.load(local+'redundant.npy')
        for i in cl_block:
            iredund = np.append(iredund,i)
        iarr = np.array([],dtype=int)
        if not cont: np.savez(rb+'partial',ptA=ptA,iarr=iarr)
        partials = glob.glob(rb+'*.npz')
        while len(partials) > 0:
            level = 0
            with np.load(partials[0]) as f:
                ptA = f['ptA']
                iarr = f['iarr']
            os.remove(partials[0])
            route_builder.main(ptA,iarr,ptB,rb+'capitol_limited',level,\
                               iredund,arrive=arrive)
            partials = glob.glob(rb+'*.npz')


text:
from typing import List

import pandas
import numpy
from django.core.management import base

from eahub.profiles.models import Profile

class Command(base.BaseCommand):
    def handle(self, *args, **options):
        df_to_override = pandas.read_csv(f"data/users-to-override.csv")
        emails_to_override: List[str] = df_to_override["email"].to_list()

        for date_str in ["03-29", "04-12"]:
            df = pandas.read_csv(f"data/users-{date_str}.csv")
            emails_to_import: List[str] = df["user"].to_list()
            emails_existing = Profile.objects.filter(
                user__email__in=emails_to_import
            ).values_list("user__email", flat=True)
            emails_existing = [email for email in emails_existing]

            emails_filtered = []
            for email in emails_to_import:
                if email not in emails_existing or email in emails_to_override:
                    emails_filtered.append(email)

            df_to_import = df.loc[df["user"].isin(emails_filtered)]
            print(df.count())
            print(df_to_import.count())
            chunks_amount = len(df_to_import) // 25

            for chunk_index, chunk in enumerate(
                numpy.array_split(df_to_import, chunks_amount)
            ):
                chunk.to_csv(f"data/users-{date_str}-filtered-part{chunk_index}.csv")


text:
""" kernels.py """
import numpy as np
import chainer.links as L

def rotate3(a):
    """ rotate 3x3 array clockwise by one
    0   1   2
    3   4   5
    6   7   8
    """
    save = a[0]
    a[0] = a[1]
    a[1] = a[2]
    a[2] = a[5]
    a[5] = a[8]
    a[8] = a[7]
    a[7] = a[6]
    a[6] = a[3]
    a[3] = save
    return a

def rotation_check(a, b):
    """ Is matrix a a rotation of matrix b?
    Both must be 3x3 numpy arrays and are assumed binary, 0/1 (but should work if not)
    Returns:
        -1 if not
        0 if equal (same after 0 rotations)
        1..8 for how many rotations needed to make match

    """
    assert a.shape == (9,)
    assert b.shape == (9,)

    if sum(a) != sum(b):
        return -1
    if a[4] != b[4]:        # Center bits don't rotate, must be =
        return -1

    a = a.reshape(9)
    b = b.reshape(9)
    for i in range(8):
        if np.array_equal(a, b):
            return i
        else:
            b = rotate3(b)

    return -2        # same # of bits but not a match

def make_similiarity_matrix(kernels, k_props):
    """ How similar are two kernels?
        kernels: array of all 512 (3x3) kernels
        k_props: array of (index, rotation) for all kernels
        Returns:
        k_sim:  512 x 512 array of |a-b|

    """
    length = len(kernels)           # Should be 512
    kS = np.zeros((length, length), dtype=np.float16)
    for i in range(length):
        a = np.asarray(kernels[0:512][i][0:1][0])
        a = a.reshape(9)
        for j in range(length):
            b = np.asarray(kernels[0:512][j][0:1][0])
            b = b.reshape(9)
            if i == j:
                diff = 0
            else:
                diff = np.sum(np.abs(a - b))
            kS[i][j] = (9.0 - diff)/9.0

    return kS

def update_csm(csm, z):
    """Update avg evidence given z
        csm['p'] - Cross support matrix, probability, 512 x 512
        csm['n'] - Cross support matrix, n or count, 512 x 512
        z   - layer output, m x 512 (512=# of kernels/channels)
    Returns:
        updated csm (which is not really necessary because its just a reference)
    """
    batch_size, kernels, x_dim, y_dim = z.shape
    csm_p = csm['p']
    csm_n = csm['n']
    winner = np.argmax(z.data, axis=1)
    winner_value = np.amax(z.data, axis=1)

    for b in range(batch_size):
        for k in range(kernels):
            for x in range(x_dim):
                for y in range(y_dim):
                    n = batch_size * (kernels - 1)
                    e = np.sum(z.data, axis=1) - winner_value
                    em = np.sum(e, axis=0)
                    p = em / (batch_size * (kernels - 1))
                    #winner = z[b][k][x][y]
                    csm_n[winner] += 1

        csm_p += (y[b] - csm_p)/csm_n

def make_cross_support_matrix(size):
    """ What is the likelihood of k[i] given evidence for all k?
        Build the cross_support_matrix (csm) where each time k[i] wins
        update avg evidence for all other k.
        size    - size the array will be, also the # of kernels
        Returns:
        csm  512 x 512 array of |a-b|
    """
    csm = {'p': 0, 'r': 0}
    csm['p'] = np.zeros((size, size), dtype=np.float16)
    csm['n'] = np.zeros((size, size), dtype=np.int32)
    return csm

def similarity_check(kernels, k_props):
    """ How similar are two kernels?
        kernels: array of all 512 (3x3) kernels
        k_props: array of (index, rotation) for all kernels
        Returns:
        |argmin(sum(a-b))| across all rotations

    """
    kS = np.zeros((72,72), dtype=int)
    k_uniques = get_uniques(kernels, k_props)
    length = len(k_uniques)
    for i in range(length):
        a = np.asarray(k_uniques[i][1][0:1][0])
        a = a.reshape(9)
        for j in range(length):
            b = np.asarray(k_uniques[j][1][0:1][0])
            b = b.reshape(9)
            if i == j:
                min = 0
            else:
                min = 81                    # Max possible diff
                for r in range(8):
                    diff = np.sum(np.abs(a - b))
                    min = min if min < diff else diff
                    b = rotate3(b)
            kS[i][j] = min

    return kS

def make_array(i):
    s = str(bin(i))
    s = s[2:]
    a = np.zeros((9), dtype=np.float32)
    for x in range(9):
        if len(s) > x:
            x += 1
            if s[-x] == '1':
                index = 9 - x
                a[index] = 1

    return a

def get_uniques(k, k_props):
    c = 0
    k_uniques = []
    for i in range(len(k)):
        if k_props[i][0] == -1:
            k_uniques.append([i, k[i]])
            c += 1
    return k_uniques

def display_uniques(k, k_props):
    c = 0
    for i in range(len(k)):
        if k_props[i][0] == -1:
            display(k[i])
            c += 1
    #print(c, " uniques")

def display(a):
    b = a.reshape((3,3))
    print(b)

def make_kernels():
    """
    Returns:
        kernels: array of all 512 (3x3) kernels
        k_props: array of (index, rotation) for all kernels

    :return:
    """
    k = []
    k_props = np.zeros((512,2), dtype=np.float32)
    k_props -= 1
    for i in range(512):
        a = make_array(i)
        k.append(a)
    for i in range(512):
        for j in range(i, 512):
            if i == j:
                continue
            if np.sum(k[j]) > np.sum(k[i]):
                continue
            if k_props[j][0] != -1:
                continue                   # Has already matched
            b = np.copy(k[j])
            r = rotation_check(k[i], b)
            if r >= 0:
                k_props[j] = [i, r]
                continue

    kernels = np.reshape(k, (512,1,3,3))
    return kernels, k_props

if __name__ == '__main__':
    make_kernels()

max_stars_repo_name: pransil/capsNet

max_issues_repo_name: pransil/capsNet

max_forks_repo_name: pransil/capsNet


text:
# -*- coding: utf-8 -*-
#
from __future__ import division

import numpy
import sympy

from .helpers import _s3
from ..helpers import untangle

class LiuVinokur(object):
    """
    Y. Liu and M. Vinokur,
    Exact Integrations of Polynomials and Symmetric Quadrature Formulas over
    Arbitrary Polyhedral Grids,
    Journal of Computational Physics, 140, 122–147 (1998).
    DOI: 10.1006/jcph.1998.5884,
    <https://doi.org/10.1006/jcph.1998.5884>.
    """

    def __init__(self, index, symbolic=False):
        frac = sympy.frac if symbolic else lambda x, y: x / y
        sqrt = numpy.vectorize(sympy.sqrt) if symbolic else numpy.sqrt

        self.name = "LiuVinokur({})".format(index)
        if index == 1:
            self.degree = 1
            data = [(1, _s3(symbolic))]
        elif index == 2:
            self.degree = 1
            data = [(frac(1, 3), _r_alpha(1))]
        elif index == 3:
            self.degree = 2
            data = [(frac(1, 3), _r_alpha(-frac(1, 2)))]
        elif index == 4:
            self.degree = 2
            data = [(frac(3, 4), _s3(symbolic)), (frac(1, 12), _r_alpha(1))]
        elif index == 5:
            self.degree = 3
            data = [
                (-frac(9, 16), _s3(symbolic)),
                # ERR Wrongly specified in the article as 25 (instead of 2/5).
                (frac(25, 48), _r_alpha(frac(2, 5))),
            ]
        elif index == 6:
            self.degree = 3
            sqrt21 = sqrt(21)
            data = [
                ((1 + sqrt21) / 120, _r_alpha(1)),
                ((39 - sqrt21) / 120, _r_alpha((1 - sqrt21) / 10)),
            ]
        elif index == 7:
            self.degree = 3
            data = [
                (frac(9, 20), _s3(symbolic)),
                (frac(1, 20), _r_alpha(1)),
                (frac(2, 15), _r_alpha(-frac(1, 2))),
            ]
        elif index == 8:
            self.degree = 4
            sqrt10 = sqrt(10)
            sqrt_b = sqrt(950 - 220 * sqrt10)
            a1 = (-10 + 5 * sqrt10 + sqrt_b) / 30
            a2 = (-10 + 5 * sqrt10 - sqrt_b) / 30
            data = [
                ((5 * a2 - 2) / (60 * a1 ** 2 * (a2 - a1)), _r_alpha(a1)),
                ((5 * a1 - 2) / (60 * a2 ** 2 * (a1 - a2)), _r_alpha(a2)),
            ]
        elif index == 9:
            self.degree = 4
            data = [
                (frac(27, 80), _s3(symbolic)),
                (frac(8, 105), _r_alpha(-frac(1, 2))),
                (frac(81, 560), _r_alpha(frac(2, 3))),
            ]
        elif index == 10:
            self.degree = 4
            sqrt13 = sqrt(13)
            data = [
                ((11 - 1 * sqrt13) / 360, _r_alpha(1)),
                ((80 - 16 * sqrt13) / 360, _r_alpha(-frac(1, 2))),
                ((29 + 17 * sqrt13) / 360, _r_alpha((-1 + sqrt13) / 6)),
            ]
        elif index == 11:
            self.degree = 4
            sqrt3 = sqrt(3)
            data = [
                (+frac(9, 20), _s3(symbolic)),
                (-frac(1, 60), _r_alpha(1)),
                (+frac(1, 10), _r_gamma_delta((3 + sqrt3) / 6, (3 - sqrt3) / 6)),
            ]
        elif index == 12:
            self.degree = 5
            sqrt15 = sqrt(15)
            data = [
                (frac(9, 40), _s3(symbolic)),
                ((155 - sqrt15) / 1200, _r_alpha((1 + sqrt15) / 7)),
                ((155 + sqrt15) / 1200, _r_alpha((1 - sqrt15) / 7)),
            ]
        else:
            assert index == 13
            self.degree = 5
            data = [
                (frac(81, 320), _s3(symbolic)),
                (frac(1, 90), _r_alpha(1)),
                (frac(16, 225), _r_alpha(-frac(1, 2))),
                (frac(2401, 14400), _r_alpha(frac(4, 7))),
            ]

        self.bary, self.weights = untangle(data)
        self.points = self.bary[:, 1:]
        return

def _r_alpha(alpha):
    """From the article:

    mu_i = (1 + (n-1) alpha) / n,
    mu_j = (1 - alpha) / n    for j!=i,

    where n is the number of vertices
    """
    a = (1 + 2 * alpha) / 3
    b = (1 - alpha) / 3
    return numpy.array([[a, b, b], [b, a, b], [b, b, a]])

def _r_gamma_delta(gamma, delta):
    """From the article:

    mu_i = (1 + (n-1) gamma - delta) / n,
    mu_j = (1 + (n-1) delta - gamma) / n,
    mu_k = (1 - gamma - delta) / n    for k!=i, k!=j,

    where n is the number of vertices
    """
    a = (1 + 2 * gamma - delta) / 3
    b = (1 + 2 * delta - gamma) / 3
    c = (1 - gamma - delta) / 3
    return numpy.array(
        [[a, b, c], [c, a, b], [b, c, a], [a, c, b], [b, a, c], [c, b, a]]
    )

max_stars_repo_name: melvyniandrag/quadpy

max_issues_repo_name: melvyniandrag/quadpy

max_forks_repo_name: melvyniandrag/quadpy

reason: import numpy,import sympy


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Module for tools used in vaspy
"""

import bz2
from itertools import zip_longest
import os
import re
import numpy as np
from typing import List, Iterable, Sequence, Tuple, Union, IO, Any, Optional

def open_by_suffix(filename: str) -> IO[str]:
    """Open file."""
    if os.path.splitext(filename)[1] == ".bz2":
        thefile = bz2.open(filename, mode="rt")
    else:
        thefile = open(filename, mode="rt")
    return thefile

def each_slice(
    iterable: Iterable, n: int, fillvalue: Optional[float] = None
) -> Iterable[Any]:
    """each_slice(iterable, n[, fillvalue]) => iterator

    make new iterator object which get n item from [iterable] at once.
    """
    args = [iter(iterable)] * n
    return zip_longest(*args, fillvalue=fillvalue)

_RERANGE = re.compile(r"(\d+)-(\d+)")
_RESINGLE = re.compile(r"\d+")

def atom_selection_to_list(
    input_str: str, number: bool = True
) -> List[Union[int, str]]:
    """Return list of ordered "String" represents the number.

    Parameters
    ----------
    input_str: str
        range of the atoms. the numbers deliminated by "-" or ","

    Returns
    --------
    list
        ordered "String" represents the number.

    Example
    --------

    >>> atom_selection_to_list("1-5,8,8,9-15,10", False)
    ['1', '10', '11', '12', '13', '14', '15', '2', '3', '4', '5', '8', '9']
    >>> atom_selection_to_list("1-5,8,8,9-15,10")
    [1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, 14, 15]

    """
    array = input_str.split(",")
    output = set()
    for each in array:
        if re.search(_RERANGE, each):
            start, stop = re.findall(_RERANGE, each)[0]
            # Version safety
            output |= set(str(i) for i in range(int(start), int(stop) + 1))
        elif re.search(_RESINGLE, each):
            output.add(each)
    if number:
        return sorted(int(i) for i in output)
    return sorted(list(output))

def atomtypes_atomnums_to_atoms(
    atomtypes: Iterable[str], atomnums: Iterable[int]
) -> Tuple[str, ...]:
    """Return list representation for atom in use.

    Parameters
    ------------
    atomtypes: list
        atom names
    atomnums: list
        atom numbers

    Examples
    --------
    >>> test_nums = [2, 3, 2, 1]
    >>> test_elements = ['Si', 'Ag', 'H', 'Si']
    >>> atomtypes_atomnums_to_atoms(test_elements, test_nums)
    ('Si', 'Si', 'Ag', 'Ag', 'Ag', 'H', 'H', 'Si')

    """
    atoms = []
    for elem, nums in zip(atomtypes, atomnums):
        for _ in range(nums):
            atoms.append(elem)
    return tuple(atoms)

def atoms_to_atomtypes_atomnums(atoms: List[str]) -> Tuple[List[str], List[int]]:
    r"""Return atomnums and atomtypes list.

    Returns
    --------
    atomnums
        list of number of atoms
    atomtypes
        list of atomnames

    Examples
    --------
    >>> test = ['Si', 'Si', 'Ag', 'Ag', 'Ag', 'H', 'H', 'Si']
    >>> atoms_to_atomtypes_atomnums(test)
    (['Si', 'Ag', 'H', 'Si'], [2, 3, 2, 1])

    """
    thelast = ""
    atomnums: List[int] = []
    atomtypes: List[str] = []
    while atoms:
        atom = atoms.pop(0)
        if thelast == atom:
            atomnums[-1] = atomnums[-1] + 1
        else:
            atomnums.append(1)
            atomtypes.append(atom)
        thelast = atom
    return atomtypes, atomnums

def cuboid(crystal_axes: Union[Sequence[List[float]], np.ndarray]) -> np.ndarray:
    """Return the coordinates for cuboid that includes tetrahedron represented by vectors.

    Parameters
    ------------
    vectors: array-like.
        Three vectors for tetrahedron.  (Crystal axis a,b,c)

    Return
    """
    a = np.array(crystal_axes[0])
    b = np.array(crystal_axes[1])
    c = np.array(crystal_axes[2])
    o = np.array((0, 0, 0))
    points = np.array((o, a, b, c, a + b, a + c, b + c, a + b + c))
    box = np.array((np.min(points.T, axis=1), np.max(points.T, axis=1))).T
    return box
    """return np.array(
        (
            (np.array([p[1] - p[0] for p in box])[0], 0, 0),
            (0, np.array([p[1] - p[0] for p in box])[1], 0),
            (0, 0, np.array([p[1] - p[0] for p in box])[2]),
        )
    )"""

if __name__ == "__main__":
    import argparse

    def EACH_SLICE_DEMO(L: Sequence[Any], n: int) -> List[Any]:
        return list(each_slice(L, n))

    demo = {
        "EACH_SLICE_DEMO": (range(10), 3),
        "removeall": ([1, 0, 0, 1, 0, 1, 0, 0], 0),
        "flatten": (
            (1, [range(2), 3, set([4, 5]), [6]], frozenset([7, 8])),  # Version safety
        ),
        "parse_Atomselection": ("1-5,8,9,11-15",),
    }
    argcounts = {
        "EACH_SLICE_DEMO": 2,
        "removeall": 2,
        "flatten": 1,
        "parse_Atomselection": 1,
    }
    available = ["all"] + list(demo.keys())
    parser = argparse.ArgumentParser(
        description="""collection of tools used in this package.""",
        formatter_class=argparse.RawTextHelpFormatter,
        epilog="""-a/--args option arguments are interpleted by eval().
so strings must be given with quotations("" or '').
Because command line regards spaces as break,
list argument must be written without any space.
(i.e. [1,2,3,4,5] is valid, while [1, 2, 3, 4, 5] is invalid.)""",
    )
    parser.add_argument(
        "choice",
        metavar="funcname",
        nargs="+",
        choices=available,
        help="""Demonstrate choosen function.
*all* shows all function in the choice.
If -a/--args option is given, get argument(s) from command line.
Otherwise use prepared argument(s).""",
    )
    parser.add_argument(
        "-a",
        "--args",
        metavar="values",
        nargs="+",
        action="append",
        dest="values",
        help="""Use given argument(s) for demonstration.
You have to use this option for each function.
See epilog for notices for argument notation.""",
    )
    args = parser.parse_args()

    if "all" in args.choice:
        choices = demo.keys()
    else:
        choices = args.choice
    index = 0
    for func in choices:
        if args.values is None or len(args.values) <= index:
            values = demo[func]
        else:
            if argcounts[func] != len(args.values[index]):
                print(
                    """argument number not match (require {0}, given {1})
use default values.""".format(
                        argcounts[func], len(args.values[index])
                    )
                )
                values = demo[func]
            else:
                values = [eval(s) for s in args.values[index]]
            index += 1
        print("Demonstrate function {0}()\ninput:  ".format(func), end="")
        print(*values, sep=" : ")
        print("output: ", end="")
        print(vars()[func](*values))

max_stars_repo_name: arafune/vaspy

max_issues_repo_name: arafune/vaspy

max_forks_repo_name: arafune/vaspy


text:
import torch
import numpy as np

PIXELS_PER_WORLD = 5.5
HEIGHT = 144
WIDTH = 256
FOV = 90
MAP_SIZE = 256
CAM_HEIGHT = 1.3

class Converter(torch.nn.Module):
    def __init__(
            self, w=WIDTH, h=HEIGHT, fov=FOV,
            map_size=MAP_SIZE, pixels_per_world=PIXELS_PER_WORLD,
            hack=0.4, cam_height=CAM_HEIGHT):
        super().__init__()

        F = w / (2 * np.tan(fov * np.pi / 360))

        self.map_size = map_size
        self.pixels_per_world = pixels_per_world
        self.w = w
        self.h = h
        self.fy = F
        self.fx = 1.1 * F
        self.hack = hack
        self.cam_height = cam_height

        self.register_buffer('position', torch.FloatTensor([map_size // 2, map_size + 1]))

    def forward(self, map_coords):
        return self.map_to_cam(map_coords)

    def map_to_cam(self, map_coords):
        world_coords = self.map_to_world(map_coords)
        cam_coords = self.world_to_cam(world_coords)

        return cam_coords

    def map_to_world(self, pixel_coords):
        relative_pixel = pixel_coords - self.position
        relative_pixel[..., 1] *= -1

        return relative_pixel / self.pixels_per_world

    def cam_to_map(self, points):
        world_coords = self.cam_to_world(points)
        map_coords = self.world_to_map(world_coords)

        return map_coords

    def cam_to_world(self, points):
        z = (self.fy * self.cam_height) / (points[..., 1] - self.h / 2)
        x = (points[..., 0] - self.w / 2) * (z / self.fx)
        y = z - self.hack

        result = torch.stack([x, y], points.ndim-1)
        result = result.reshape(*points.shape)

        return result

    def world_to_cam(self, world):
        z = world[..., 1] + self.hack
        x = (self.fx * world[..., 0]) / z + self.w / 2
        y = (self.fy * self.cam_height) / z + self.h / 2

        result = torch.stack([x, y], world.ndim-1)
        result[..., 0] = torch.clamp(result[..., 0], 0, self.w-1)
        result[..., 1] = torch.clamp(result[..., 1], 0, self.h-1)
        result = result.reshape(*world.shape)

        return result

    def world_to_map(self, world):
        map_coord = world * self.pixels_per_world
        map_coord[..., 1] *= -1
        map_coord += self.position

        return map_coord

max_stars_repo_name: AIasd/ADFuzz

max_issues_repo_name: scope-lab-vu/Resonate-Dynamic-Risk

max_forks_repo_name: AIasd/ADFuzz


text:
import math

import numpy as np
from numba import njit

from .shape import Shape
from .shape import merge_bounds
from .triangle import rasterize_triangle

class Quadrangle(Shape):
    PARAMS_LEN = 8

    def __init__(self, points, alpha):
        super().__init__()
        self.points = points.astype(np.int64)
        self.alpha = alpha

    def __repr__(self):
        return f'Quadrangle(A={self.points[0]}, B={self.points[1]}, C={self.points[2]}, ' \
            f'D={self.points[3]})'

    @staticmethod
    def _random(w, h, alpha, rng, scale=1):
        r = rng.randint(min(w, h) // 2)
        cx = rng.randint(w)
        cy = rng.randint(h)
        angles = np.sort(rng.uniform(0, 2 * math.pi, 4))
        xs = [cx + r * math.cos(angle) for angle in angles]
        ys = [cy + r * math.sin(angle) for angle in angles]
        points = np.concatenate((np.array(xs).reshape(4, 1), np.array(ys).reshape(4, 1)), axis=1)
        assert points.shape == (4, 2), f'Shape of points: {points.shape}, expected: (4, 2)'
        if scale != 1:
            center = np.sum(points, axis=0) / 4
            random_shift = [rng.randint(w), rng.randint(h)]
            points = scale * (points - center) + random_shift
        return Quadrangle(points, alpha)

    @staticmethod
    def _from_params(*params):
        return Quadrangle(points=np.array(params[:-1]).reshape(4, 2), alpha=params[-1])

    @staticmethod
    def _from_normalized_params(w, h, *params):
        points = np.empty(shape=(4, 2))
        points[:, 0] = [int(x * w) for x in params[:-1][::2]]
        points[:, 1] = [int(y * h) for y in params[:-1][1::2]]
        return Quadrangle(
            points=points,
            alpha=params[-1]
        )

    @staticmethod
    def without_edge(points, edge, alpha):
        remove_edge(points=points, edge=np.array(edge))
        return Quadrangle(points=points, alpha=alpha)

    @staticmethod
    def _params_intervals():
        return lambda w, h: np.array([w, h, w, h, w, h, w, h])

    def get_bounds(self, h=None, w=None):
        return rasterize_quadrangle(self.points)

    def get_alpha(self):
        return self.alpha

    def _params(self):
        return self.points.reshape(-1, ).astype(np.float64)

    def _normalized_params(self, w, h):
        return np.append(
            self.points.reshape(-1, ).astype(np.float64) / np.array([w, h, w, h, w, h, w, h]),
            self.alpha
        )

class Rectangle(Shape):
    PARAMS_LEN = 5

    def __init__(self, cx, cy, w, h, rotation, alpha):
        super().__init__()
        self.cx = int(cx)
        self.cy = int(cy)
        self.w = max(1, int(w))
        self.h = max(1, int(h))
        self.rotation = rotation
        self.alpha = alpha

    def __repr__(self):
        return f'Rectangle(cx={self.cx}, cy={self.cy}, w={self.w}, h={self.h}, rotation={self.rotation})'

    @staticmethod
    def _random(w, h, alpha, rng, scale=1):
        cx = rng.randint(w)
        cy = rng.randint(h)
        rw = rng.randint(1, w) * scale
        rh = rng.randint(1, h) * scale
        rot = rng.uniform(0, math.pi)
        return Rectangle(cx=cx, cy=cy, w=rw, h=rh, rotation=rot, alpha=alpha)

    @staticmethod
    def _from_params(*params):
        return Rectangle(*params)

    @staticmethod
    def _from_normalized_params(w, h, *params):
        return Rectangle(
            cx=int(params[0] * w),
            cy=(params[1] * h),
            w=(params[2] * w),
            h=(params[3] * h),
            rotation=params[4] * math.pi,
            alpha=params[5]
        )

    def get_bounds(self, h=None, w=None):
        return rasterize_rectangle(self.cx, self.cy, self.w, self.h, self.rotation)

    def get_alpha(self):
        return self.alpha

    def _params(self):
        return np.array([self.cx, self.cy, self.w, self.h, self.rotation], dtype=np.float64)

    def _normalized_params(self, w, h):
        return np.append(
            np.array([self.cx, self.cy, self.w, self.h, self.rotation], dtype=np.float64) /
            np.array([w, h, w, h, math.pi]),
            self.alpha
        )

    @staticmethod
    def _params_intervals():
        return lambda w, h: np.array([w, h, w - 1, h - 1, math.pi])

@njit("i8[:,:](i8[:,:])")
def rasterize_quadrangle(points):
    bounds1 = rasterize_triangle(points[:-1])
    triangle2_points = np.empty(shape=(3, 2), dtype=np.int64)
    triangle2_points[0] = points[0]
    triangle2_points[1] = points[2]
    triangle2_points[2] = points[3]
    bounds2 = rasterize_triangle(triangle2_points)
    bounds1_len = bounds1.shape[0]
    bounds = np.empty(shape=(bounds1_len + bounds2.shape[0], 3), dtype=np.int64)
    bounds[:bounds1_len] = bounds1
    bounds[bounds1_len:] = bounds2
    return merge_bounds(bounds)

@njit("f8(f8)")
def deg_to_rad(deg):
    return deg * math.pi / 180

@njit("f8(f8)")
def rad_to_deg(rad):
    return rad * 180 / math.pi

@njit("i8[:,:](i8,i8,i8,i8,f8)")
def rasterize_rectangle(cx, cy, w, h, rot):
    a = (rot - deg_to_rad(90)) + math.asin(h / (math.sqrt(h * h + w * w)))
    b = (math.sqrt(h * h + w * w) / 2)
    x1 = cx + math.sin(a) * b
    y1 = cy - math.cos(a) * b
    x2 = x1 + math.cos(rot) * w
    y2 = y1 + math.sin(rot) * w
    c = deg_to_rad(rad_to_deg(rot) + 90)
    x3 = x2 + math.cos(c) * h
    y3 = y2 + math.sin(c) * h
    x4 = x1 + math.cos(c) * h
    y4 = y1 + math.sin(c) * h
    points = np.empty(shape=(4, 2), dtype=np.int64)
    points[0] = x1, y1
    points[1] = x2, y2
    points[2] = x3, y3
    points[3] = x4, y4
    return rasterize_quadrangle(points)

@njit("(i8[:,:],i8,i8)")
def move_point(points, index_start, index_end):
    start = points[index_start]
    end = points[index_end]
    sign = start - end

    if sign[1] == 0 or abs(sign[0]) > abs(sign[1]):
        new_y = start[1]
        new_x = start[0] + 1 if sign[0] < 0 else start[0] - 1
    elif sign[1] > 0:
        new_y = start[1] - 1
        new_x = start[0]
    else:
        new_y = start[1] + 1
        new_x = start[0]

    points[index_start, 0] = new_x
    points[index_start, 1] = new_y

@njit("(i8[:,:],i8[:])")
def remove_edge(points, edge):
    move_point(points, edge[0], (edge[0] + 3) % 4)
    move_point(points, edge[1], (edge[1] + 1) % 4)

max_stars_repo_name: trejsu/shaper

max_issues_repo_name: trejsu/shaper

max_forks_repo_name: trejsu/shaper

reason: import numpy,from numba


text:
import pyomo.environ as pyo
import numpy as np
import time
from pyomo.environ import *
from pyomo.opt import SolverFactory

model = pyo.ConcreteModel()

model.x = pyo.Var(bounds=(-np.inf,3))
model.y = pyo.Var(bounds=(0,np.inf))

x = model.x
y = model.y

model.C1 = pyo.Constraint(expr= x+y<=8)
model.C2 = pyo.Constraint(expr= 8*x+3*y>=-24)
model.C3 = pyo.Constraint(expr= -6*x+8*y<=48)
model.C4 = pyo.Constraint(expr= 3*x+5*y<=15)

model.obj = pyo.Objective(expr= -4*x-2*y, sense=minimize)

tempo_inicial = time.time()
opt = SolverFactory('glpk')
opt.solve(model)
tempo = time.time()-tempo_inicial

x_value = pyo.value(x)
y_value = pyo.value(y)

print('tempo:',tempo)
print('x:',x_value)
print('y:',y_value)

reason: import numpy,import pyomo,from pyomo


text:
from matrx.agents.capabilities.capability import SenseCapability
from matrx.actions.action import Action, ActionResult
from matrx.utils.utils import get_all_classes
from matrx.objects.env_object import EnvObject
import numpy as np

class AgentBody(EnvObject):

    def __init__(self, location, possible_actions, sense_capability, class_callable,
                 callback_agent_get_action, callback_agent_set_action_result, callback_agent_observe,
                 callback_agent_get_messages, callback_agent_set_messages, callback_agent_initialize,
                 callback_agent_log,
                 visualize_size, visualize_shape, visualize_colour, visualize_depth, visualize_opacity,
                 is_traversable, team, name, is_movable,
                 is_human_agent, customizable_properties,
                 **custom_properties):
        """
        This class is a representation of an agent's body in the GridWorld.

        It is used as a measure to keep the AgentBrain code and Environment code separate. This AgentBody is used by
        the environment to update the GUI, perform actions, and update properties. It is kept in sync with the Agent's
        brain every iteration.

        It inherits from EnvObject which allows you set any custom properties you want. In addition it also has all the
        mandatory properties of an EnvObject plus a few extra. Which is the team name the agent is part of (if any) and
        what the Agent's body is carrying.

        In addition the Agent's body keeps a set of callbacks to methods inside the Agent. This forms the connection
        between the GridWorld (that calls them) and the Agent (that defined them).

        :param location: List or tuple of length two. Mandatory. The location of the Agent's body in the grid world.
        :param possible_actions: The list of Action class names this agent may be able to perform. This allows you to
        create agents that can only perform a couple of the available actions.
        :param sense_capability: The SenseCapability object.
        :param class_callable: The Agent class; in other words, the class of the agent's brain. This is stored here so
        that the Visualizer (which visualizes an agent based on this Agent's body object) and agents knows what kind of agent
        it is. Allows you to visualize certain agent types in a certain way.

        :param callback_agent_get_action: The callback function as defined by the Agent instance of which this is an
        Agent's body of. It is called each tick by the GridWorld. As such the GridWorld determines when the Agent can
        perform an action by calling this function which is stored in the Agent's Agent's body.
        :param callback_agent_set_action_result: Same as the callback_get_action but is used by GridWorld to set the
        ActionResult object in the Agent after performing the action. This allows the Agent to know how its planned
        action went.
        :param callback_agent_observe: Similar to callback_agent_get_action, is used by GridWorld to obtain the
        processed state dictionary of the Agent. As the GridWorld does not know exactly what the Agent is allowed to
        see or not, the 'observe' preprocesses the given state further. But to accurately visualize what the agent sees
        we have to obtain that pre-processed state, which is done through this callback.
        :param callback_agent_initialize: Call the initialize function in an Agent's Brain. Is done at the initialize
        of a GridWorld.

        :param callback_agent_get_messages: A callback function that allows the GridWorld to obtain the agent's messages
        that need to be send to different agents.
        :param callback_agent_set_messages: A callback function that allows the GridWorld to set any message send by
        some agent to a list of received messages in an agent.

        :param name: String Defaults to "Agent". The name of the agent, does not need to be unique.
        :param is_human_agent: Boolean. Defaults to False. Boolean to signal that the agent represented by this Agent's
        body is a human controlled agent.
        :param customizable_properties: List. Optional, default obtained from defaults.json. The list of attribute names
        that can be customized by other objects (including Agent's body and as an extension any Agent).
        :param is_traversable: Boolean. Optional, default obtained from defaults.json. Signals whether other objects can
        be placed on top of this object.
        :param carried_by: List. Optional, default obtained from defaults.json. A list of who is carrying this object.
        :param team: The team name the agent is part of. Defaults to the team name similar to the Agent's body unique
        ID, as such denoting that by default each Agent's body belongs to its own team and as an extension so does its
        "brain" the Agent.

        :param visualize_size: Float. Optional, default obtained from defaults.json. A visualization property used by
        the Visualizer. Denotes the size of the object, its unit is a single grid square in the visualization (e.g. a
        value of 0.5 is half of a square, object is in the center, a value of 2 is twice the square's size centered on
        its location.)
        :param visualize_shape: Int. Optional, default obtained from defaults.json. A visualization property used by the
        Visualizer. Denotes the shape of the object in the visualization.
        :param visualize_colour: Hexcode string. Optional, default obtained from defaults.json. A visualization property
        used by the Visualizer. Denotes the
        :param visualize_depth: Integer. Optional, default obtained from defaults.json. A visualization property that
        is used by the Visualizer to draw objects in layers.
        :param visualize_opacity: Integer. Opacity of object. Between 0.0 and 1.0.
        :param **custom_properties: Optional. Any other keyword arguments. All these are treated as custom attributes.
        For example the property 'heat'=2.4 of an EnvObject representing a fire.
        """

        # A list of EnvObjects or any class that inherits from it. Denotes all objects the Agent's body is currently
        # carrying. Note that these objects do not exist on the WorldGrid anymore, so removing them in this list deletes
        # them permanently.
        self.is_carrying = []  # list of EnvObjects that this object carries

        # The property that signals whether the agent this Agent's body represents is a human agent
        self.is_human_agent = is_human_agent

        # Save the other attributes the GridWorld expects an Agent's body to have access to an Agent's brain
        self.get_action_func = callback_agent_get_action
        self.set_action_result_func = callback_agent_set_action_result
        self.filter_observations = callback_agent_observe
        self.get_messages_func = callback_agent_get_messages
        self.set_messages_func = callback_agent_set_messages
        self.get_log_data = callback_agent_log
        self.brain_initialize_func = callback_agent_initialize

        # Set all mandatory properties
        self.is_traversable = is_traversable
        self.sense_capability = sense_capability
        self.action_set = possible_actions
        self.is_movable = is_movable

        # Set visualization properties
        self.visualize_depth = visualize_depth
        self.visualize_colour = visualize_colour
        self.visualize_shape = visualize_shape
        self.visualize_size = visualize_size
        self.visualize_opacity = visualize_opacity

        # Parse the action_set property if set to the wildcard "*" denoting all actions
        if self.action_set == "*":
            self.action_set = list(get_all_classes(Action, omit_super_class=True).keys())

        # Defines an agent is blocked by an action which takes multiple time steps. Is updated based on the speed with
        # which an agent can perform actions.
        self.__is_blocked = False

        # Place holders for action information
        self.__current_action = None
        self.__current_action_args = None

        # Denotes the last action performed by the agent, at what tick and how long it must take. Set to -infinite, such
        # that the agent is not deemed 'busy' at tick 0. Edit: changed from -inf to 1000000, to be JSON serializable
        self.__last_action_duration_data = {"duration_in_ticks": -1000000, "tick": -1000000, "action_name": None,
                                            "action_result": None}

        # We set a placeholder for the 'team' property so that it can be found in self.properties
        self.team = ""

        # Call the super constructor (we do this here because then we have access to all of EnvObject, including a
        # unique id
        super().__init__(location, name, customizable_properties=customizable_properties, is_traversable=is_traversable,
                         class_callable=class_callable,
                         visualize_size=visualize_size, visualize_shape=visualize_shape,
                         visualize_colour=visualize_colour, visualize_depth=visualize_depth,
                         visualize_opacity=visualize_opacity,
                         **custom_properties)

        # the GUI cannot differentiate capital letters (in the url), so make the agent ID lowercase
        self.obj_id = self.obj_id.lower()

        # If there was no team name given, the Agent's body (and as an extension its Agent's brain) is part of its own
        # team which is simply its object id + "_team". For this we need the object id, which was made in the EnvObject
        # constructor, that is why we call this AFTER calling that.
        if team is None:
            self.team = self.obj_id + "_team"
        self.change_property("team", self.team)

    def _set_agent_busy(self, curr_tick, action_duration):
        """
        specify the duration of the action in ticks currently being executed by the
        agent, and its starting tick
        """
        self.__last_action_duration_data = {"duration_in_ticks": action_duration, "tick": curr_tick}

    def _check_agent_busy(self, curr_tick):
        """
        check if the agent is done with executing the action
        """
        self.__is_blocked = curr_tick <= (self.current_action_tick_started + self.current_action_duration_in_ticks)

        return self.__is_blocked

    def _at_last_action_duration_tick(self, curr_tick):
        """ Returns True if this agent is at its last tick of the action's duration."""
        is_last_tick = curr_tick == (self.current_action_tick_started + self.current_action_duration_in_ticks)
        return is_last_tick

    def _get_duration_action(self):
        """ Returns the action we are waiting for 'self.current_action_duration_in_ticks', gets called in the GridWorld
        when we are at the last tick on which we should wait (see self._at_last_action_duration_tick)."""
        action_name = self.current_action
        action_kwargs = self.current_action_args

        return action_name, action_kwargs

    def _set_current_action(self, action_name, action_args):
        """
        Sets the current action of the agent. Since the GridWorld performs the mutate of an action first, and then waits
        for the duration to pass, we also have the result available.
        """
        self.__current_action = action_name
        self.__current_action_args = action_args

    def _set_agent_changed_properties(self, props: dict):
        """
        The Agent has possibly changed some of its properties during its OODA loop. Here the agent properties are also
        updated in the Agent's body, if it is allowed to change them as defined in 'customizable_properties' list.
        """
        # get all agent properties of this Agent's body in one dictionary
        body_properties = self.properties

        # check for each property if it has been changed by the agent, and if we need
        # to update our local copy (here in Agent's body) of the agent properties to match that
        for prop in props.keys():
            if prop not in body_properties.keys():
                raise Exception(f"Agent {self.obj_id} tried to remove the property {prop}, which is not allowed.")

            # check if the property has changed, and skip if not the case
            if props[prop] == body_properties[prop]:
                continue

            # if the agent has changed the property, check if the agent has permission to do so
            if prop not in self.customizable_properties:
                raise Exception(f"Agent {self.obj_id} tried to change a non-writable property: {prop}.")

            # The agent changed the property and the agent had permission to do so
            # update special properties
            self.change_property(prop, props[prop])

    def change_property(self, property_name, property_value):
        """
        Changes the value of an existing (!) property.
        :param property_name: The name of the property.
        :param property_value:  The value of the property.
        :return: The new properties.
        """

        # We check if it is a custom property and if so change it simply in the dictionary
        if property_name in self.custom_properties.keys():
            self.custom_properties[property_name] = property_value
        else:  # else we need to check if property_name is a mandatory class attribute that is also a property
            if property_name == "is_traversable":
                assert isinstance(property_value, bool)
                self.is_traversable = property_value
            elif property_name == "name":
                assert isinstance(property_value, str)
                self.obj_name = property_value
            elif property_name == "location":
                assert isinstance(property_value, list) or isinstance(property_value, tuple)
                self.location = property_value
            elif property_name == "class_inheritance":
                assert isinstance(property_value, list)
                self.class_inheritance = property_value
            elif property_name == "visualize_size":
                assert isinstance(property_value, int)
                self.visualize_size = property_value
            elif property_name == "visualize_colour":
                assert isinstance(property_value, str)
                self.visualize_colour = property_value
            elif property_name == "visualize_opacity":
                assert isinstance(property_value, int)
                self.visualize_opacity = property_value
            elif property_name == "visualize_shape":
                assert isinstance(property_value, int)
                self.visualize_shape = property_value
            elif property_name == "visualize_depth":
                assert isinstance(property_value, int)
                self.visualize_depth = property_value
            elif property_name == "team":
                assert isinstance(property_value, str)
                self.team = property_value
            elif property_name == "sense_capability":
                assert isinstance(property_value, SenseCapability)
                self.sense_capability = property_value
            elif property_name == "is_human_agent":
                assert isinstance(property_value, bool)
                self.is_human_agent = property_value
            elif property_name == "action_set":
                assert isinstance(property_value, list)
                self.action_set = property_value
            elif property_name == "is_movable":
                assert isinstance(property_value, bool)
                self.is_movable = property_value
            # We deliberately ignore the current_action property, and several others such as agent_id as these can never
            # be altered as they are governed by the GridWorld

        return self.properties

    @property
    def location(self):
        """
        We override the location pythonic property here so we can override its setter.
        :return: The location tuple of the form; (x, y).
        """
        return tuple(self.__location)

    @location.setter
    def location(self, loc):
        """
        Overrides the setter of the location (pythonic) property so we can transfer also all carried objects with us
        on any location change made anywhere.
        :param loc:
        :return:
        """
        assert isinstance(loc, list) or isinstance(loc, tuple)
        assert len(loc) == 2
        # Set the location to our private location xy list
        self.__location = loc

        # Carrying action is done here
        # First we check if we even have a 'carrying' property, as the future might hold an Agent's body who
        # specifically removes this property. In that case we return.
        if 'carrying' not in self.properties.keys():
            return
        # Next we retrieve whatever it is the Agent's body is carrying (if we have a 'carrying' property at all)
        carried_objs = self.properties['carrying']
        # If we carry nothing, we are done
        if len(carried_objs) == 0:
            return
        # Otherwise we loop over all objects and adjust their location accordingly (since these are also EnvObjects,
        # their setter for location gets called, in the case we are carrying an Agent's body this setter is called
        for obj in carried_objs:
            obj.location = loc  # this requires all objects in self.properties['carrying'] to be of type EnvObject

    @property
    def properties(self):
        """
        Returns the custom properties of this object, but also any mandatory properties such as location, name,
        is_traversable and all visualization properties (those are in their own dictionary under 'visualization').

        In the case we return the properties of a class that inherits from EnvObject, we check if that class has

        :return: All mandatory and custom properties in a dictionary.
        """

        # Copy the custom properties
        properties = self.custom_properties.copy()

        # Add all mandatory properties. Make sure that these are updated if one are added to the constructor!
        properties['team'] = self.team
        properties['name'] = self.obj_name
        properties['obj_id'] = self.obj_id  # we return id as well, but this should never ever be modified!
        properties['location'] = self.location
        properties['is_movable'] = self.is_movable
        properties['action_set'] = self.action_set
        properties['carried_by'] = self.carried_by
        properties['is_human_agent'] = self.is_human_agent
        properties['is_traversable'] = self.is_traversable
        properties['class_inheritance'] = self.class_inheritance
        properties['is_blocked_by_action'] = self.is_blocked
        properties['is_carrying'] = [obj.properties for obj in self.is_carrying]
        properties['sense_capability'] = self.sense_capability.get_capabilities()
        properties['visualization'] = {
            "size": self.visualize_size,
            "shape": self.visualize_shape,
            "colour": self.visualize_colour,
            "depth": self.visualize_depth,
            "opacity": self.visualize_opacity
        }

        # Add the current action and all of its data
        properties['current_action'] = self.current_action
        if self.current_action is not None:  # all None actions are 'idle' actions and have no name or result
            properties['current_action_args'] = self.current_action_args  # the action arguments
        else:
            properties['current_action_args'] = {}

        properties['current_action_duration'] = self.current_action_duration_in_ticks
        properties['current_action_started_at_tick'] = self.current_action_tick_started

        return properties

    @properties.setter
    def properties(self, property_dictionary: dict):
        """
        Here to protect the 'properties' variable. It does not do anything and should not do anything!
        """
        pass

    @property
    def current_action(self):
        return self.__current_action

    @property
    def current_action_duration_in_ticks(self):
        return self.__last_action_duration_data["duration_in_ticks"]

    @property
    def current_action_tick_started(self):
        return self.__last_action_duration_data["tick"]

    @property
    def current_action_args(self):
        return self.__current_action_args

    @property
    def is_blocked(self):
        return self.__is_blocked

max_stars_repo_name: thaije/matrx

max_issues_repo_name: thaije/matrx

max_forks_repo_name: matrxs/MATRXS


text:
"""Test if dependencies are present in the system."""

import shutil
import unittest

class Tests(unittest.TestCase):

    def test_cython(self):
        import cython

    def test_numpy(self):
        import numpy as np
        self.assertIsNotNone(np.zeros((10, 10), dtype=int))

    @unittest.skip('not ready yet')
    def test_pyopencl(self):
        import pyopencl

    def test_gfortran(self):
        gfortran_path = shutil.which('gfortran')
        self.assertIsNotNone(gfortran_path)

    def test_swig(self):
        swig_path = shutil.which('swig')
        self.assertIsNotNone(swig_path)

    def test_typed_ast(self):
        from typed_ast import ast3
        self.assertGreaterEqual(ast3.LATEST_MINOR_VERSION, 6)

max_stars_repo_name: EraYaN/transpyle

max_issues_repo_name: EraYaN/transpyle

max_forks_repo_name: EraYaN/transpyle


text:
from __future__ import division
import numpy as np
from pycuda.compiler import SourceModule
import pycuda.autoinit
from pycuda import gpuarray
import pycuda.driver as drv

AtomicCode='''
__global__ void atomic_ker(int* add_out, int* max_out)
{
    int tid = blockIdx.x * blockDim.x + threadIdx.x;
    atomicExch(add_out, 0);
    __syncthreads();
    atomicAdd(add_out, 1);
    atomicMax(max_out, tid);
}
'''

atomic_mod = SourceModule(AtomicCode)
atomic_ker = atomic_mod.get_function('atomic_ker')

add_out = gpuarray.empty((1,), dtype=np.int32)
max_out = gpuarray.empty((1,), dtype=np.int32)

atomic_ker(add_out, max_out, grid=(1, 1, 1), block=(100, 1, 1))

print("Atomic operations test:")
print("add_out: {}".format(add_out.get()[0]))
print("max_out: {}".format(max_out.get()[0]))

reason: import numpy,import pycuda,from pycuda


text:
"""
Core module for computer visionary stuffs
---
@author TaoPR (github.com/starcolon)
"""

import numpy as np
from PIL import Image
from .filters import *

def get_sample_dim(): # n
  return 240 # total vector dimensionality : n^2 * 3

def generate_filtered(sample,max_combination=5):
  # Make transformation functions
  ts = random_filters(max_combination)
  # Apply generated transformations
  results = [transform(t)(sample) for t in ts]
  return ts, results

def load_img(path):
  return Image.open(path)

# Make a 2D feature vector
def img_to_feature(img):
  d   = min(img.size)
  img = img.crop((0,0,d,d)) # Make it square
  img = img.resize((get_sample_dim(),get_sample_dim())) # Unify the dimension
  img = img if img.mode=='HSV' else img.convert('HSV')

  # Normalise each channel
  h,s,v = img.split()
  h = np.multiply(h, 1.0/360)
  s = np.multiply(s, 1.0/255)
  v = np.multiply(v, 1.0/255)

  return np.array([h,s,v])

def inverse_trans(v):
  return inverse(v)

# Apply a sample image with a transformation filter
def apply_filter(v,inverse=False):
  def to(sample):
    return transform(v,inverse)(sample)
  return to

max_stars_repo_name: tao-pr/photo-auto-balance