text:
from .context import constraint, data
import copy
import numpy as np
from nose.tools import raises
from pyparsing import ParseBaseException
from math import erf, sqrt

class TestConstraint:
    def __init__(self):
        pass

    @classmethod
    def setup_class(cls):

        cls.line1 = 'A<B at 6 weight 1'
        cls.line2 = '42>=A At B=17 everytime before Weight 5 AltPenalty C<=42 Min 3.4e-7#Comment  '
        cls.line3 = 'A<16 between B=3.14, 702 weight 8 min 9'
        cls.line4 = 'A<16 always weight 6'
        cls.line5 = 'A<16 once weight 6'
        cls.line6 = 'A<B at 6 confidence 0.98'
        cls.line7 = 'A<16 between 5,B=5 confidence 0.95 tolerance 15'
        cls.line8 = 'A at B=6 <= A at B=5 weight 6'
        cls.line9 = 'A at 7 > C at B=4 before confidence 0.95 tolerance 1'
        cls.line10 = 'A<B at C=6 pmin 0.1 pmax 0.95 tolerance 1'
        cls.line11 = 'A<16 once between B=3.14, 702 weight 8 min 9'

        cls.err1 = 'A<16 always weight 2 confidence 0.95'
        cls.err2 = 'A<16 always confidence 0.98 tolerance 15 min 1'
        cls.err3 = 'A<16 always tolerance 5'

        cls.f1 = 'bngl_files/p1_5.prop'
        cls.f2 = 'bngl_files/con_test.prop'
        cls.f3 = 'bngl_files/con_test_likelihood.prop'
        cls.dat2 = 'bngl_files/con_test.gdat'

        cls.model = 'a.bngl'
        cls.suf = 'b'
        cls.cset = constraint.ConstraintSet(cls.model, cls.suf)

    @classmethod
    def teardown_class(cls):
        pass

    def test_grammar_at(self):
        p = self.cset.parse_constraint_line(self.line1)
        assert list(p.ineq) == ['A','<','B']
        assert p.enforce[0] == 'at'
        assert list(p.enforce[1]) == ['6']
        assert list(p.weight_expr) == ['weight', '1']
        assert float(p.weight_expr.weight) == 1.
        assert not bool(p.weight_expr.min)

        p = self.cset.parse_constraint_line(self.line2)
        assert list(p.ineq) == ['42', '>=', 'A']
        assert p.enforce[0] == 'at'
        assert list(p.enforce[1]) == ['B', '17']
        assert p.enforce[2] == 'everytime'
        assert p.enforce[3] == 'before'
        assert p.weight_expr.weight == '5'
        assert list(p.weight_expr.altpenalty) == ['C', '<=', '42']
        assert p.weight_expr.min == '3.4E-7'

    def test_grammar_between(self):
        p = self.cset.parse_constraint_line(self.line3)
        assert list(p.ineq) == ['A', '<', '16']
        assert p.enforce[1][0] == 'B'
        assert p.enforce[1][1] == '3.14'
        assert len(p.enforce[2]) == 1
        assert p.enforce[2][0] == '702'
        p = self.cset.parse_constraint_line(self.line11)
        assert p.enforce[0] == 'once between'
        assert p.enforce[1][0] == 'B'

    def test_grammar_other(self):
        p = self.cset.parse_constraint_line(self.line4)
        assert p.enforce[0] == 'always'
        assert p.weight_expr.weight == '6'

    def test_grammar_likelihood(self):
        p = self.cset.parse_constraint_line(self.line6)
        assert not p.weight_expr
        assert p.likelihood_expr.confidence == '0.98'
        assert not p.likelihood_expr.tolerance
        p = self.cset.parse_constraint_line(self.line7)
        assert p.likelihood_expr.confidence == '0.95'
        assert p.likelihood_expr.tolerance == '15'

    def test_grammar_splitat(self):
        p = self.cset.parse_constraint_line(self.line8)
        assert not p.ineq
        assert p.split.obs1 == 'A'
        assert list(p.split.at1[1]) == ['B', '6']
        assert p.split.sign == '<='
        assert p.weight_expr.weight == '6'
        p = self.cset.parse_constraint_line(self.line9)
        assert list(p.split.at1[1]) == ['7']
        assert p.split.obs2 == 'C'
        assert list(p.split.at2[1]) == ['B', '4']
        assert p.split.at2[2] == 'before'
        assert p.likelihood_expr.confidence == '0.95'

    def test_grammar_pmin(self):
        p = self.cset.parse_constraint_line(self.line10)
        assert not p.likelihood_expr.confidence
        assert p.likelihood_expr.pmin == '0.1'
        assert p.likelihood_expr.pmax == '0.95'

    @raises(ParseBaseException)
    def test_grammar_invalid1(self):
        p = self.cset.parse_constraint_line(self.err1)

    @raises(ParseBaseException)
    def test_grammar_invalid2(self):
        p = self.cset.parse_constraint_line(self.err2)

    @raises(ParseBaseException)
    def test_grammar_invalid3(self):
        p = self.cset.parse_constraint_line(self.err3)

    def test_load_file(self):
        cs = copy.deepcopy(self.cset)
        cs.load_constraint_file(self.f1)

        assert cs.constraints[0].quant1 == 'Ag_free'
        assert cs.constraints[0].or_equal is False
        assert cs.constraints[0].atvar is None
        assert cs.constraints[0].atval == 6.

        assert cs.constraints[1].quant2 == 42.
        assert cs.constraints[1].atvar == 'RP'
        assert cs.constraints[1].min_penalty == 3.4e-7
        assert cs.constraints[1].alt1 == 'R0'
        assert cs.constraints[1].alt2 == 42.

        assert cs.constraints[2].startvar == 'RP'
        assert cs.constraints[2].startval == 3.14
        assert cs.constraints[2].endvar is None
        assert cs.constraints[2].endval == 702.

        assert isinstance(cs.constraints[3], constraint.AlwaysConstraint)
        assert isinstance(cs.constraints[4], constraint.OnceConstraint)
        assert cs.constraints[4].weight == 6.

        assert isinstance(cs.constraints[5], constraint.SplitAtConstraint)
        assert cs.constraints[5].atval1 == 15
        assert cs.constraints[5].atvar1 == 'R0'
        assert cs.constraints[5].atvar2 is None
        assert cs.constraints[5].before1
        assert not cs.constraints[5].before2

    def test_penalties(self):
        d = data.Data()
        d.load_data('bngl_files/con_test.gdat')
        d_dict = {self.model: {self.suf: d}}

        cs = copy.deepcopy(self.cset)
        cs.load_constraint_file(self.f2)

        assert cs.constraints[0].penalty(d_dict) == 0
        assert cs.constraints[1].penalty(d_dict) == 4
        np.testing.assert_almost_equal(cs.constraints[2].penalty(d_dict), 0.4)
        np.testing.assert_almost_equal(cs.constraints[3].penalty(d_dict), 0.4)
        assert cs.constraints[4].penalty(d_dict) == 0
        assert cs.constraints[5].penalty(d_dict) == 10
        assert cs.constraints[6].penalty(d_dict) == 0
        assert cs.constraints[7].penalty(d_dict) == 25
        assert cs.constraints[8].penalty(d_dict) == 20
        assert cs.constraints[9].penalty(d_dict) == 20
        np.testing.assert_almost_equal(cs.constraints[10].penalty(d_dict), 1.8)
        assert cs.constraints[11].penalty(d_dict) == 1
        assert cs.constraints[12].penalty(d_dict) == 1
        assert cs.constraints[13].penalty(d_dict) == 0
        np.testing.assert_almost_equal(cs.constraints[14].penalty(d_dict), 1.6)
        np.testing.assert_almost_equal(cs.constraints[15].penalty(d_dict), 0.1)
        assert cs.constraints[16].penalty(d_dict) == 10
        assert cs.constraints[17].penalty(d_dict) == 3

        np.testing.assert_almost_equal(cs.total_penalty(d_dict), 98.3)

    def test_penalty_scale(self):
        d = data.Data()
        d.load_data('bngl_files/con_test.gdat')
        d_dict = {self.model: {self.suf: d}}

        cs = copy.deepcopy(self.cset)
        cs.load_constraint_file(self.f2, scale=2.0)

        assert cs.constraints[0].penalty(d_dict) == 0
        assert cs.constraints[1].penalty(d_dict) == 8
        np.testing.assert_almost_equal(cs.constraints[2].penalty(d_dict), 0.8)
        np.testing.assert_almost_equal(cs.constraints[3].penalty(d_dict), 0.8)

        np.testing.assert_almost_equal(cs.total_penalty(d_dict), 196.6)

    def test_likelihood_penalties(self):
        d = data.Data()
        d.load_data('bngl_files/con_test.gdat')
        d_dict = {self.model: {self.suf: d}}

        cs = copy.deepcopy(self.cset)
        cs.load_constraint_file(self.f3)

        def cdf(x, sigma):
            return (1. + erf(x / sigma / sqrt(2.) )) / 2.
        np.testing.assert_almost_equal(cdf(1,1)-cdf(-1,1), 0.682689492137)  # Check this gaussian CDF is working

        assert cs.constraints[0].penalty(d_dict) == 0
        np.testing.assert_almost_equal(cs.constraints[1].penalty(d_dict), -np.log(0.95))
        np.testing.assert_almost_equal(cs.constraints[2].penalty(d_dict), -np.log(0.05))
        np.testing.assert_almost_equal(cs.constraints[3].penalty(d_dict), -np.log(0.5))
        np.testing.assert_almost_equal(cs.constraints[4].penalty(d_dict), -np.log(0.9 * cdf(-1.,1.) + 0.05))
        np.testing.assert_almost_equal(cs.constraints[5].penalty(d_dict), -np.log(0.9 * cdf(1., 2.) + 0.05))
        np.testing.assert_almost_equal(cs.constraints[6].penalty(d_dict), -np.log(cdf(-3., 3.)))
        np.testing.assert_almost_equal(cs.constraints[7].penalty(d_dict), -np.log(0.75 * cdf(1., 2.) + 0.1))
        np.testing.assert_almost_equal(cs.constraints[8].penalty(d_dict), -np.log(cdf(-1., 1.)))

max_stars_repo_name: arakkkkk/PyBNF

max_issues_repo_name: arakkkkk/PyBNF

max_forks_repo_name: arakkkkk/PyBNF


text:
"""
Utility functions to generate plotly figures from dataframe. Called in app.py
"""

import plotly.graph_objects as go
import plotly.express as px
import numpy as np
import pandas as pd
import plotly.io as pio
from plotly.validators.scatter.marker import SymbolValidator

from data_input import normalize_by_population

pio.templates.default = "plotly_white"

FIRST_LINE_HEIGHT = 600

LABEL_FONT_SIZE = 18

def make_map(df, df_fatalities, df_recovered):
    """
    Build figure with map of total number of cases

    Parameters
    ----------
    df: pandas DataFrame
        Our cases to plot
    pop: pandas DataFrame
        The population, used to normalize
    """
    normalized_values = normalize_by_population(df)
    # Plot per Million individual
    normalized_values *= 1e6
    hovertemplate = ('Country:%{customdata[0]}
' +
                     'Active cases per million: %{customdata[1]:.1f}
' +
                     'Active cases: %{customdata[2]}
' +
                     'Fatalities: %{customdata[3]}
' +
                     'Recovered: %{customdata[4]}'
                     )
    fig = px.choropleth(df, locations='iso',
                    color=np.log10(normalized_values),
                    custom_data=[df['country_region'], normalized_values,
                                 df['value'], df_fatalities['value'],
                                 df_recovered['value']],
                    color_continuous_scale='Plasma_r',
                    labels={'color': 'Active
cases
per
Million'})
    fig.update_layout(title='Click on map to add/remove a country',
            coloraxis_colorbar_tickprefix='1.e',
            coloraxis_colorbar_len=0.6,
            coloraxis_colorbar_title_font_size=LABEL_FONT_SIZE,
            margin=dict(l=0.03, r=0, b=0),
            height=FIRST_LINE_HEIGHT,
            geo_projection_scale=1.26)
    fig.update_traces(
            hovertemplate=hovertemplate,
        )
    return fig

def make_timeplot(df_measure, df_prediction):
    """
    Build figure showing evolution of number of cases vs. time for all countries.
    The visibility of traces is set to 0 so that the interactive app will
    toggle the visibility.

    Parameters
    ----------
    df_measure: pandas DataFrame
        DataFrame of measured cases, created by :func:`data_input.get_data`, of wide format.

    df_prediction: pandas DataFrame
        DataFrame of predictions, with similar structure as df_measure
    """
    # mode = 'confirmed'
    mode = 'active'
    df_measure_confirmed = df_measure[mode]
    colors = px.colors.qualitative.Dark24
    n_colors = len(colors)
    fig = go.Figure()
    hovertemplate_measure = '%{meta}
%{x}
%{y:.0f}<extra></extra>'
    hovertemplate_prediction = '%{meta}
prediction
%{x}
%{y:.0f}<extra></extra>'
    for i, country in enumerate(df_measure_confirmed.columns):
        fig.add_trace(go.Scatter(x=df_measure_confirmed.index,
                                 y=df_measure_confirmed[country],
                                 name=country[1], mode='markers+lines',
                                 marker_symbol = SymbolValidator().values[i],
                                 marker_color=colors[i%n_colors],
                                 line_color=colors[i%n_colors],
                                 meta=country[1],
                                 hovertemplate=hovertemplate_measure,
                                 visible=False))
    prediction = df_prediction['prediction']
    upper_bound = df_prediction['upper_bound']
    lower_bound = df_prediction['lower_bound']
    for i, country in enumerate(prediction.columns):
        # Do not plot predictions for a country with less than 50 cases
        if df_measure_confirmed[country][-1] < 50:
            continue
        fig.add_trace(go.Scatter(x=prediction.index,
                                 y=prediction[country],
                                 name='+' + country[1], mode='lines',
                                 line_dash='dash',
                                 line_color=colors[i%n_colors],
                                 showlegend=False,
                                 meta=country[1],
                                 hovertemplate=hovertemplate_prediction,
                                 visible=False))
        fig.add_trace(go.Scatter(x=upper_bound.index,
                                 y=upper_bound[country],
                                 name='+' + country[1], mode='lines',
                                 line_dash='dot',
                                 line_color=colors[i%n_colors],
                                 showlegend=False,
                                 visible=False,
                                 hoverinfo='skip',
                                 line_width=.8))
        fig.add_trace(go.Scatter(x=lower_bound.index,
                                 y=lower_bound[country],
                                 name='+' + country[1], mode='lines',
                                 line_dash='dot',
                                 line_color=colors[i%n_colors],
                                 showlegend=False,
                                 visible=False,
                                 hoverinfo='skip',
                                 line_width=.8))

    last_day = df_measure_confirmed.index.max()
    day = pd.DateOffset(days=1)
    fig.update_layout(title='',
            xaxis=dict(rangeslider_visible=True,
                range=(last_day - 10 * day,
                       last_day + 4 * day)))
    fig.update_layout(
        showlegend=True,
        updatemenus=[
        dict(
            type = "buttons",
            direction = "left",
            buttons=list([
                dict(
                    args=[{"visible": [False,]*len(df_measure_confirmed.columns)}],
                    label="Reset",
                    method="update",
                ),
                dict(
                    args=[{'yaxis': {'type':'log'},
                           "legend": {'x':0.65, 'y':0.1,
                                      "font":{"size":18},
                                      }}],
                    label="log",
                    method="relayout",
                ),
                dict(
                    args=[{'yaxis': {'type':'linear'},
                           "legend": {'x':0.05, 'y':0.8,
                                      "font":{"size":18},
                                      }}],
                    label="lin",
                    method="relayout",
                ),

            ]),
            pad={"r": 10, "t": 0, "b": 0},
            showactive=True,
            x=0.05,
            xanchor="left",
            y=1.05,
            yanchor="top",
            font_color='black',
        ),
        ],
        xaxis_tickfont_size=LABEL_FONT_SIZE - 4,
        yaxis_tickfont_size=LABEL_FONT_SIZE - 4,
        height=FIRST_LINE_HEIGHT,
        margin=dict(t=0, b=0.02),
        # The legend position + font size
        # See https://plot.ly/python/legend/#style-legend
        legend=dict(x=.05, y=.8, font_size=LABEL_FONT_SIZE,
                    )
    )
    fig.add_annotation(
            x=0.1,
            y=0.95,
            xref='paper',
            yref='paper',
            showarrow=False,
            font_size=LABEL_FONT_SIZE,
            text="Active cases")
    fig.add_annotation(
            x=0.25,
            y=-0.13,
            xref='paper',
            yref='paper',
            showarrow=False,
            font_size=LABEL_FONT_SIZE - 6,
            font_color="DarkSlateGray",
            text="Drag handles below to change time window")

    return fig

if __name__ == '__main__':
    from data_input import get_all_data, tidy_most_recent

    df, df_prediction = get_all_data()
    # most recent date, tidy format (one column for countries)
    df_tidy = tidy_most_recent(df)
    df_tidy_fatalities = tidy_most_recent(df, 'death')
    df_tidy_recovered = tidy_most_recent(df, 'recovered')

    fig1 = make_map(df_tidy, df_tidy_fatalities, df_tidy_recovered)
    fig2 = make_timeplot(df, df_prediction)

ext: py

max_stars_repo_name: ashwinpn/covid-dashboard

max_issues_repo_name: ashwinpn/covid-dashboard

max_forks_repo_name: ashwinpn/covid-dashboard

reason: import numpy


text:
import functools
import json
import logging
import os
import random
from argparse import Namespace
import pickle
import numpy as np

import click
import torch
import torch.nn.functional as F
from torch.utils.data import DataLoader
from tqdm import tqdm
from transformers import WEIGHTS_NAME
from wikipedia2vec.dump_db import DumpDB

from luke.utils.entity_vocab import MASK_TOKEN, PAD_TOKEN, UNK_TOKEN

from ..utils.trainer import Trainer, trainer_args
from .model import LukeForEntityDisambiguation

# import added
from examples.utils.mention_db import MentionDB, BertLowercaseNormalizer
from transformers.tokenization_bert import BasicTokenizer
from .utils import EntityLinkingDataset, convert_documents_to_features

logger = logging.getLogger(__name__)

@click.group(name='entity-linking')
def cli():
    pass

@cli.command()
@click.option('--cached-dataset', type=click.Path(exists=True), default='/mnt/usbdisk1/entity_linking/data-3/generated/test_train_data/cached_datasets.pkl')
@click.option('--cached-titles', type=click.Path(exists=True), default='/mnt/usbdisk1/entity_linking/data-3/generated/test_train_data/cached_titles.pkl')
@click.option('--data-dir', type=click.Path(exists=True), default='/mnt/usbdisk1/entity_linking/data-3/generated/test_train_data')
@click.option('--mentiondb-file', type=click.Path(exists=True), default='/mnt/usbdisk1/entity_linking/data-3/mention_db_from_p_e_m_v2')
@click.option('--titles-file', type=click.Path(exists=True), default='/mnt/usbdisk1/entity_linking/data-3/enwiki_20181220_titles.txt')
@click.option('--redirects-file', type=click.Path(exists=True), default='/mnt/usbdisk1/entity_linking/data-3/enwiki_20181220_redirects.tsv')
@click.option('-t', '--test-set', default=['test_a', 'test_b', 'ace2004', 'aquaint', 'msnbc', 'wikipedia'],
              multiple=True)
@click.option('--do-train/--no-train', default=True)
@click.option('--do-eval/--no-eval', default=True)
@click.option('--num-train-epochs', default=10)
@click.option('--train-batch-size', default=1) # * acm_step =  4 8 16 32 64
@click.option('--max-seq-length', default=128) # 512
@click.option('--max-candidate-length', default=20)
@click.option('--max-entity-length', default=32)
@click.option('--masked-entity-prob', default=0.15) # default 0.9
@click.option('--candidate-generation/--no-candidate-generation', default=True)
@click.option('--use-context-entities/--no-context-entities', default=False)
@click.option('--context-entity-selection-order', default='highest_prob',
              type=click.Choice(['natural', 'random', 'highest_prob']))
@click.option('--document-split-mode', default='simple', type=click.Choice(['simple', 'per_mention'])) # 使わん
@click.option('--fix-entity-emb/--update-entity-emb', default=True)
@click.option('--fix-entity-bias/--update-entity-bias', default=True)
@click.option('--seed', default=1) # set_seed
@trainer_args
@click.pass_obj
def run(common_args, **task_args):
    task_args.update(common_args)
    args = Namespace(**task_args)

    logger.info('Set Seed')
    random.seed(args.seed)
    np.random.seed(args.seed)
    torch.manual_seed(args.seed)
    torch.cuda.manual_seed_all(args.seed)

    logger.info('Loading Cached Datasets')
    with open(args.cached_dataset, 'rb') as f:
        dataset = pickle.load(f)
    print(args.tokenizer)

    logger.info('Loading Cached Titles')
    with open(args.cached_titles, 'rb') as f:
        entity_titles = pickle.load(f)

    logger.info('Building Entity Vocab')
    entity_vocab = {PAD_TOKEN: 0, MASK_TOKEN: 2, UNK_TOKEN: 1}
    for n, title in enumerate(sorted(entity_titles), 2): # [NO_E]も入る
        entity_vocab[title] = n

    assert '[NO_E]' in entity_titles

    model_config = args.model_config
    model_config.entity_vocab_size = len(entity_vocab) # これを orig_emb[title] or orig_emb[UNK] or new_ones
    logger.info('Model configuration: %s', model_config)

    model_weights = args.model_weights
    orig_entity_vocab = args.entity_vocab
    orig_entity_emb = model_weights['entity_embeddings.entity_embeddings.weight'] # 事前学習済みのエンティティ埋め込み (Ve ~= 1M)
    vocab_size = orig_entity_emb.shape[0]

    if orig_entity_emb.size(0) != len(entity_vocab):
        orig_entity_bias = model_weights['entity_predictions.bias']

        assert '[UNK]' in orig_entity_vocab

        entity_emb = orig_entity_emb.new_zeros((len(entity_titles) + 3, model_config.entity_emb_size)) # ここにslot filling していく
        entity_bias = orig_entity_bias.new_zeros(len(entity_titles) + 3)

        num_unk_entity = 0
        num_valid_entity = 0
        for title, index in entity_vocab.items(): # entity_titles が valid title or UNK or NO_E に場合分けして slot filling
            if title in orig_entity_vocab:
                num_valid_entity += 1
                entity_emb[index] = orig_entity_emb[orig_entity_vocab[title]]
                entity_bias[index] = orig_entity_bias[orig_entity_vocab[title]]
            elif title == '[NO_E]': # 1 で初期化
                entity_emb[index] = orig_entity_emb.new_ones(model_config.entity_emb_size)
                entity_bias[index] = orig_entity_bias.new_ones(1)
                logger.info('Successfully Initialized for [NO_E]')
            else:
                num_unk_entity += 1
                entity_emb[index] = orig_entity_emb[orig_entity_vocab['[UNK]']]
                entity_bias[index] = orig_entity_bias[orig_entity_vocab['[UNK]']]

        logger.info('# of valid entities: %d', num_valid_entity)
        logger.info('# of unknown entities: %d', num_unk_entity)

        model_weights['entity_embeddings.entity_embeddings.weight'] = entity_emb
        model_weights['entity_embeddings.mask_embedding'] = entity_emb[1].view(1, -1)
        model_weights['entity_predictions.decoder.weight'] = entity_emb
        model_weights['entity_predictions.bias'] = entity_bias
        del orig_entity_bias, entity_emb, entity_bias
    del orig_entity_emb

    logger.info('Building Model')
    model = LukeForEntityDisambiguation(model_config)
    model.load_state_dict(model_weights, strict=False)
    logger.info('Being on device')
    model.to(args.device)

    def collate_fn(batch, is_eval=False):
        def create_padded_sequence(attr_name, padding_value):
            tensors = [torch.tensor(getattr(o, attr_name), dtype=torch.long) for o in batch]
            return torch.nn.utils.rnn.pad_sequence(tensors, batch_first=True, padding_value=padding_value)

        ret = dict(
            word_ids=create_padded_sequence('word_ids', args.tokenizer.pad_token_id),
            word_segment_ids=create_padded_sequence('word_segment_ids', 0),
            word_attention_mask=create_padded_sequence('word_attention_mask', 0),
            entity_ids=create_padded_sequence('entity_ids', 0),
            entity_position_ids=create_padded_sequence('entity_position_ids', -1),
            entity_segment_ids=create_padded_sequence('entity_segment_ids', 0),
            entity_attention_mask=create_padded_sequence('entity_attention_mask', 0),
        )
        ret['entity_candidate_ids'] = create_padded_sequence('entity_candidate_ids', 0)

        if is_eval:
            ret['document'] = [o.document for o in batch]
            ret['mentions'] = [o.mentions for o in batch]
            ret['target_mention_indices'] = [o.target_mention_indices for o in batch] # instanceにK個mentionがあれば，[range(K)]となる

        return ret

    # train -> test_b
    if args.do_train:
        logger.info('*****Training*****')
        logger.info('Converting Documents to Features')
        train_data = convert_documents_to_features(
            dataset.test_b, args.tokenizer, entity_vocab, 'train', args.max_seq_length,
            args.max_candidate_length, args.max_mention_length, args.max_entity_length)
        train_dataloader = DataLoader(train_data, batch_size=args.train_batch_size, collate_fn=collate_fn, shuffle=True)
        logger.info('Fix entity embeddings during training: %s', args.fix_entity_emb)
        if args.fix_entity_emb:
            model.entity_embeddings.entity_embeddings.weight.requires_grad = False
        logger.info('Fix entity bias during training: %s', args.fix_entity_bias)
        if args.fix_entity_bias:
            model.entity_predictions.bias.requires_grad = False
        num_train_steps = len(train_dataloader) // args.gradient_accumulation_steps * args.num_train_epochs
        trainer = EntityLinkingTrainer(args, model, train_dataloader, num_train_steps)
        trainer.train()

    results = {}
    torch.cuda.empty_cache()

    if args.do_eval:
        model.eval()

        for dataset_name in args.test_set:
            logger.info('***** Evaluating: %s *****', dataset_name)
            eval_documents = getattr(dataset, dataset_name)
            eval_data = convert_documents_to_features(
                eval_documents, args.tokenizer, entity_vocab, 'eval', args.max_seq_length,
                args.max_candidate_length, args.max_mention_length, args.max_entity_length)
            eval_dataloader = DataLoader(eval_data, batch_size=1,
                                         collate_fn=functools.partial(collate_fn, is_eval=True))
            predictions_file = None
            if args.output_dir:
                predictions_file = os.path.join(args.output_dir, 'eval_predictions_%s.jsonl' % dataset_name)
            results[dataset_name] = evaluate(args, eval_dataloader, model, entity_vocab, predictions_file)

        if args.output_dir:
            output_eval_file = os.path.join(args.output_dir, 'eval_results.txt')
            with open(output_eval_file, 'w') as f:
                json.dump(results, f, indent=2, sort_keys=True)

    return results

@cli.command()
@click.option('--data-dir', type=click.Path(exists=True), default='/mnt/usbdisk1/entity_linking/data-3/generated/test_train_data')
@click.option('--mentiondb-file', type=click.Path(exists=True), default='/mnt/usbdisk1/entity_linking/data-3/mention_db_from_p_e_m_v2')
@click.option('--titles-file', type=click.Path(exists=True), default='/mnt/usbdisk1/entity_linking/data-3/enwiki_20181220_titles.txt')
@click.option('--redirects-file', type=click.Path(exists=True), default='/mnt/usbdisk1/entity_linking/data-3/enwiki_20181220_redirects.tsv')
def cache_datasets_and_titles(data_dir, mentiondb_file, titles_file, redirects_file):
    logger.info('Building Datasets')
    dataset = EntityLinkingDataset(data_dir, mentiondb_file, titles_file, redirects_file)
    logger.info('Pickling Datasets')
    cache_file = os.path.join(
        data_dir,
        "cached_datasets.pkl",
    )
    with open(cache_file, 'wb') as f:
        pickle.dump(dataset, f)
    logger.info('--Pickled')
    logger.info(cache_file)
    logger.info('Building Entity Titles')
    entity_titles = []
    for data in dataset.get_all_datasets():
        for document in data:
            for mention in document.mentions:
                entity_titles.append(mention.title)
                for candidate in mention.candidates:
                    entity_titles.append(candidate.title)
    logger.info('Frozing Titles (Time Consuming...)')
    entity_titles = frozenset(entity_titles)
    logger.info('Pickling Entity Titles')
    cache_file = os.path.join(
        data_dir,
        "cached_titles.pkl",
    )
    with open(cache_file, 'wb') as f:
        pickle.dump(entity_titles, f)
    logger.info('--Pickled')
    logger.info(cache_file)
    logger.info('DONE')

@cli.command()
@click.argument('dump_db_file', type=click.Path(exists=True))
@click.argument('mention_db_file', type=click.Path(exists=True))
@click.argument('out_file', type=click.File('w'))
@click.option('--data-dir', type=click.Path(exists=True), default='data/entity-disambiguation')
def create_candidate_list(dump_db_file, mention_db_file, out_file, data_dir):
    dump_db = DumpDB(dump_db_file)
    titles = set()
    valid_titles = frozenset(dump_db.titles())

    reader = EntityLinkingDataset(data_dir, mention_db_file)
    for documents in tqdm(reader.get_all_datasets()):
        for document in tqdm(documents):
            for mention in document.mentions:
                candidates = mention.candidates
                for candidate in candidates:
                    title = dump_db.resolve_redirect(candidate.title)
                    if title in valid_titles:
                        titles.add(title)

    for title in titles:
        out_file.write(title + '\n')

@cli.command()
@click.argument('dump_db_file', type=click.Path(exists=True))
@click.argument('out_file', type=click.File(mode='w'))
def create_title_list(dump_db_file, out_file):
    dump_db = DumpDB(dump_db_file)

    for title in dump_db.titles():
        out_file.write(f'{title}\n')

@cli.command()
@click.argument('dump_db_file', type=click.Path(exists=True))
@click.argument('out_file', type=click.File(mode='w'))
def create_redirect_tsv(dump_db_file, out_file):
    dump_db = DumpDB(dump_db_file)

    for src, dest in dump_db.redirects():
        out_file.write(f'{src}\t{dest}\n')

class EntityLinkingTrainer(Trainer):
    def _create_model_arguments(self, batch):
        batch['entity_labels'] = batch['entity_ids'].clone()
        for index, entity_length in enumerate(batch['entity_attention_mask'].sum(1).tolist()):
            masked_entity_length = max(1, round(entity_length * self.args.masked_entity_prob))
            permutated_indices = torch.randperm(entity_length)[:masked_entity_length]
            batch['entity_ids'][index, permutated_indices[:masked_entity_length]] = 1  # [MASK]
            batch['entity_labels'][index, permutated_indices[masked_entity_length:]] = -1

        return batch

def evaluate(args, eval_dataloader, model, entity_vocab, output_file=None):
    predictions = []
    context_entities = [] #
    labels = []
    documents = []
    mentions = []
    reverse_entity_vocab = {v: k for k, v in entity_vocab.items()}
    for item in tqdm(eval_dataloader, leave=False):  # the batch size must be 1
        inputs = {k: v.to(args.device)
                  for k, v in item.items() if k not in ('document', 'mentions', 'target_mention_indices')}
        entity_ids = inputs.pop('entity_ids')
        entity_attention_mask = inputs.pop('entity_attention_mask')
        input_entity_ids = entity_ids.new_full(entity_ids.size(), 1)  # [MASK]
        entity_length = entity_ids.size(1)
        with torch.no_grad():
            logits = model(entity_ids=input_entity_ids, entity_attention_mask=entity_attention_mask, **inputs)[0]
            result = torch.argmax(logits, dim=2).squeeze(0)

        for index in item['target_mention_indices'][0]:
            predictions.append(result[index].item())
            labels.append(entity_ids[0, index].item())
            documents.append(item['document'][0])
            mentions.append(item['mentions'][0][index])
            context_entities.append([])

    num_correct = 0
    num_mentions = 0
    num_mentions_with_candidates = 0

    num_gold = 0
    num_pred = 0

    eval_predictions = []
    for prediction, label, document, mention, cxt in zip(predictions, labels, documents, mentions, context_entities):

        assert not (mention.candidates and prediction == 0)
        assert label != 0

        if reverse_entity_vocab[label] != '[NO_E]':
            num_gold += 1

        if reverse_entity_vocab[label] != '[NO_E]' and prediction == label:
            num_correct += 1

        if reverse_entity_vocab[prediction] != '[NO_E]':
            num_pred += 1

        num_mentions += 1

        #if mention.candidates:
            # num_mentions_with_candidates += 1

        eval_predictions.append(dict(
            document_id=document.id,
            document_words=document.words,
            document_length=len(document.words),
            mention_length=len(document.mentions),
            mention=dict(label=mention.title,
                            text=mention.text,
                            span=(mention.start, mention.end),
                            candidate_length=len(mention.candidates),
                            candidates=[dict(prior_prob=c.prior_prob, title=c.title) for c in mention.candidates]),
            prediction=reverse_entity_vocab[prediction],
            context_entities=cxt
        ))

    if output_file:
        with open(output_file, 'w') as f:
            for obj in eval_predictions:
                f.write(json.dumps(obj) + '\n')

    precision = num_correct / num_pred
    recall = num_correct / num_gold
    f1 = 2.0 * precision * recall / (precision + recall)

    logger.info('f1: %.5f', f1)
    logger.info('precision: %.5f', precision)
    logger.info('recall: %.5f', recall)
    logger.info('#mentions: %d', num_mentions)
    logger.info('#gold mentions: %d', num_gold)
    logger.info('#prediction as entity: %d', num_pred)
    logger.info('#correct: %d', num_correct)

    return dict(precision=precision, recall=recall, f1=f1)


text:
# Author:       Brian Murphy
# Date started: 16/03/2020
# Last updated: <08/05/2020 09:52:57 (BrianM)>

import numpy as np
import time

def default_PSO_params(params):
    """
    This is to set the default PSO parameters. Note by default constriction will be applied over inertia weighting.
    To prevent this, specify the wdamp parameter that isn't 1.

    Will display info by default

    Early stopping is disabled by default
    """
    if 'maxit' not in params:
        params['maxit'] = 30
    if 'n_part' not in params:
        params['n_part'] = 15
    if 'w' not in params:
        params['w'] = 1
    if 'wdamp' not in params:
        params['wdamp'] = 1
    if 'c1' not in params:
        params['c1'] = 2.05
    if 'c2' not in params:
        params['c2'] = 2.05
    if 'con' not in params and params['wdamp'] != 1 and params['w'] != 1:
        params['con'] = 0.7298437881283576
    if 'display_info' not in params:
        params['display_info'] = 1
    if 'early_stopping' not in params:
        params['early_stopping'] = 0
    if 'early_stopping_rounds' not in params:
        params['early_stopping_rounds'] = 10
    if 'BPSO' not in params:
        params['BPSO'] = 0
    if 'velocity_limit_scale' not in params and params['BPSO'] == 0:
        params['velocity_limit_scale'] = 0.2
    elif params['BPSO'] == 1:
        params['velocity_limit_scale'] = 1

    return params

class Particle:
    def __init__(self):
        self.position = 0
        self.velocity = 0
        self.pbest_position = self.position
        self.pbest_value = float('inf')

    def initial_position(self, varmin, varmax, BPSO):
        if not BPSO:
            self.position = np.random.uniform(low=varmin, high=varmax, size=[1, len(varmin)])[0]
        else:
            self.position = np.random.uniform(low=0, high=1, size=[1, len(varmin)])[0]
        self.pbest_position = self.position
        self.velocity = np.random.uniform(low=varmin, high=varmax, size=[1, len(varmin)])[0]

    def update_velocity(self, con, w, c1, c2, gbest_position, min_velocity, max_velocity):
        self.velocity = con * ((w * self.velocity)
                               + (c1 * np.random.rand() * (self.pbest_position - self.position))
                               + (c2 * np.random.rand() * (gbest_position - self.position)))
        # Apply velocity limits
        self.velocity = np.max([self.velocity, min_velocity], axis=0)
        self.velocity = np.min([self.velocity, max_velocity], axis=0)

    def update_position(self, varmin, varmax, BPSO):
        if not BPSO:
            self.position = self.position + self.velocity
            # Apply position limits
            self.position = np.max([self.position, varmin], axis=0)
            self.position = np.min([self.position, varmax], axis=0)
        else:
            sig_velocity = self.sigmoid(self.velocity)
            self.position = np.array([1 if np.random.rand(1) < p else 0 for p in sig_velocity])

    def sigmoid(self, x):
        return 1.0/(1.0 + np.exp(-x))

def examine_particles(particles_vector, gbest_position):
    converged = 0
    ham_dist = []
    for particle in particles_vector:
        if all(particle.position == gbest_position):
            converged += 1
        ham_dist.append(sum(particle.position != gbest_position))
    return {'number_converged': converged, 'particle_hamming_distance': ham_dist}

def PSO(problem, params, *args):
    """
    Syntax: gbest_value, gbest_position, best_costs = PSO.PSO(problem, params)

    Inputs:
        problem       - This is a mandatory dictionary that contains the cost function, parameter minimum vales
                        and parameter maximum values
        params        - These are the parameters needed to specify PSO parameters. Ideally specified by the user but
                        default values are available

    Outputs:
        gbest_value     - This is the best performance achieved by PSO
        gbest_position  - This is the set of parameters that gave the best performance
        best_costs      - This is a numpy array containing the costs achieved for each iteration

    Example:
        import numpy as np
        import PSO

        print('Find the centre of a circle of the form (x-a)^2 + (y-b)^2 = r^2 where x, y and r equal 3, 4 and 5.')

        # Define the cost function
        def circle_cost(positions):
            return np.abs(25 - ((3 - positions[0]) ** 2 + (4 - positions[1]) ** 2))

        cost_function = circle_cost

        # Set maximum values of a and b
        varmin = [-1000, -1000]  # a min, b min
        varmax = [1000, 1000]  # a max, b max

        # Create problem dictionary
        problem = {'cost_function': cost_function, 'varmin': varmin, 'varmax': varmax}

        # Create PSO parameters dictionary
        params = {'maxit': 15, 'n_part': 30, 'c1': 2, 'c2': 2, 'w': 0.9, 'wdamp': 0.2, 'con': 1, 'display_info': 1,
                  'early_stopping': 1, 'early_stopping_rounds': 7}

        # Run PSO and return lowest cost, a,b values and the costs per iteration
        gbest_value, gbest_position, best_costs = PSO.PSO(problem, params)

        print('Found the centre of the circle as a = %f and b = %f' % (gbest_position[0], gbest_position[1]))
        print('The error was %f' % gbest_value)

    """
    if not args:
        args = []

    # Setting the random seed based off the current time
    np.random.seed(int(time.time()))

    cost_function = problem['cost_function']  # This tells PSO what the cost function is
    varmin = np.array(problem['varmin'])  # list of minimum lower bound values
    varmax = np.array(problem['varmax'])  # list of maximum upper bound values

    params = default_PSO_params(params)

    maxit = params['maxit']  # maximum number of PSO iterations
    n_part = params['n_part']  # number of PSO particles

    w = params['w']  # inertia coefficient
    wdamp = params['wdamp']  # damping factor for inertia coefficient
    c1 = params['c1']  # personal acceleration coefficient
    c2 = params['c2']  # global acceleration coefficient
    con = params['con']  # constriction value

    display_info = params['display_info']  # flag to show iteration info
    max_velocity = params['velocity_limit_scale'] * (varmax - varmin)
    min_velocity = -max_velocity

    particles_vector = [Particle() for _ in range(n_part)]  # create the particles

    gbest_value = float('inf')
    gbest_position = []

    for particle in particles_vector:  # set particles initial position within confines
        particle.initial_position(varmin, varmax, params['BPSO'])
        particle.pbest_position = particle.position
        particle.pbest_value = cost_function(particle.position, args)
        if particle.pbest_value < gbest_value:
            gbest_value = particle.pbest_value
            gbest_position = particle.pbest_position

    best_costs = np.zeros(maxit+1)
    best_costs[0] = gbest_value  # track best costs
    particle_debug = []
    # Enter main PSO loop
    for it in range(maxit):
        particle_debug.append(examine_particles(particles_vector, gbest_position))
        for particle in particles_vector:
            # Update velocity
            particle.update_velocity(con, w, c1, c2, gbest_position, min_velocity, max_velocity)

            # Update position
            particle.update_position(varmin, varmax, params['BPSO'])

            # Evaluate the cost at the new function
            cost_at_new_position = cost_function(particle.position, args)
            if cost_at_new_position < particle.pbest_value:
                particle.pbest_value = cost_at_new_position
                particle.pbest_position = particle.position
                if cost_at_new_position < gbest_value:
                    gbest_value = cost_at_new_position
                    gbest_position = particle.pbest_position

        best_costs[it+1] = gbest_value

        if display_info:
            print('Iteration: ', str(it), '  Best cost: ', str(gbest_value), '  Best position: ', str(gbest_position))

        w = w * wdamp

        if params['early_stopping'] and it > params['early_stopping_rounds']:
            if np.abs(best_costs[it+1] - best_costs[it+1 - params['early_stopping_rounds']]) < 0.0001:

                if display_info:
                    print('Stopping at round: ', str(it), ' as there was no performnce increase in ',
                          str(params['early_stopping_rounds']), 'rounds')
                    print('Best cost: ', str(gbest_value), '  Best position: ', str(gbest_position))
                return gbest_value, gbest_position, best_costs

    if display_info:
        print('Finished the PSO process')
        print('Best cost: ', str(gbest_value), '  Best position: ', str(gbest_position))
    return gbest_value, gbest_position, best_costs


text:
"""
NCL_lcmask_1.py
===============
This script illustrates the following concepts:
   - Drawing filled contours over a Lambert Conformal map
   - Zooming in on a particular area on a Lambert Conformal map
   - Creating a custom plot boundary
   - Using a blue-white-red color map
   - Setting contour levels using a min/max contour level and a spacing

See following URLs to see the reproduced NCL plot & script:
    - Original NCL script: http://ncl.ucar.edu/Applications/Scripts/lcmask_1.ncl
    - Original NCL plot: http://ncl.ucar.edu/Applications/Images/lcmask_1_1_lg.png and http://ncl.ucar.edu/Applications/Images/lcmask_1_2_lg.png
"""

###############################################################################
# Import packages:

import cartopy.crs as ccrs
import matplotlib.pyplot as plt
import numpy as np
import xarray as xr

import geocat.datafiles as gdf
from geocat.viz import cmaps as gvcmaps
from geocat.viz import util as gvutil

###############################################################################
# Read in data:

# Open a netCDF data file using xarray default engine and load the data into
# xarrays and disable time decoding due to missing necessary metadata
ds = xr.open_dataset(gdf.get("netcdf_files/atmos.nc"), decode_times=False)
# Extract a slice of the data
ds = ds.isel(time=0).drop_vars(names=["time"])
ds = ds.isel(lev=0).drop_vars(names=["lev"])
V = ds.V
# Ensure longitudes range from 0 to 360 degrees
V = gvutil.xr_add_cyclic_longitudes(V, "lon")

###############################################################################
# Plot unmasked data:

# Generate figure and projection using Cartopy
plt.figure(figsize=(7, 10))
proj = ccrs.LambertConformal(central_longitude=0, standard_parallels=(45, 89))
# Set axis projection
ax = plt.axes(projection=proj, frameon=False)
# Set extent to include all longitudes and the northern hemisphere
ax.set_extent((0, 359, 0, 89), crs=ccrs.PlateCarree())
ax.coastlines(linewidth=0.5)

# Plot data and create colorbar
newcmp = gvcmaps.BlWhRe

wind = V.plot.contourf(ax=ax,
                       cmap=newcmp,
                       transform=ccrs.PlateCarree(),
                       add_colorbar=False,
                       levels=24)
cbar = plt.colorbar(wind,
                    ax=ax,
                    orientation='horizontal',
                    drawedges=True,
                    ticks=np.arange(-48, 48, 8),
                    pad=0.1,
                    aspect=12)
cbar.ax.tick_params(length=0)  # remove tick marks but leave in labels

# Use geocat.viz.util convenience function to add left and right titles
gvutil.set_titles_and_labels(ax,
                             lefttitle=V.long_name,
                             lefttitlefontsize=16,
                             righttitle=V.units,
                             righttitlefontsize=16)

plt.show()

###############################################################################
# Mask data
masked = V.where(V.lat > 20)
masked = masked.where(masked.lat < 80)
masked = masked.where(masked.lon > 90)
masked = masked.where(masked.lon < 220)

# Rotate data to match NCL example
masked['lon'] = masked['lon'] + 180

###############################################################################
# Plot masked data

# Generate figure and projection using Cartopy
plt.figure(figsize=(10, 7))
proj = ccrs.LambertConformal(central_longitude=-22.5,
                             standard_parallels=(45, 89))
# Set axis projection
ax = plt.axes(projection=proj)
ax.coastlines(linewidth=0.5)

# Make a custom boundary using convenience function
gvutil.set_map_boundary(ax, [-85, 40], [20, 80], south_pad=1)

# Plot data and create colorbar
wind = masked.plot.contourf(ax=ax,
                            cmap=newcmp,
                            transform=ccrs.PlateCarree(),
                            add_colorbar=False,
                            levels=24)
cbar = plt.colorbar(wind,
                    ax=ax,
                    orientation='horizontal',
                    drawedges=True,
                    ticks=np.arange(-40, 44, 4),
                    pad=0.1,
                    aspect=18)
cbar.ax.tick_params(length=0)  # remove tick marks but leave in labels

# Use geocat.viz.util convenience function to add left and right titles
gvutil.set_titles_and_labels(ax,
                             lefttitle=V.long_name,
                             lefttitlefontsize=16,
                             righttitle=V.units,
                             righttitlefontsize=16)
plt.show()

max_stars_repo_name: NCAR/GeoCAT-examples

max_issues_repo_name: netgodz/GeoCAT-examples

max_forks_repo_name: netgodz/GeoCAT-examples


text:
#!/usr/bin/env python

import os
import argparse
import jinja2
import numpy as np
import matplotlib
if __name__ == '__main__':
    matplotlib.use('Agg')
import matplotlib.pyplot as plt

from astropy.table import Table, vstack, Column
from astropy.io import ascii
from Ska.Matplotlib import plot_cxctime
from Chandra.Time import DateTime
from kadi import events
from Ska.engarchive import fetch

FILE_DIR = os.path.dirname(os.path.abspath(__file__))

def get_options():
    parser = argparse.ArgumentParser(
        description="Update attitude error plots")
    parser.add_argument("--outdir",
                        default=".")
    parser.add_argument("--datadir",
                        default=".")
    parser.add_argument("--recent-start",
                        help="Date to use as reference for highlighted/recent data," +
                        " default is to highlight last 60 days")

    opt = parser.parse_args()
    return opt

def get_obs_table(start, stop):
    """
    Make a data table of obsids with one shot magnitudes and att errors

    :param start: start time for dwell range to fetch
    :param stop: stop time for range
    :returns: astropy table of observation data
    """
    manvrs = events.manvrs.filter(start=start, stop=stop, n_dwell__gte=1)
    obs_data = []
    for m in manvrs:
        obs = {}
        try:
            obsid = m.get_obsid()
        except ValueError:
            obsid = 0
        if obsid is None:
            obsid = 0

        obs['obsid'] = obsid
        obs['date'] = m.start
        obs['time'] = DateTime(m.start).secs
        obs['manvr_angle'] = m.angle
        obs['one_shot'] = m.one_shot
        obs['one_shot_yaw'] = m.one_shot_yaw
        obs['one_shot_pitch'] = m.one_shot_pitch
        obs['dwell_duration'] = DateTime(m.next_nman_start).secs - DateTime(m.npnt_start).secs

        try:
            all_err = {}
            for err_name, err_msid in zip(['roll_err', 'pitch_err', 'yaw_err'],
                                          ['AOATTER1', 'AOATTER2', 'AOATTER3']):
                err = fetch.Msid(err_msid,
                                 DateTime(m.npnt_start).secs + 500,
                                 m.next_nman_start)
                if len(err.times):
                    events.dumps.interval_pad = (0, 300)
                    err.remove_intervals(events.dumps)
                    events.tsc_moves.interval_pad = (0, 300)
                    err.remove_intervals(events.tsc_moves)
                    err.remove_intervals(events.ltt_bads)
                all_err[err_name] = err

            # If there are no samples left, that is not an error condition, and the
            # attitude errors should just be counted as 0.
            if len(all_err['roll_err'].vals) == 0:
                obs['roll_err'] = 0
                obs['point_err'] = 0
            else:
                obs['roll_err'] = np.degrees(
                    np.percentile(np.abs(all_err['roll_err'].vals), 99)) * 3600
                point_err = np.sqrt((all_err['pitch_err'].vals ** 2) +
                                    (all_err['yaw_err'].vals ** 2))
                obs['point_err'] = np.degrees(np.percentile(point_err, 99)) * 3600

        # If there are issues indexing into the AOATTER data, that's an error condition
        # that should just be captured by a large/bogus value.
        except IndexError:
            obs['point_err'] = 999
            obs['roll_err'] = 999
        obs_data.append(obs)

    return Table(obs_data)

def one_shot_plot(ref_data, recent_data, outdir='.'):

    # Grab the date of the start and stop of the two data sets.
    d0_str = ref_data['date'][0][0:8]
    d1_str = ref_data['date'][-1][0:8]
    d2_str = recent_data['date'][0][0:8]
    d3_str = recent_data['date'][-1][0:8]

    plt.figure(figsize=(7, 4))
    plt.plot(ref_data['manvr_angle'], ref_data['one_shot'], 'b+',
             markersize=5, markeredgewidth=1.0, alpha=.25, label=f'{d0_str} to {d1_str}')
    plt.plot(recent_data['manvr_angle'], recent_data['one_shot'], 'rx',
             markersize=5, markeredgewidth=.8, label=f'{d2_str} to {d3_str}')
    plt.grid()
    plt.xlim(0, 185)
    plt.ylim(ymin=0)
    plt.ylabel('One Shot (arcsec)')
    plt.xlabel('Manvr Angle (deg)')
    plt.title('One Shot Magnitude', fontsize=12, y=1.05)
    plt.legend(loc='upper left', fontsize=8)
    plt.tight_layout()
    plt.savefig(os.path.join(outdir, 'one_shot_vs_angle.png'))

def att_err_time_plots(ref_data, recent_data, min_dwell_time=1000, outdir='.'):
    ref_data = ref_data[ref_data['dwell_duration'] >= min_dwell_time]
    recent_data = recent_data[recent_data['dwell_duration'] >= min_dwell_time]

    # Grab the date of the start and stop of the two data sets.
    d0_str = ref_data['date'][0][0:8]
    d1_str = ref_data['date'][-1][0:8]
    d2_str = recent_data['date'][0][0:8]
    d3_str = recent_data['date'][-1][0:8]

    for i, ax, msid in zip([1, 2], ['roll', 'point'], ['abs(aoatter1)', 'rss(aoatter2,aoatter3)']):
        default_ylims = [0, 1.4]
        if ax == 'roll':
            default_ylims = [0, 12]
        fig = plt.figure(figsize=(5, 3.5))
        plot_cxctime(ref_data['time'], ref_data[f'{ax}_err'], 'b+', markersize=5,
                     alpha=.5, label=f'{d0_str} to {d1_str}')
        plot_cxctime(recent_data['time'], recent_data[f'{ax}_err'], 'rx', markersize=5,
                     alpha=.5, label=f'{d2_str} to {d3_str}')
        plt.margins(x=.05, y=.05)
        plt.ylabel(f'{ax} err (arcsec)', fontsize=9)
        plt.title(f'per obs 99th percentile {ax} err\n ({msid})',
                  fontsize=12)
        plt.grid()
        ylims = plt.ylim()
        setlims = plt.ylim(np.min([ylims[0], default_ylims[0]]),
                           np.max([ylims[1], default_ylims[1]]))
        plt.ylim(setlims)
        plt.xticks(fontsize=7)
        plt.tight_layout()
        plt.savefig(os.path.join(outdir, f'{ax}_err_vs_time.png'))

def att_err_hist(ref_data, recent_data, label=None, min_dwell_time=1000, outdir='.'):
    ref_data = ref_data[ref_data['dwell_duration'] >= min_dwell_time]
    recent_data = recent_data[recent_data['dwell_duration'] >= min_dwell_time]

    # Grab the date of the start and stop of the two data sets.
    d0_str = ref_data['date'][0][0:8]
    d1_str = ref_data['date'][-1][0:8]
    d2_str = recent_data['date'][0][0:8]
    d3_str = recent_data['date'][-1][0:8]

    for i, ax, msid in zip([1, 2], ['roll', 'point'], ['abs(aoatter1)', 'rss(aoatter2, aoatter3)']):
        plt.figure(figsize=(5, 3.5))
        if ax == 'roll':
            bin_width = .25
            lim = 15
        else:
            bin_width = .05
            lim = 7.5
        bins = np.arange(0, lim + bin_width, bin_width)
        plt.hist(ref_data[f'{ax}_err'], bins=bins, log=True, density=True, color='b',
                 alpha=.4, label=f'{d0_str} to {d1_str}')
        plt.hist(recent_data[f'{ax}_err'], bins=bins, log=True, density=True, color='r',
                 alpha=.4, label=f'{d2_str} to {d3_str}')
        plt.xlabel(f'{ax} err (arcsec)')
        plt.legend(loc='upper right', fontsize=7)
        plt.title(f'per obs 99th percentile {ax} err\n({msid})',
                  fontsize=12)
        plt.grid()
        plt.tight_layout()
        plt.savefig(os.path.join(outdir, f'{ax}_err_hist.png'))

def update_file_data(data_file, start, stop):
    if os.path.exists(data_file):
        print(f"Reading previous data from {data_file}")
        last_data = Table.read(data_file, format='ascii')
        new_data = get_obs_table(last_data[-5]['date'], stop)
        if new_data['date'][0] > last_data['date'][-1]:
            data = vstack([last_data, new_data])
        else:
            idx_old_data = np.flatnonzero(last_data['date'] >= new_data['date'][0])[0]
            data = vstack([last_data[0:idx_old_data], new_data])
        data = data[data['date'] >= start.date]
    else:
        data = get_obs_table(start, stop)
    data.sort('date')
    data.write(data_file, format='ascii', overwrite=True)
    return data

def update(datadir, outdir, full_start, recent_start,
           point_lim=7.5, roll_lim=15):

    data_file = os.path.join(datadir, 'data.dat')
    dat = update_file_data(data_file, full_start, DateTime())
    recent_data = dat[dat['time'] >= recent_start.secs]
    ref_data = dat[dat['time'] < recent_start.secs]

    one_shot_plot(ref_data, recent_data, outdir=outdir)

    comments_file = os.path.join(outdir, 'comments.dat')
    if os.path.exists(comments_file):
        comments = ascii.read(comments_file, Reader=ascii.FixedWidthTwoLine)
        # convert to a dict
        comments = {comment['obsid']: comment['comment'] for comment in comments}
    else:
        comments = {}

    ok = (recent_data['point_err'] < point_lim) | (recent_data['roll_err'] < roll_lim)
    outliers = recent_data[~ok]
    comments_col = Column(data=[comments.get(row['obsid'], '') for row in outliers],
                          name='comment')
    outliers.add_column(comments_col)
    recent_data = recent_data[ok]

    ok = (ref_data['point_err'] < point_lim) | (ref_data['roll_err'] < roll_lim)
    ref_data = ref_data[ok]
    att_err_time_plots(ref_data, recent_data, outdir=outdir)
    att_err_hist(ref_data, recent_data, outdir=outdir)

    template_html = open(os.path.join(FILE_DIR, 'index_template.html')).read()
    template = jinja2.Template(template_html)
    out_html = template.render(outliers=outliers, one_shot_start=ref_data['date'][0])
    with open(os.path.join(opt.outdir, 'index.html'), 'w') as fh:
        fh.write(out_html)

if __name__ == '__main__':
    opt = get_options()
    if not os.path.exists(opt.outdir):
        os.makedirs(opt.outdir)

    # Set start of time ranges for data
    if opt.recent_start is None:
        recent_start = DateTime(-60)
    else:
        recent_start = DateTime(opt.recent_start)

    update(outdir=opt.outdir, datadir=opt.datadir,
           full_start=recent_start - 365, recent_start=recent_start)

reason: import numpy,from astropy


text:
##############################################
# Random Util Stuff
##############################################
import torch
import torch.nn as nn
import numpy as np
import math
from torch.autograd import Variable
import torch.nn.functional as F

def normal_sample(mu, logvar, use_cuda=False):
    """
    Reparmaterization trick for normal distribution
    """
    eps = Variable(torch.randn(mu.size()))
    if use_cuda:
        eps = eps.cuda()
    std = torch.exp(logvar / 2.0)
    return mu + eps * std

def normal_sample_deterministic(mu, logvar, eps, use_cuda=False):
    """
    Reparmaterization trick for normal distribution
    """
    if use_cuda:
        eps = eps.cuda()
    std = torch.exp(logvar / 2.0)
    return mu + eps * std

def gumbel_sample(shape, use_cuda=False, eps=1e-20):
    """
    Sample from gumbel distribution
    """
    U = torch.rand(shape).cuda() if use_cuda else torch.rand(shape)
    return -Variable(torch.log(-torch.log(U + eps) + eps))

def gumbel_softmax_sample(logits, temp, use_cuda=False):
    """
    Sample from a gumbel softmax distribution
    Args:
        logits (Tensor [batch, num classes])
    """
    y = logits + gumbel_sample(logits.size(), use_cuda=use_cuda)
    return F.softmax(y / temp, dim=1)

def top_k_logits(logits, k):
    vals,_ = torch.topk(logits,k)
    mins = vals[:,-1].unsqueeze(dim=1).expand_as(logits)
    return torch.where(logits < mins, torch.ones_like(logits)*-1e10,logits)

def get_context_vector(encoded_sents, target, future=False, use_cuda=False):
    """
    Get create the context vector for the sentence given at index target for
    state classification. Do this by max pooling the sentences before the target sentence
    Args:
        encoded_sents (Tensor, [num_sents, batch, encoder dim])
        target (int) : Index of the target sentence (starts at 0)
        future (bool) : If true, use the vectors from the future instead of the past
    Ret:
        context vector (Tensor [batch, encoder dim])
    """

    if target == 0 and not future:
        #return encoded_target[0, :, :]
        return torch.zeros(encoded_sents[0,:,:].shape).cuda() if use_cuda else torch.zeros(encoded_sents[0,:,:].shape)
    elif target == encoded_sents.shape[0]-1 and future:
        #return encoded_target[encoded_sents.shape[0]-1, :, :]
        return torch.zeros(encoded_sents[0,:,:].shape).cuda() if use_cuda else torch.zeros(encoded_sents[0,:,:].shape)

    if not future:
        sents = encoded_sents[:target, :, :] #[sents, batch, encoder dim]
    else:
        sents = encoded_sents[target+1:, :, :] #[sents, batch, encoder dim]

#    maxpool, _ = torch.max(sents, dim=0) #[batch, encoder dim]
    maxpool = torch.mean(sents, dim=0) #[batch, encoder dim]

    return maxpool

max_stars_repo_name: StonyBrookNLP/SLDS-Stories

max_issues_repo_name: StonyBrookNLP/SLDS-Stories

max_forks_repo_name: StonyBrookNLP/SLDS-Stories


text:
import numpy as np

def integrand_sin(x):
    return x**2 * np.sin(x)

def simpson(f, a, b, nstrips):
    x, dx = np.linspace(a, b, num=2*nstrips+1, endpoint=True, retstep=True)
    return dx / 3 * (f(x[0]) + f(x[-1]) + 4 * np.sum(f(x[1:-1:2])) + 2 * np.sum(f(x[2:-1:2])))

nstrips_all = 10 * 2**np.arange(8)
dx = 1 / nstrips_all
errs = np.zeros(len(nstrips_all))
for i in range(len(nstrips_all)):
    errs[i] = abs(simpson(integrand_sin, 0, 1, nstrips_all[i]) - (2 * np.sin(1) + np.cos(1) - 2))

max_stars_repo_name: IanHawke/numerical-methods-course

max_issues_repo_name: IanHawke/numerical-methods-course

max_forks_repo_name: IanHawke/numerical-methods-course


text:
import time
from collections import OrderedDict
import convpoint.knn.lib.python.nearest_neighbors as knn

import random
import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np
from modules.functional.backend import _backend
from modules.functional.sampling import gather, furthest_point_sample
import modules.functional as mf

__all__ = ['YOGO']

def knn_search(input_pts, query_pts, k):
    knn_idx  = knn.knn_batch(input_pts.permute(0, 2, 1).data.cpu(),
        query_pts.permute(0, 2, 1).data.cpu(), k
        )
    return knn_idx.astype(np.int64)

def conv1x1_1d(inplanes, out_planes, stride=1, groups=1):
    """1x1 convolution"""
    return nn.Conv1d(inplanes, out_planes, kernel_size=1, stride=stride,
                     groups=groups, bias=False)

def conv1x1(inplanes, out_planes, stride=1, groups=1):
    """1x1 convolution"""
    return nn.Conv1d(inplanes, out_planes, kernel_size=1, stride=stride,
                     groups=groups, bias=False)

class MatMul(nn.Module):
    """A wrapper class such that we can count the FLOPs of matmul
    """
    def __init__(self):
        super(MatMul, self).__init__()

    def forward(self, A, B):
        return torch.matmul(A, B)

class Transformer(nn.Module):
    def __init__(self, token_c, t_layer=1, head=2, kqv_groups=1,
                 norm_layer_1d=nn.Identity):
        super(Transformer, self).__init__()

        self.k_conv = nn.ModuleList()
        self.q_conv = nn.ModuleList()
        self.v_conv = nn.ModuleList()
        self.kqv_bn = nn.ModuleList()
        self.kq_matmul = nn.ModuleList()
        self.kqv_matmul = nn.ModuleList()
        self.ff_conv = nn.ModuleList()
        for _ in range(t_layer):
            self.k_conv.append(nn.Sequential(
                conv1x1_1d(token_c, token_c // 2, groups=kqv_groups),
                norm_layer_1d(token_c // 2)
            ))
            self.q_conv.append(nn.Sequential(
                conv1x1_1d(token_c, token_c // 2, groups=kqv_groups),
                norm_layer_1d(token_c // 2)
            ))
            self.v_conv.append(nn.Sequential(
                conv1x1_1d(token_c, token_c, groups=kqv_groups),
                norm_layer_1d(token_c)
            ))
            self.kq_matmul.append(MatMul())
            self.kqv_matmul.append(MatMul())
            self.kqv_bn.append(norm_layer_1d(token_c))
            # zero-init
            #nn.init.constant_(self.kqv_bn[-1].weight, 0)
            self.ff_conv.append(nn.Sequential(
                conv1x1_1d(token_c, token_c * 2),
                norm_layer_1d(token_c * 2),
                nn.ReLU(inplace=True),
                conv1x1_1d(token_c * 2, token_c),
                norm_layer_1d(token_c),
            ))
            # initialize the bn weight to zero to improves the training
            # stability.
            #nn.init.constant_(self.ff_conv[-1][1].weight, 1)

        self.token_c = token_c
        self.t_layer = t_layer
        self.head = head

    def forward(self, x):
        N = x.shape[0]
        for _idx in range(self.t_layer):
            k = self.k_conv[_idx](x).view(
                N, self.head, self.token_c // 2 // self.head, -1)
            q = self.q_conv[_idx](x).view(
                N, self.head, self.token_c // 2 // self.head, -1)
            v = self.v_conv[_idx](x).view(
                N, self.head, self.token_c // self.head, -1)
            # N, h, L, C/h * N, h, C/h, L -> N, h, L, L
            kq = self.kq_matmul[_idx](k.permute(0, 1, 3, 2), q)
            # N, h, L, L
            kq = F.softmax(kq / np.sqrt(self.token_c / 2 / self.head), dim=2)
            # N, h, C/h, L * N, h, L, L -> N, h, C/h, L
            kqv = self.kqv_matmul[_idx](v, kq).view(N, self.token_c, -1)
            kqv = self.kqv_bn[_idx](kqv)
            x = x + kqv
            x = x + self.ff_conv[_idx](x)

        return x

class Projector(nn.Module):
    def __init__(self, token_c, planes, head=2, min_group_planes=64,
                 norm_layer_1d=nn.Identity):
        super(Projector, self).__init__()

        if token_c != planes:
            self.proj_value_conv = nn.Sequential(
                conv1x1_1d(token_c, planes),
                norm_layer_1d(planes))
        else:
            self.proj_value_conv = nn.Identity()

        self.proj_key_conv = nn.Sequential(
            conv1x1_1d(token_c, planes),
            norm_layer_1d(planes)
        )
        self.proj_query_conv = nn.Sequential(
            conv1x1_1d(planes, planes),
            norm_layer_1d(planes)
        )
        self.proj_kq_matmul = MatMul()
        self.proj_matmul = MatMul()
        self.proj_bn = norm_layer_1d(planes)
        # zero-init
        #nn.init.constant_(self.proj_bn.weight, 1)

        self.ff_conv = nn.Sequential(
            conv1x1_1d(planes, 2 * planes),
            norm_layer_1d(2 * planes),
            nn.ReLU(inplace=True),
            conv1x1_1d(2 * planes, planes),
            norm_layer_1d(planes)
            )

        self.head = head

    def forward(self, x, x_t):
        N, _, L = x_t.shape
        h = self.head
        # -> N, h, C/h, L
        proj_v = self.proj_value_conv(x_t).view(N, h, -1, L)
        # -> N, h, C/h, L
        proj_k = self.proj_key_conv(x_t).view(N, h, -1, L)
        proj_q = self.proj_query_conv(x)
        N, C, _ = proj_q.shape
        # -> N, h, HW, c/H
        proj_q = proj_q.view(N, h, C // h, -1).permute(0, 1, 3, 2)
        # N, h, HW, C/h * N, h, C/h, L -> N, h, HW, L
        proj_coef = F.softmax(
            self.proj_kq_matmul(proj_q, proj_k) / np.sqrt(C / h), dim=3)

        # N, h, C/h, L * N, h, L, HW -> N, h, C/h, HW
        x_p = self.proj_matmul(proj_v, proj_coef.permute(0, 1, 3, 2))
        # -> N, C, H, W
        _, _, S = x.shape
        x_p = self.proj_bn(x_p.view(N, -1, S))

        x = x + self.ff_conv(x + x_p)

        return x

class RIM(nn.Module):
    def __init__(self, token_c, input_dims, output_dims,
                 head=2, min_group_planes=1, norm_layer_1d=nn.Identity,
                 **kwargs):
        super(RIM, self).__init__()

        self.transformer = Transformer(
            token_c, norm_layer_1d=norm_layer_1d, head=head,
            **kwargs)

        if input_dims == output_dims:
            self.feature_block = nn.Identity()
        else:
            self.feature_block = nn.Sequential(
                    conv1x1_1d(input_dims, output_dims),
                    norm_layer_1d(output_dims)
                    )

        self.projectors = Projector(
                    token_c, output_dims, head=head,
                    min_group_planes=min_group_planes,
                    norm_layer_1d=norm_layer_1d)

        self.dynamic_f = nn.Sequential(
            conv1x1_1d(input_dims, token_c),
            norm_layer_1d(token_c),
            nn.ReLU(inplace=True),
            conv1x1_1d(token_c, token_c),
            norm_layer_1d(token_c)
            )

    def forward(self, in_feature, in_tokens, knn_idx):
        #in_feature: B, N, C
        #in_coords: B, N, 3
        B, L, K = knn_idx.shape
        B, C, N = in_feature.shape

        gather_fts = gather(
                in_feature, knn_idx.view(B, -1)
                ).view(B, -1, L, K)

        tokens = self.dynamic_f(gather_fts.max(dim=3)[0])

        t_c = tokens.shape[1]

        if in_tokens is not None:
            tokens += in_tokens

        tokens = self.transformer(tokens)

        out_feature = self.projectors(
                    self.feature_block(in_feature), tokens
                    )

        return out_feature, tokens

class RIM_ResidualBlock(nn.Module):
    def __init__(self, inc, outc, token_c, norm_layer_1d):
        super(RIM_ResidualBlock, self).__init__()
        if inc != outc:
            self.res_connect = nn.Sequential(
                    nn.Conv1d(inc, outc, 1),
                    norm_layer_1d(outc),
                    )
        else:
            self.res_connect = nn.Identity()
        self.vt1 = RIM(
               token_c, inc, inc, norm_layer_1d=norm_layer_1d)
        self.vt2 = RIM(
               token_c, inc, outc, norm_layer_1d=norm_layer_1d)

    def forward(self, inputs):
        in_feature, tokens, knn_idx = inputs
        out, tokens = self.vt1(in_feature, tokens, knn_idx)
        out, tokens = self.vt2(out, tokens, knn_idx)

        return out, tokens

class YOGO(nn.Module):
    def __init__(self, **kwargs):
        super().__init__()

        cr = kwargs['width_r']
        cs = [32, 64, 128, 256, 256]
        cs = [int(cr * x) for x in cs]

        self.token_l = kwargs['token_l']
        self.token_s = kwargs['token_s']
        self.token_c = kwargs['token_c']

        self.group_ = kwargs['group']
        self.ball_r = kwargs['ball_r']

        norm_layer = kwargs['norm']

        self.stem = nn.Sequential(
            conv1x1_1d(22, cs[0]),
            norm_layer(cs[0]),
            nn.ReLU(inplace=True),
            conv1x1_1d(cs[0], cs[0]),
            norm_layer(cs[0])
        )

        self.stage1 = nn.Sequential(
            RIM_ResidualBlock(cs[0], cs[1], token_c=self.token_c, norm_layer_1d=norm_layer),
        )

        self.stage2 = nn.Sequential(
            RIM_ResidualBlock(cs[1], cs[2], token_c=self.token_c, norm_layer_1d=norm_layer),
        )

        self.stage3 = nn.Sequential(
            RIM_ResidualBlock(cs[2], cs[3], token_c=self.token_c, norm_layer_1d=norm_layer),
        )

        self.stage4 = nn.Sequential(
            RIM_ResidualBlock(cs[3], cs[4], token_c=self.token_c, norm_layer_1d=norm_layer),
        )

        self.classifier = nn.Sequential(
            conv1x1_1d(cs[4], cs[4]),
            norm_layer(cs[4]),
            nn.ReLU(inplace=True),
            conv1x1_1d(cs[4], kwargs['num_classes']),
            )

    def forward(self, x):

        coords = x[:, :3, :]

        one_hot_vectors = x[:, -16:, :]

        B, _, N = x.shape

        feature_stem = self.stem(x)
        if self.training:
            token_l = int(
                np.random.randint(self.token_l-8, self.token_l+8))
            center_pts = furthest_point_sample(
                coords, token_l)
        else:
            center_pts = furthest_point_sample(
                coords, self.token_l)

        if self.group_ == 'ball_query':
            knn_idx = mf.ball_query(
                center_pts, coords, self.ball_r, self.token_s
                )
        else:
            knn_idx = knn_search(coords, center_pts, self.token_s)
            knn_idx = torch.from_numpy(knn_idx).cuda()

        feature1, tokens = self.stage1((feature_stem, None, knn_idx))

        feature2, tokens = self.stage2((feature1, tokens, knn_idx))

        feature3, tokens = self.stage3((feature2, tokens, knn_idx))

        feature4, tokens = self.stage4((feature3, tokens, knn_idx))

        out = self.classifier(feature4)
        return out


text:
import numpy as np
import pytest

from cocos.numerics._utilities import _compute_slice_length

test_data = [(slice(0, 3, 2), np.array([1, 2, 3, 4, 5, 6, 7, 8])),
             (slice(1, 3, 2), np.array([1, 2, 3, 4, 5, 6, 7, 8])),
             (slice(2, 3, 2), np.array([1, 2, 3, 4, 5, 6, 7, 8])),
             (slice(3, 3, 2), np.array([1, 2, 3, 4, 5, 6, 7, 8])),
             (slice(1, 3, 3), np.array([1, 2, 3, 4, 5, 6, 7, 8])),
             (slice(1, 2, 2), np.array([1, 2, 3, 4, 5, 6, 7, 8])),
             (slice(3, 3, 2), np.array([1, 2, 3, 4, 5, 6, 7, 8]))]

@pytest.mark.parametrize("index, x", test_data)
def test_compute_slice_length(index: slice, x: np.ndarray):
    y = x[index]
    computed_length = _compute_slice_length(index, len(x))
    if not len(y) == computed_length:
        print(f'computed length = {computed_length}, actual length = {len(y)}')
    assert len(y) == computed_length

max_stars_repo_name: michaelnowotny/cocos

max_issues_repo_name: michaelnowotny/cocos

max_forks_repo_name: michaelnowotny/cocos


text:
# Module to solve the Thomas-Fermi problem in 1D

import numpy as np

def create_K_matrix(x, E_scale=1.0, sigma=1.0, x_0 = 1.0):
    '''
    Input:
        x : discrete 1D grid
        E_scale : energy scale for the K matrix, default units is eV
        sigma : impact paramter to prevent blow up at the same point
    Output:
        K : matrix of size x.size times x.size

    K(x1,x2) = E_scale / sqrt((x1 - x2)^2 + sigma^2)
    '''
    r = np.abs(x[:,np.newaxis] - x)
    K = E_scale/np.sqrt(r**2 + sigma**2)*np.exp(-r/x_0)
    return K

def create_A_matrix(x,V,K,mu_l,N_dot,mask,dot_info,lead_info):
    '''
    Convinience function
    Input:
        x    : discrete 1D grid
        V    : potential
        K    : Coulomb interaction matrix between two points
        mask : array specifying the nature of each point 'l1','d','b' or 'l2'
        dot_info : information about number of dots and their start and end points
    Output:
        A : matrix A used in solution of TF, A z = b
    '''

    #set up the A matrix
    N_grid = len(x)
    # 2 are leads
    N_D = len(dot_info)
    N_islands = N_D + 2

    N_bar = len(filter(lambda x: x == 'b',mask))
    N_A = N_grid + N_islands + N_bar
    A = np.zeros([N_A,N_A])

    A[:N_grid,:N_grid] = K
    # bottom left part
    B = np.zeros((N_D,N_grid))
    # constraint for sum of electron density over a dot
    for i in range(N_D):
        B[i,dot_info[i][0]:dot_info[i][1] + 1] = 1

    # set the sum and mu_dot constraint equations
    A[N_grid:N_grid + N_D,:N_grid] = B
    A[:N_grid,N_grid:N_grid + N_D] = -B.T

    for i in range(len(lead_info)):
        # sum over leads
        A[N_grid + N_D + i,lead_info[i][0]:lead_info[i][1] + 1] = 1
        # minus the unkown electron number on the leads
        A[N_grid + N_D + i,N_grid + N_D + i] = -1

    # barrier lagrange multiplier unknowns
    bar_constraint = 0
    for i in range(len(mask)):
        if mask[i] == 'b':
            A[i,N_grid + N_islands + bar_constraint] = -1
            bar_constraint += 1
    # barrier n = 0
    bar_constraint = 0
    for i in range(len(mask)):
        if mask[i] == 'b':
            A[N_grid + N_islands + bar_constraint,i] = 1
            bar_constraint += 1
    return A

def create_b_matrix(x,V,K,mu_l,N_dot,mask,dot_info,lead_info):
    '''
    Convinience function
    Input:
        x    : discrete 1D grid
        V    : potential
        K    : Coulomb interaction matrix between two points
        mask : array specifying the nature of each point 'l1','d','b' or 'l2'
        dot_info : information about number of dots and their start and end points
        lead_info : information about leads and their start and end points
        mu_l : (mu_L1, mu_L2) tuple with the lead potentials
        N_dot: vector with number of electrons in each dot, can be of size 0 i.e no dot
    Output:
        b : vector b used in solution of TF, A z = b
    '''
    N_grid = len(x)
    # 2 are leads
    N_D = len(dot_info)
    N_islands = N_D + 2
    N_bar = len(filter(lambda x: x == 'b',mask))

    N_A = N_grid + N_islands + N_bar

    # set up the RHS
    b = np.zeros(N_A)
    b[:N_grid] = -V

    # lead potentials on the RHS
    # notice the all essential +=
    for i in range(len(lead_info)):
        b[lead_info[i][0]:lead_info[i][1] + 1] += mu_l[i]

    b[N_grid:N_grid+N_D] = N_dot
    return b

def solve_thomas_fermi(x,V,K,mu_l,N_dot):
    '''
    Input:
        x    : discrete 1D grid
        V    : potential
        K    : Coulomb interaction matrix between two points
        mu_l : (mu_L1, mu_L2) tuple with the lead potentials
        N_dot: vector with number of electrons in each dot, can be of size 0 i.e no dot
    Output:
        (n, mu) where
        n    : electronic charge density as a function of x
        mu   : chemical potential as a function of x
               mu(x) = mu_L when x in leads

    Solves the Thomas-Fermi equation V - mu + K n = 0 along with the constraint that integral of electron density in a dot is a constant and electron density in the barrier region is zero.
    '''
    #solve the equation A z = b
    # z = (n mu)^T
    N_grid = len(x)

    mu_avg = 0.5*(mu_l[0] + mu_l[1])
    mask = dot_classifier.get_mask(x,V,K,mu_avg)
    # dictionary index by dot number, gives [dot_begin_index,dot_end_index]
    dot_info = dot_classifier.get_dot_info(mask)
    lead_info = dot_classifier.get_lead_info(mask)

    A = create_A_matrix(x,V,K,mu_l,N_dot,mask,dot_info,lead_info)
    b = create_b_matrix(x,V,K,mu_l,N_dot,mask,dot_info,lead_info)
    z = np.linalg.solve(A,b)

    # return the electron density and the dot chemical potentials
    n,mu = z[:N_grid],z[N_grid:N_grid + len(dot_info)]
    # also add the lead chemical potentials to simplify further calculations
    mu = np.concatenate(([mu_l[0]],mu,[mu_l[1]]))
    return n,mu

def calculate_thomas_fermi_energy(V,K,n,mu,N_dot):
    '''
    Input:
        V : potential profile
        K : Coulomb interaction matrix
        n : electorn density
        mu : chemical potenial profile, includes leads and dot potentials
    Output:
        E : Thomas-Fermi energy

    E = V n + 1/2 n K n
    '''
    E = np.sum(V*n) + 0.5 * np.sum(n*np.dot(K,n.T))
    return E

max_stars_repo_name: jpzwolak/quantum-ml

max_issues_repo_name: jpzwolak/quantum-ml

max_forks_repo_name: jpzwolak/quantum-ml


text:
import numpy as np

from ..consts.constants import *

class Cell(object):
    """Representations of cellular network cells.

    Attributes:
        _xcoord: (number) the x coordinate of the cell.
        _ycoord: (number) the y coordinate of the cell.
        _cell_type: (str) the type of the cell:
            - fixed_macro
            - macro
            - mirco
            - nano
            - pico
        _connected_users: (list of) users currently connected to the cell.
        _state: (boolean) the state of the cell(turned on or not).
        _cost: (int) the cost of deploying a cell.
        _min_users: (int) the minimum number of users that must be maintained for
                  the cell to remain active.
        _max_users: (int) the maximum number of users that the cell can hold.
        _radius: (number) the radius that the cell covers.
        _power: (number) the power at which the cell is operating.
        _frequency: (number) the frequency at which the cell is operating.
    """

    def __init__(self, xcoord, ycoord, cell_type):

        self._xcoord = xcoord
        self._ycoord = ycoord
        self._cell_type = cell_type
        self._connected_users = []
        self._state = True

        self._set_attributes()

    # getters
    def get_xcoord(self):
        return self._xcoord

    def get_ycoord(self):
        return self._ycoord

    def get_num_connected_users(self):
        return len(self._connected_users)

    def get_min_users(self):
        return self._min_users

    def get_cell_type(self):
        return self._cell_type

    def get_state(self):
        return self._state

    def get_cost(self):
        return self._cost

    def get_radius(self):
        return self._radius

    def get_power(self):
        return self._power

    def get_frequency(self):
        return self._frequency

    # setters
    def set_coords(self, x, y):
        self._xcoord = x
        self._ycoord = y

    def set_state(self, state):
        self._state = state

    def _set_attributes(self):
        """Set the values of attributes depending on the cell type."""
        properties = {
            "fixed_macro":
            {
                "cost": FIXED_MACRO_COST,
                "min_users": FIXED_MACRO_MIN_USERS,
                "max_users": FIXED_MACRO_MAX_USERS,
                "radius": FIXED_MACRO_RADIUS,
                "power": FIXED_MACRO_POWER,
                "frequency": FIXED_MACRO_FREQ
            },

            "macro":
            {
                "cost": MACRO_COST,
                "min_users": MACRO_MIN_USERS,
                "max_users": MACRO_MAX_USERS,
                "radius": MACRO_RADIUS,
                "power": MACRO_POWER,
                "frequency": MACRO_FREQ
            },

            "micro":
            {
                "cost": MICRO_COST,
                "min_users": MICRO_MIN_USERS,
                "max_users": MICRO_MAX_USERS,
                "radius": MICRO_RADIUS,
                "power": MICRO_POWER,
                "frequency": SMALL_CELL_FREQ
            },

            "pico":
            {
                "cost": PICO_COST,
                "min_users": PICO_MIN_USERS,
                "max_users": PICO_MAX_USERS,
                "radius": PICO_RADIUS,
                "power": PICO_POWER,
                "frequency": SMALL_CELL_FREQ
            },

            "femto":
            {
                "cost": FEMTO_COST,
                "min_users": FEMTO_MIN_USERS,
                "max_users": FEMTO_MAX_USERS,
                "radius": FEMTO_RADIUS,
                "power": FEMTO_POWER,
                "frequency": SMALL_CELL_FREQ
            }
        }

        cell_type = self.get_cell_type()
        self._cost = properties[cell_type]["cost"]
        self._min_users = properties[cell_type]["min_users"]
        self._max_users = properties[cell_type]["max_users"]
        self._radius = properties[cell_type]["radius"]
        self._power = properties[cell_type]["power"]
        self._frequency = properties[cell_type]["frequency"]

    def add_user(self, user):
        """Adds the user to the list of connected users."""
        self._connected_users.append(user)

    def is_available(self):
        """Returns whether the base station is available for the user to connect to."""

        if self.get_num_connected_users() < self._max_users:
            return True
        return False

    def check_if_needed(self):
        """Checks if the cell is neeeded based on the number of connected users.

        if the number of connected users is less than the minimum then the cell
        is turned off and the users are left unconnected."""

        if self.get_num_connected_users() < self.get_min_users():
            self.set_state(False)

            # remove each user
            for user in self._connected_users:
                user.set_connected_bs(None)
            self._connected_users = []

    def pprint(self):
        """Returns information about cell in a human readable format."""

        return """
        x = {}
        y = {}
        type = {}
        # users = {}
        """.format(self.get_xcoord(),
                   self.get_ycoord(),
                   self.get_cell_type(),
                   self.get_num_connected_users())


text:
# -*- coding: utf-8 -*-
"""Padding transformer, pad unequal length panel to max length or fixed length."""
import numpy as np
import pandas as pd

from sktime.transformations.base import BaseTransformer

__all__ = ["PaddingTransformer"]
__author__ = ["abostrom"]

class PaddingTransformer(BaseTransformer):
    """Padding panel of unequal length time series to equal, fixed length.

    Pads the input dataset to either a optional fixed length
    (longer than the longest series).
    Or finds the max length series across all series and dimensions and
    pads to that with zeroes.

    Parameters
    ----------
    pad_length  : int, optional (default=None) length to pad the series too.
                if None, will find the longest sequence and use instead.
    """

    _tags = {
        "scitype:transform-input": "Series",
        # what is the scitype of X: Series, or Panel
        "scitype:transform-output": "Series",
        # what scitype is returned: Primitives, Series, Panel
        "scitype:instancewise": False,  # is this an instance-wise transform?
        "X_inner_mtype": "nested_univ",  # which mtypes do _fit/_predict support for X?
        "y_inner_mtype": "None",  # which mtypes do _fit/_predict support for X?
        "fit_is_empty": False,
    }

    def __init__(self, pad_length=None, fill_value=0):
        self.pad_length = pad_length
        self.fill_value = fill_value
        super(PaddingTransformer, self).__init__()

    def _fit(self, X, y=None):
        """Fit transformer to X and y.

        private _fit containing the core logic, called from fit

        Parameters
        ----------
        X : nested pandas DataFrame of shape [n_instances, n_features]
            each cell of X must contain pandas.Series
            Data to fit transform to
        y : ignored argument for interface compatibility
            Additional data, e.g., labels for transformation

        Returns
        -------
        self : reference to self
        """
        if self.pad_length is None:
            n_instances, _ = X.shape
            arr = [X.iloc[i, :].values for i in range(n_instances)]
            self.pad_length_ = _get_max_length(arr)
        else:
            self.pad_length_ = self.pad_length

        return self

    def _create_pad(self, series):
        out = np.full(self.pad_length_, self.fill_value, float)
        out[: len(series)] = series.iloc[: len(series)]
        return out

    def _transform(self, X, y=None):
        """Transform X and return a transformed version.

        private _transform containing core logic, called from transform

        Parameters
        ----------
        X : nested pandas DataFrame of shape [n_instances, n_features]
            each cell of X must contain pandas.Series
            Data to fit transform to
        y : ignored argument for interface compatibility
            Additional data, e.g., labels for transformation

        Returns
        -------
        Xt : nested pandas DataFrame of shape [n_instances, n_features]
            each cell of Xt contains pandas.Series
            transformed version of X
        """
        n_instances, _ = X.shape

        arr = [X.iloc[i, :].values for i in range(n_instances)]

        max_length = _get_max_length(arr)

        if max_length > self.pad_length_:
            raise ValueError(
                "Error: max_length of series \
                    is greater than the one found when fit or set."
            )

        pad = [pd.Series([self._create_pad(series) for series in out]) for out in arr]
        Xt = pd.DataFrame(pad).applymap(pd.Series)

        return Xt

def _get_max_length(X):
    def get_length(input):
        return max(map(lambda series: len(series), input))

    return max(map(get_length, X))

max_stars_repo_name: FedericoGarza/sktime

max_issues_repo_name: FedericoGarza/sktime

max_forks_repo_name: FedericoGarza/sktime


text:
import numpy as np
from ..unit import Unit
from murmeltier.utils import assert_equal_or_none

class Identity(Unit):
    '''
    Lets values pass without modifying them
    '''
    def __init__(self, in_specs = None, out_specs = None, **kwargs):
        in_specs = out_specs = assert_equal_or_none(in_specs = in_specs, out_specs = out_specs)
        self.config(in_specs = in_specs, out_specs = out_specs)

    def get_output(self, input):
        return input

max_stars_repo_name: malyvsen/evo-ai

max_issues_repo_name: malyvsen/evo-ai

max_forks_repo_name: malyvsen/evo-ai


text:
import rolldecayestimators.special_symbol as ss
import sympy as sp
import sympy.physics.mechanics as me

t = ss.Symbol(name='t',description='time',unit='s')
delta = ss.Symbol(name='delta',description='Rudder angle',unit='rad')
r = me.dynamicsymbols('r')  # yaw rate [rad/s]
r_1d = r.diff()
r_2d = r_1d.diff()

K = ss.Symbol(name='K',description='Nomoto coefficient',unit=r'1/s')
T_1 = ss.Symbol(name='T_1',description='Nomoto time coefficient',unit='s')
T_2 = ss.Symbol(name='T_2',description='Nomoto time coefficient',unit='s')

max_stars_repo_name: martinlarsalbert/manoeuvring

max_issues_repo_name: martinlarsalbert/manoeuvring

max_forks_repo_name: martinlarsalbert/manoeuvring

reason: import sympy