text:
# coding=utf-8

"""
Module to apply a previously trained model to estimate the epigenome
for a specific cell type in a different species
"""

import os as os
import pandas as pd
import numpy as np
import numpy.random as rng
import operator as op
import multiprocessing as mp
import json as json
import pickle as pck

from scipy.interpolate import LSQUnivariateSpline as kspline

from crplib.auxiliary.seq_parsers import get_twobit_seq
from crplib.auxiliary.hdf_ops import load_masked_sigtrack, get_valid_hdf5_groups, get_mapindex_groups
from crplib.auxiliary.file_ops import create_filepath
from crplib.auxiliary.modeling import select_dataset_subset, load_model, \
    load_model_metadata, get_scorer, load_ml_dataset, determine_scoring_method, apply_preprocessor
from crplib.mlfeat.featdef import feat_mapsig, get_online_version
from crplib.auxiliary.constants import CHROMOSOME_BOUNDARY
from crplib.metadata.md_signal import MD_SIGNAL_COLDEFS
from crplib.metadata.md_signal import gen_obj_and_md as gen_sigobj
from crplib.metadata.md_regions import MD_REGION_COLDEFS
from crplib.metadata.md_regions import gen_obj_and_md as genregobj

def load_dataset(fpath, groups, features, subset='', ycol='', ytype=None):
    """
    :param fpath:
    :param groups:
    :param features:
    :param subset:
    :param ycol:
    :param ytype:
    :return:
    """
    with pd.HDFStore(fpath, 'r') as hdf:
        if isinstance(groups, (list, tuple)):
            dataset = pd.concat([hdf[grp] for grp in sorted(groups)], ignore_index=True)
        else:
            dataset = hdf[groups]
        dataset = select_dataset_subset(dataset, subset)
        y_depvar = None
        if ytype is not None:
            y_depvar = dataset.loc[:, ycol].astype(ytype, copy=True)
        name_col = [cn for cn in dataset.columns if cn in ['name', 'source']]
        sample_names = []
        if name_col:
            sample_names = dataset.loc[:, name_col[0]].tolist()
        predictors = dataset.loc[:, features]
    return predictors, sample_names, y_depvar

def run_permutation_test(data, output, model, numperm, scorer, extra_scorer=''):
    """
    :param data:
    :param output:
    :param model:
    :param numperm:
    :param scorer:
    :return:
    """
    perm_scores = []
    extra_scores = []
    if extra_scorer:
        extra_name = extra_scorer
        extra_scorer = get_scorer(extra_scorer)
    for _ in range(numperm):
        perm_out = rng.permutation(output)
        perm_scores.append(scorer(model, data, perm_out))
        if extra_scorer:
            extra_scores.append(extra_scorer(model, data, perm_out))
    # the float here for later JSONification
    assert len(perm_scores) == numperm, \
        'Permutation failed, generated only {} permutation scores, needed {}'.format(len(perm_scores), numperm)
    perm_stats = {'perm_mean': float(np.mean(perm_scores)),
                  'perm_median': float(np.median(perm_scores)),
                  'perm_min': float(np.min(perm_scores)),
                  'perm_std': float(np.std(perm_scores)),
                  'perm_var': float(np.var(perm_scores)),
                  'perm_max': float(np.max(perm_scores)),
                  'perm_95pct': float(np.percentile(perm_scores, 95)),
                  'perm_5pct': float(np.percentile(perm_scores, 5))}
    if extra_scorer:
        extra_stats = {'perm_mean': float(np.mean(extra_scores)),
                       'perm_median': float(np.median(extra_scores)),
                       'perm_min': float(np.min(extra_scores)),
                       'perm_std': float(np.std(extra_scores)),
                       'perm_var': float(np.var(extra_scores)),
                       'perm_max': float(np.max(extra_scores)),
                       'perm_95pct': float(np.percentile(extra_scores, 95)),
                       'perm_5pct': float(np.percentile(extra_scores, 5))}
        perm_stats['perm_extra_{}'.format(extra_name)] = extra_stats
    return perm_stats

############################

def smooth_signal_estimate(signal, res):
    """
    :param signal:
    :param res:
    :return:
    """
    # for default resolution of 25, spans roughly one nucleosome
    window = res * 10
    for pos in range(CHROMOSOME_BOUNDARY, len(signal), window):
        try:
            smoother = kspline(range(pos, pos+window),  # x-axis
                               signal[pos:pos+window],  # y-axis
                               t=[j+res for j in range(pos, pos + window - res, res)],  # knots
                               k=3)  # degree of polynomial, cubic is default
            signal[pos:pos+window] = smoother(range(pos, pos+window))
        except Exception as err:
            if pos + window > len(signal):
                break
            else:
                raise err
    signal = np.clip(signal, 0, signal.max())
    return signal

def make_signal_estimate(params):
    """
    :param params:
    :return:
    """
    chrom = params['chrom']
    model = pck.load(open(params['modelfile'], 'rb'))
    seq = get_twobit_seq(params['seqfile'], params['chrom'])
    chromlen = len(seq)
    res = params['resolution']
    index_groups = get_mapindex_groups(params['targetindex'], '')
    with pd.HDFStore(params['targetindex'], 'r') as idx:
        mask = idx[index_groups[chrom]['mask']]
    map_sig = load_masked_sigtrack(params['inputfile'], '',
                                   params['inputgroup'], chrom, chromlen, mask=mask)
    mapfeat = feat_mapsig
    est_sig = np.zeros(chromlen, dtype=np.float64)
    lolim = CHROMOSOME_BOUNDARY
    hilim = int(chromlen // res * res)
    comp_seqfeat = get_online_version(params['features'], params['kmers'])
    get_values = op.itemgetter(*tuple(params['feature_order']))
    chunks = []
    positions = []
    for pos in range(lolim, hilim, res):
        chunk = {'seq': seq[pos:pos+res]}
        positions.append(pos)
        chunk.update(mapfeat(map_sig[pos:pos+res]))
        chunk = comp_seqfeat(chunk)
        chunks.append(get_values(chunk))
        if len(chunks) >= 10000:
            y_hat = model.predict(np.array(chunks))
            for idx, val in zip(positions, y_hat):
                est_sig[idx:idx+res] = val
            chunks = []
            positions = []
    if len(chunks) > 0:
        y_hat = model.predict(np.array(chunks))
        for idx, val in zip(positions, y_hat):
            est_sig[idx:idx+res] = val
    if not params['nosmooth']:
        est_sig = smooth_signal_estimate(est_sig, res)
    return chrom, est_sig

def assemble_params_estsig(args):
    """
    :param args:
    :return:
    """
    all_groups = get_valid_hdf5_groups(args.inputfile, args.inputgroup)
    model_md = load_model_metadata(args)
    commons = {'modelfile': args.modelfile, 'resolution': int(model_md['resolution']),
               'seqfile': args.seqfile, 'targetindex': args.targetindex, 'inputfile': args.inputfile,
               'inputgroup': args.inputgroup, 'features': model_md['features'],
               'kmers': model_md['kmers'], 'feature_order': model_md['feature_order'],
               'nosmooth': args.nosmooth}
    arglist = []
    for g in all_groups:
        chrom = g.rsplit('/', 1)[1]
        tmp = dict(commons)
        tmp['chrom'] = chrom
        arglist.append(tmp)
    return arglist

def run_estimate_signal(logger, args):
    """
    :param logger:
    :param args:
    :return:
    """
    logger.debug('Assembling worker parameters')
    arglist = assemble_params_estsig(args)
    logger.debug('Created parameter list of size {}'.format(len(arglist)))
    with pd.HDFStore(args.outputfile, 'w', complevel=9, complib='blosc', encoding='utf-8') as hdfout:
        metadata = pd.DataFrame(columns=MD_SIGNAL_COLDEFS)
        with mp.Pool(args.workers) as pool:
            logger.debug('Start processing...')
            resit = pool.imap_unordered(make_signal_estimate, arglist, chunksize=1)
            for chrom, valobj in resit:
                logger.debug('Processed chromosome {}'.format(chrom))
                group, valobj, metadata = gen_sigobj(metadata, args.outputgroup, chrom, args.inputfile, valobj)
                hdfout.put(group, valobj, format='fixed')
                hdfout.flush()
                logger.debug('Estimated signal data saved')
        logger.debug('Saving metadata')
        hdfout.put('metadata', metadata)
        hdfout.flush()
    logger.debug('All chromosomes processed')
    return 0

def build_output_dataframe(dset, pred, probs, classes, merge, reduce):
    """
    :param dset:
    :param pred:
    :param probs:
    :param classes:
    :param merge:
    :return:
    """
    dset = dset.assign(class_pred=pred)
    class_cols = ['class_prob_' + cls for cls in map(str, list(map(int, classes)))]
    dset = pd.concat([dset, pd.DataFrame(probs, columns=class_cols)], axis='columns')
    if reduce:
        dset = dset.loc[dset.class_pred.isin(reduce), :]
    if merge:
        assert len(reduce) == 1, 'Merging overlapping regions in dataset w/o reducing to single class not supported'
        dset.drop([col for col in dset.columns if col.startswith('ft')], axis='columns', inplace=True)
        dset.sort_values(by=['start', 'end'], axis='index', ascending=True, inplace=True)
        dset.index = np.arange(dset.shape[0])
        # TODO
        # in spare time, find out if there is a more
        # native way in Pandas to merge overlapping intervals...
        new_rows = []
        cur_start = dset.loc[0, 'start']
        cur_end = dset.loc[0, 'end']
        cur_probs = []
        cur_names = set()
        get_name = 'name' if 'name' in dset.columns else 'source'
        assert get_name in dset.columns, 'No naming column exists for regions'
        class_prob = 'class_prob_' + str(reduce[0])
        for row in dset.itertuples(index=False):
            if row.start <= cur_end:
                cur_end = row.end
                cur_probs.append(row.__getattribute__(class_prob))
                cur_names.add(row.__getattribute__(get_name))
            else:
                regname = cur_names.pop()  # return any or unique
                new_rows.append([cur_start, cur_end, regname, np.average(cur_probs)])
                cur_names = set()
                cur_probs = []
                cur_start = row.start
                cur_end = row.end
                cur_probs.append(row.__getattribute__(class_prob))
                cur_names.add(row.__getattribute__(get_name))
        new_cols = ['start', 'end', 'name', 'class_prob']
        dset = pd.DataFrame(new_rows, columns=new_cols)
    return dset

def model_scan_regions(params):
    """
    :param params:
    :return:
    """
    model_md = json.load(open(params['modelmetadata'], 'r'))
    model = pck.load(open(params['modelfile'], 'rb'))
    feat_order = model_md['feature_order']
    featdata, _, _ = load_region_data(params['inputfile'], params['inputgroup'], feat_order, params['classlabels'])
    y_pred = model.predict(featdata)
    y_prob = model.predict_proba(featdata)
    class_order = model.classes_
    with pd.HDFStore(params['inputfile'], 'r') as hdf:
        full_dataset = hdf[params['inputgroup']]
    df = build_output_dataframe(full_dataset, y_pred, y_prob, class_order, params['merge'], params['reduce'])
    return params['chrom'], df

def assemble_params_scnreg(args, logger):
    """
    :return:
    """
    all_groups = get_valid_hdf5_groups(args.inputfile, args.inputgroup)
    logger.debug('Identified {} valid groups in input file'.format(len(all_groups)))
    merge_regions = False
    with pd.HDFStore(args.inputfile, 'r') as hdf:
        md = hdf['metadata']
        res = md.loc[0, 'resolution']
        if res > 0:
            merge_regions = True
    logger.debug('Detected - merge regions: {}'.format(merge_regions))
    if not args.modelmetadata:
        fpath_md = args.modelfile.rsplit('.', 1)[0] + '.json'
    else:
        fpath_md = args.modelmetadata
    commons = vars(args)
    del commons['module_logger']
    del commons['execute']
    commons['modelmetadata'] = fpath_md
    arglist = []
    for g in all_groups:
        tmp = dict(commons)
        tmp['inputgroup'] = g
        _, tmp['chrom'] = os.path.split(g)
        tmp['merge'] = merge_regions
        arglist.append(tmp)
    logger.debug('Build argument list of size {} to process'.format(len(arglist)))
    return arglist

def run_scan_regions(logger, args):
    """
    :param logger:
    :param args:
    :return:
    """
    arglist = assemble_params_scnreg(args, logger)
    with pd.HDFStore(args.outputfile, 'w') as hdf:
        metadata = pd.DataFrame(columns=MD_REGION_COLDEFS)
        with mp.Pool(args.workers) as pool:
            resit = pool.imap_unordered(model_scan_regions, arglist, chunksize=1)
            for chrom, dataobj in resit:
                logger.debug('Received data for chromosome {}'.format(chrom))
                grp, dataobj, metadata = genregobj(metadata, args.outputgroup, chrom, [args.inputfile, args.modelfile], dataobj)
                hdf.put(grp, dataobj, format='fixed')
                hdf.flush()
                logger.debug('Flushed data to file')
        hdf.put('metadata', metadata, format='table')
    return 0

def run_regression_testdata(args, model, model_md, dataset, y_true, logger):
    """
    :param args:
    :param model:
    :param model_md:
    :param dataset:
    :param y_true:
    :return:
    """
    logger.debug('Making prediction for test dataset')
    scoring_method = determine_scoring_method(args, model_md, logger)
    scorer = get_scorer(scoring_method)
    y_pred = model.predict(dataset)
    model_perf = scorer(model, dataset, y_true)
    out_metadata = {}
    if args.numperm > 0:
        logger.debug('Running permutation test')
        perf_score, permscores, permstats, permparams = run_permutation_test(dataset, y_true, model, args.cvperm,
                                                                             args.numperm, args.workers, scorer)
        if not np.isclose(model_perf, perf_score, rtol=1e-05, atol=1e-05):
            # turns out this happens regularly... not sure why
            logger.warning('Performance scores not close: {} vs {}'.format(model_perf, perf_score))
        out_metadata['permutation_test'] = {'perm_scores': permscores, 'perm_params': permparams,
                                            'perm_stats': permstats}
        logger.debug('Running randomization test')
        randstats = run_randomization_test(dataset, y_true, model, args.numperm, scorer)
        out_metadata['randomization_test'] = {'rand_num': args.numperm, 'rand_stats': randstats}
    logger.debug('Collecting metadata')
    out_metadata['scoring'] = scoring_method
    out_metadata['performance'] = float(model_perf)
    out_metadata['targets'] = {'true': list(map(float, y_true)),
                               'pred': list(map(float, y_pred))}
    logger.debug('Regression of test data finished')
    return {'testing_info': out_metadata}

def run_classification_testdata(args, model, model_md, dataset, y_true, logger):
    """
    :param args:
    :param model:
    :param model_md:
    :param dataset:
    :param y_true:
    :return:
    """
    logger.debug('Making prediction for test dataset')
    scoring_method = determine_scoring_method(args, model_md, logger)
    scorer = get_scorer(scoring_method)
    y_pred = model.predict(dataset)
    model_perf = scorer(model, dataset, y_true)
    class_order = list(map(int, model.classes_))
    logger.debug('Getting class probabilities')
    try:
        y_prob = pd.DataFrame(model.predict_proba(dataset), columns=class_order)
        pred_class_prob = (y_prob.lookup(y_prob.index, y_pred)).tolist()
        true_class_prob = (y_prob.lookup(y_prob.index, y_true)).tolist()
    except AttributeError:
        logger.warning('Model has no function to predict class probabilities - skipping...')
        pred_class_prob = []
        true_class_prob = []
        y_prob = pd.DataFrame(columns=class_order, index=[], dtype=np.float32)
    testing_info = dict()
    testing_info['scoring'] = scoring_method
    testing_info['performance'] = float(model_perf)
    if args.numperm > 0:
        logger.debug('Running permutation tests: {}'.format(args.numperm))
        permstats = run_permutation_test(dataset, y_true, model, args.numperm, scorer, args.extrascorer)
        testing_info['permutation_test'] = {'perm_num': args.numperm, 'perm_stats': permstats}
    logger.debug('Collecting metadata')
    testing_info['targets'] = {'true': list(map(int, y_true)),
                               'pred': list(map(int, y_pred)),
                               'order': class_order}
    testing_info['probabilities'] = {'true': true_class_prob,
                                     'pred': pred_class_prob,
                                     'all': [list(map(float, row)) for row in y_prob.as_matrix().tolist()]}
    logger.debug('Classification of test data finished')
    return {'testing_info': testing_info}

def run_classification_newdata(model, dataset, logger):
    """
    :param model:
    :param dataset:
    :return:
    """
    y_pred = model.predict(dataset)
    class_order = list(map(int, model.classes_))
    y_prob = pd.DataFrame(model.predict_proba(dataset), columns=class_order)
    pred_class_prob = list(map(float, y_prob.lookup(y_prob.index, y_pred)))
    out_metadata = {}
    logger.debug('Collecting metadata')
    out_metadata['targets'] = {'pred': list(map(int, y_pred)),
                               'order': class_order}
    out_metadata['probabilities'] = {'pred': pred_class_prob,
                                     'all': [list(map(float, row)) for row in y_prob.as_matrix().tolist()]}
    logger.debug('Classification of test data finished')
    return {'estimate_info': out_metadata}

def run_classification(args, model, modelmd, loadgroups, logger):
    """
    :param args:
    :param modelmd:
    :param logger:
    :return:
    """
    _ = create_filepath(args.outputfile, logger)
    logger.debug('Loading dataset')
    dataset, output, dtinfo, sminfo, ftinfo = load_ml_dataset(args.inputfile,
                                                              loadgroups,
                                                              modelmd['feature_info']['order'],
                                                              args, logger)
    orig_shape = dataset.shape
    if 'preprocess_info' in modelmd:
        logger.debug('Preprocessing data')
        dataset, _ = apply_preprocessor(dataset, modelmd['preprocess_info'], 'test')
        assert dataset.shape == orig_shape, 'Shape mismatch: {} {}'.format(orig_shape, dataset.shape)
    if args.task == 'test':
        out_md = run_classification_testdata(args, model, modelmd, dataset, output, logger)
    elif args.task == 'est':
        assert output is None, 'Loaded sample outputs from dataset for prediction task'
        out_md = run_classification_newdata(model, dataset, logger)
    else:
        raise ValueError('Unknown task for classification: {}'.format(args.task))
    runinfo = dict()
    runinfo['task'] = args.task
    runinfo['model_file'] = os.path.basename(args.modelfile)
    runinfo['data_file'] = os.path.basename(args.inputfile)
    runinfo['data_group'] = args.inputgroup

    out_md['model_info'] = modelmd['model_info']
    out_md['run_info'] = runinfo
    out_md['sample_info'] = sminfo
    out_md['feature_info'] = ftinfo
    out_md['dataset_info'] = dtinfo
    logger.debug('Writing metadata of run...')
    with open(args.outputfile, 'w') as outfile:
        _ = json.dump(out_md, outfile, indent=1, sort_keys=True)
    logger.debug('Metadata saved')
    return 0

def run_regression(args, model, modelmd, loadgroups, logger):
    """
    :param args:
    :param model:
    :param modelmd:
    :param loadgroups:
    :param logger:
    :return:
    """
    _ = create_filepath(args.outputfile, logger)
    logger.debug('Loading dataset')
    dataset, output, dtinfo, sminfo, ftinfo = load_ml_dataset(args.inputfile,
                                                              loadgroups,
                                                              modelmd['feature_info']['order'],
                                                              args, logger)
    if args.task == 'test':
        out_md = run_regression_testdata(args, model, modelmd, dataset, output, logger)
    elif args.task == 'est':
        raise NotImplementedError
        assert output is None, 'Loaded sample outputs from dataset for prediction task'

        out_md = run_classification_newdata(model, dataset, logger)
    else:
        raise ValueError('Unknown task for regression: {}'.format(args.task))
    runinfo = dict()
    runinfo['task'] = args.task
    runinfo['model_file'] = os.path.basename(args.modelfile)
    runinfo['data_file'] = os.path.basename(args.inputfile)
    runinfo['data_group'] = args.inputgroup

    out_md['model_info'] = modelmd['model_info']
    out_md['run_info'] = runinfo
    out_md['sample_info'] = sminfo
    out_md['feature_info'] = ftinfo
    out_md['dataset_info'] = dtinfo
    logger.debug('Writing metadata of run...')
    with open(args.outputfile, 'w') as outfile:
        _ = json.dump(out_md, outfile, indent=1, sort_keys=True)
    logger.debug('Metadata saved')
    return 0

def run_apply_model(args):
    """
    :param args:
    :return:
    """
    logger = args.module_logger
    logger.debug('Loading model and metadata...')
    model_md = load_model_metadata(args)
    logger.debug('Metadata successfully loaded')
    model = load_model(args.modelfile)
    logger.debug('Model successfully loaded')
    md_training = model_md['dataset_info']
    drv_trg = md_training['derive_target']
    trg_var = md_training['target_var']
    if drv_trg and drv_trg is not None:
        logger.debug('Found "derive_target" value in model metadata - overwriting...')
        args.__setattr__('derivetarget', drv_trg)
        args.__setattr__('targetvar', '')
    elif trg_var:
        logger.debug('Found "target_var" in model metadata - overwriting')
        args.__setattr__('derivetarget', '')
        args.__setattr__('targetvar', trg_var)
    else:
        raise AssertionError('Invalid model metadata - '
                             'target variable name has to be specified as '
                             '"target_var" in section "dataset_info": {}'.format(md_training))
    model_type = model_md['model_info']['type']
    logger.debug('Loading groups from input data file')
    load_groups = get_valid_hdf5_groups(args.inputfile, args.inputgroup)
    if model_type == 'classifier':
        logger.debug('Applying classification model {} on task: {}'.format(model_md['model_info']['name'], args.task))
        _ = run_classification(args, model, model_md, load_groups, logger)
    elif model_type == 'regressor':
        logger.debug('Applying regression model {} on task: {}'.format(model_md['model_info']['name'], args.task))
        _ = run_regression(args, model, model_md, load_groups, logger)
    else:
        raise NotImplementedError('No support for model of type: {} '
                                  '(just classifier or regressor are supported)'.format(model_type))
    return 0

max_stars_repo_name: ptrebert/creepiest

max_issues_repo_name: ptrebert/creepiest

max_forks_repo_name: ptrebert/creepiest


text:
# encoding: utf-8
"""
@author : zhirui zhou
@contact: evilpsycho42@gmail.com
@time   : 2020/4/27 15:11
"""
import pytest
from deepseries.analysis import SeriesAnalysisModel
import numpy as np

def test_analysis_model():
    x = np.random.rand(4, 500) + 1e-3
    x[0][0] = np.nan
    x[0][1] = 0
    model = SeriesAnalysisModel(x)
    model.plot_valid()
    model.get_trend(365).plot_trend()
    model.plot_trend(0)

    model.get_autocorr(np.arange(1, 300)).plot_autocorr()

    assert model.mask.sum() == 2
    assert model.valid_lens[0] == 498

if __name__ == "__main__":
    test_analysis_model()

ext: py

max_stars_repo_name: EvilPsyCHo/Deep-Time-Series-Prediction

max_issues_repo_name: luxixiang/Deep-Time-Series-Prediction

max_forks_repo_name: luxixiang/Deep-Time-Series-Prediction

reason: import numpy


text:
import numpy as np

from ...dimensions.dim_linear import DimLinear
from ..cross_sect_base import CrossSectBase, CrossSectToken

__all__ = ['CrossSectHollowCylinder']

class CrossSectHollowCylinder(CrossSectBase):

    def __init__(self, **kwargs: any) -> None:
        '''
        Intialization function for HollowCylinder class. This function takes in
        arguments and saves the information passed to private variable to make
        them read-only
        Parameters
        ----------
        **kwargs : any
            DESCRIPTION. Keyword arguments provided to the initialization funcntion.
            The following argument names have to be included in order for the code
            to execute: name, dim_t, dim_r_o, location.

        Returns
        -------
        None
        '''
        self._create_attr(kwargs)

        super()._validate_attr()
        self._validate_attr()

    @property
    def dim_t(self):
        return self._dim_t

    @property
    def dim_r_o(self):
        return self._dim_r_o

    def draw(self, drawer):

        r = self._dim_r_o  # outer radius of hollow cylinder
        t = self._dim_t  # thickness of hollow cylinder

        x_out = type(r)(0)  # assign intial origin as DimLinear object of 0
        x_in = type(r)(0)
        x = [x_out, x_out, x_in, x_in]

        y_out = r
        y_in = r - t
        y = [-y_out, y_out, -y_in, y_in]

        # transpose list
        coords = [x, y]
        coords = list(zip(*coords))
        coords = [list(sublist) for sublist in coords]

        p = self.location.transform_coords(coords)  # shift coordinates based on anchor and theta given in location

        # draw hollow cylinder
        arc_out1 = drawer.draw_arc(self.location.anchor_xy, p[0], p[1])
        arc_out2 = drawer.draw_arc(self.location.anchor_xy, p[1], p[0])
        arc_out3 = drawer.draw_arc(self.location.anchor_xy, p[2], p[3])
        arc_out4 = drawer.draw_arc(self.location.anchor_xy, p[3], p[2])

        # get coordinate within hollow cylinder
        rad = r - t * 0.5
        inner_coord = self.location.transform_coords([[rad, type(r)(0)]])

        token = [arc_out1, arc_out2, arc_out3, arc_out4]  # compile tokens

        cs_token = CrossSectToken(inner_coord[0], token)  # create CrossSectToken object
        return cs_token

    def _validate_attr(self):

        if isinstance(self._dim_r_o, DimLinear):
            pass
        else:
            raise TypeError("dim_r_o not of type DimLinear")

        if isinstance(self._dim_t, DimLinear):
            pass
        else:
            raise TypeError("dim_t not of type DimLinear")

max_stars_repo_name: Severson-Group/MachEval

max_issues_repo_name: Severson-Group/MachEval

max_forks_repo_name: Severson-Group/MachEval


text:
import os
import sys
import logging
from code import interact
from datetime import datetime
from textwrap import dedent
import time
import numpy as np
import pandas as pd
import re
import datajoint as dj
from pymysql.err import OperationalError

from pipeline import (lab, experiment, tracking, ephys, psth, ccf, histology, get_schema_name)

pipeline_modules = [lab, ccf, experiment, ephys, histology, tracking, psth]

log = logging.getLogger(__name__)

# ==== UTILITY METHODS ====

def usage_exit():
    print(dedent(
        '''
        usage: {} cmd args

        where 'cmd' is one of:

        {}
        ''').lstrip().rstrip().format(
            os.path.basename(sys.argv[0]),
            str().join("  - {}: {}\n".format(k, v[1])
                       for k, v in actions.items())))

    sys.exit(0)

def shell(*args):
    interact('map shell.\n\nschema modules:\n\n  - {m}\n'
             .format(m='\n  - '.join(
                 '.'.join(m.__name__.split('.')[1:])
                 for m in pipeline_modules)),
             local=globals())

def logsetup(*args):
    level_map = {
        'CRITICAL': logging.CRITICAL,
        'ERROR': logging.ERROR,
        'WARNING': logging.WARNING,
        'INFO': logging.INFO,
        'DEBUG': logging.DEBUG,
        'NOTSET': logging.NOTSET,
    }
    level = level_map[args[0]] if args else logging.INFO

    logfile = dj.config.get('custom', {'logfile': None}).get('logfile', None)

    if logfile:
        handlers = [logging.StreamHandler(), logging.FileHandler(logfile)]
    else:
        handlers = [logging.StreamHandler()]

    datefmt = '%Y-%m-%d %H:%M:%S'
    msgfmt = '%(asctime)s:%(levelname)s:%(module)s:%(funcName)s:%(message)s'

    logging.basicConfig(format=msgfmt, datefmt=datefmt, level=logging.ERROR,
                        handlers=handlers)

    log.setLevel(level)

    logging.getLogger('pipeline').setLevel(level)
    logging.getLogger('pipeline.psth').setLevel(level)
    logging.getLogger('pipeline.ccf').setLevel(level)
    logging.getLogger('pipeline.ingest.session_ingest').setLevel(level)
    logging.getLogger('pipeline.ingest.behavior_ingest').setLevel(level)
    logging.getLogger('pipeline.ingest.tracking_ingest').setLevel(level)

# ==== ROUTINE TO OPERATE THE PIPELINE ====

def ingest_all(subject_id):
    populate_settings = {'reserve_jobs': True, 'suppress_errors': True, 'display_progress': True}

    from .ingest import behavior_ingest, tracking_ingest, ephys_ingest
    from .ingest.session_ingest import load_all_sessions

    print('=========== SESSION INGESTION ===========')
    load_all_sessions(subject_id)
    print('=========== BEHAVIOR INGESTION ===========')
    behavior_ingest.BehaviorIngestion.populate({'subject_id': subject_id}, **populate_settings)
    print('=========== TRACKING INGESTION ===========')
    tracking_ingest.TrackingIngestion.populate({'subject_id': subject_id}, **populate_settings)
    print('=========== EPHYS INGESTION ===========')
    ephys_ingest.EphysIngestion.populate({'subject_id': subject_id}, **populate_settings)

# ==== Action Mapper - for interactive shell ====

actions = {
    'ingest-all': (ingest_all, 'run auto ingest job (load all types)'),
    'shell': (shell, 'interactive shell')
}


text:
import numpy as np
import matplotlib.pyplot as plt

N_ANCHORS = 4
N_POINTS = 256

def generate_gradients(n_anchors):
    """Generate gradients of unit length"""
    thetas = 2 * np.pi * np.random.rand(n_anchors, n_anchors)
    gradients = np.stack((np.cos(thetas), np.sin(thetas)), axis=-1)
    return gradients

def lerp(a, b, u):
    """Smoothed interpolation"""
    uu = 6 * u ** 5 - 15 * u ** 4 + 10 * u ** 3
    return a + uu * (b - a)

def perlin(i, j, gradients):
    # Translate grid coordinates to anchor coordinates
    u, ii = np.modf(i / N_POINTS * (N_ANCHORS - 1))
    v, jj = np.modf(j / N_POINTS * (N_ANCHORS - 1))
    ii, jj = int(ii), int(jj)

    # Compute the dot products
    g1 = gradients[ii, jj] @ (u, v)
    g2 = gradients[ii + 1, jj] @ (u - 1, v)
    g3 = gradients[ii, jj + 1] @ (u, v - 1)
    g4 = gradients[ii + 1, jj + 1] @ (u - 1, v - 1)

    # Perform the interpolation
    v1 = lerp(g1, g2, u)
    v2 = lerp(g3, g4, u)
    value = lerp(v1, v2, v)

    return value

if __name__ == '__main__':
    gradients = generate_gradients(N_ANCHORS)

    grid = np.zeros((N_POINTS, N_POINTS))
    for i in range(N_POINTS):
        for j in range(N_POINTS):
            grid[i, j] = perlin(i, j, gradients)

    plt.imshow(grid, cmap="gray")
    plt.show()

max_stars_repo_name: ncherel/perlin-noise

max_issues_repo_name: ncherel/perlin-noise

max_forks_repo_name: ncherel/perlin-noise


text:
# Copyright 2015-2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for SparseSoftmaxCrossEntropyWithLogits op."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np

from tensorflow.python.eager import backprop as backprop_lib
from tensorflow.python.eager import context
from tensorflow.python.framework import config
from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import errors_impl
from tensorflow.python.framework import ops as ops_lib
from tensorflow.python.framework import test_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import gradient_checker_v2
from tensorflow.python.ops import nn_ops
import tensorflow.python.ops.nn_grad  # pylint: disable=unused-import
from tensorflow.python.platform import test

class SparseXentOpTestBase(test.TestCase):

  def _opFwdBwd(self, labels, logits):
    """Runs the op-under-test both forwards and backwards"""
    logits = ops_lib.convert_to_tensor(logits)  # needed for the gradient tape
    with backprop_lib.GradientTape() as tape:
      tape.watch(logits)
      loss = nn_ops.sparse_softmax_cross_entropy_with_logits_v2(
          labels=labels, logits=logits)
    return loss, tape.gradient(loss, logits)

  def _npXent(self, labels, logits):
    logits = np.reshape(logits, [-1, logits.shape[-1]])
    labels = np.reshape(labels, [-1])
    batch_dim = 0
    class_dim = 1
    batch_size = logits.shape[batch_dim]
    e = np.exp(logits -
               np.reshape(np.amax(logits, axis=class_dim), [batch_size, 1]))
    probs = e / np.reshape(np.sum(e, axis=class_dim), [batch_size, 1])
    labels_mat = np.zeros_like(probs).astype(probs.dtype)
    labels_mat[np.arange(batch_size), labels] = 1.0
    gradient = (probs - labels_mat)
    loss = -np.sum(labels_mat * np.log(probs + 1.0e-20), axis=1)
    return loss, gradient

  def _testXent(self, np_labels, np_logits):
    np_loss, np_gradient = self._npXent(labels=np_labels, logits=np_logits)
    tf_loss, tf_gradient = self._opFwdBwd(labels=np_labels, logits=np_logits)
    self.assertAllCloseAccordingToType(np_loss, tf_loss)
    self.assertAllCloseAccordingToType(np_gradient, tf_gradient)

  def testSingleClass(self):
    for label_dtype in np.int32, np.int64:
      tf_loss, tf_gradient = self._opFwdBwd(
          labels=np.array([0, 0, 0]).astype(label_dtype),
          logits=np.array([[1.], [-1.], [0.]]).astype(np.float32))
      self.assertAllClose([0.0, 0.0, 0.0], tf_loss)
      self.assertAllClose([[0.0], [0.0], [0.0]], tf_gradient)

  @test_util.run_gpu_only()
  def _testInvalidLabelGPU(self, invalid_label_gradient=np.nan):
    labels = [4, 3, 0, -1]
    logits = [[1., 1., 1., 1.], [1., 1., 1., 1.], [1., 2., 3., 4.],
              [1., 2., 3., 4.]]
    loss, gradient = self._opFwdBwd(labels=labels, logits=logits)
    self.assertAllClose([np.nan, 1.3862, 3.4420, np.nan],
                        loss,
                        rtol=1e-3,
                        atol=1e-3)
    self.assertAllClose(
        [[invalid_label_gradient] * 4, [0.25, 0.25, 0.25, -0.75],
         [-0.968, 0.087, 0.237, 0.6439], [invalid_label_gradient] * 4],
        gradient,
        rtol=1e-3,
        atol=1e-3)

  def testInvalidLabelGPU(self):
    """This method is structured to be easily overridden by a child class."""
    self._testInvalidLabelGPU()

  @test_util.run_in_graph_and_eager_modes(use_gpu=False)
  @test_util.disable_xla("XLA cannot assert inside of a kernel.")
  def _testInvalidLabelCPU(self, expected_regex="Received a label value of"):
    labels = [4, 3, 0, -1]
    logits = [[1., 1., 1., 1.], [1., 1., 1., 1.], [1., 2., 3., 4.],
              [1., 2., 3., 4.]]
    with self.assertRaisesRegex(
        (errors_impl.InvalidArgumentError, errors_impl.UnknownError),
        expected_regex):
      self.evaluate(
          nn_ops.sparse_softmax_cross_entropy_with_logits_v2(
              labels=labels, logits=logits))

  def testInvalidLabelCPU(self):
    """This method is structured to be easily overridden by a child class."""
    self._testInvalidLabelCPU()

  def testNpXent(self):
    # We create 2 batches of logits for testing.
    # batch 0 is the boring uniform distribution: 1, 1, 1, 1, with target 3.
    # batch 1 has a bit of difference: 1, 2, 3, 4, with target 0.
    labels = [3, 0]
    logits = [[1., 1., 1., 1.], [1., 2., 3., 4.]]

    # For batch 0, we expect the uniform distribution: 0.25, 0.25, 0.25, 0.25
    # With a hard target 3, the gradient is [0.25, 0.25, 0.25, -0.75]
    # The loss for this batch is -log(0.25) = 1.386
    #
    # For batch 1, we have:
    # exp(0) = 1
    # exp(1) = 2.718
    # exp(2) = 7.389
    # exp(3) = 20.085
    # SUM = 31.192
    # So we have as probabilities:
    # exp(0) / SUM = 0.032
    # exp(1) / SUM = 0.087
    # exp(2) / SUM = 0.237
    # exp(3) / SUM = 0.644
    # With a hard 1, the gradient is [0.032 - 1.0 = -0.968, 0.087, 0.237, 0.644]
    # The loss for this batch is [1.0 * -log(0.25), 1.0 * -log(0.032)]
    # = [1.3862, 3.4420]
    np_loss, np_gradient = self._npXent(
        labels=np.array(labels), logits=np.array(logits))
    self.assertAllClose(
        np.array([[0.25, 0.25, 0.25, -0.75], [-0.968, 0.087, 0.237, 0.6439]]),
        np_gradient,
        rtol=1.e-3,
        atol=1.e-3)
    self.assertAllClose(
        np.array([1.3862, 3.4420]), np_loss, rtol=1.e-3, atol=1.e-3)

  def testShapeMismatch(self):
    with self.assertRaisesRegex(ValueError, ".*Rank mismatch:*"):
      nn_ops.sparse_softmax_cross_entropy_with_logits_v2(
          labels=[[0, 2]], logits=[[0., 1.], [2., 3.], [2., 3.]])

  def testScalar(self):
    with self.assertRaisesRegex(ValueError, ".*Logits cannot be scalars*"):
      nn_ops.sparse_softmax_cross_entropy_with_logits_v2(
          labels=constant_op.constant(0), logits=constant_op.constant(1.0))

  def _testLabelsPlaceholderScalar(self, expected_error_message):
    with ops_lib.Graph().as_default(), self.session():
      labels = array_ops.placeholder(np.int32)
      y = nn_ops.sparse_softmax_cross_entropy_with_logits_v2(
          labels=labels, logits=[[7.]])
      with self.assertRaisesOpError(expected_error_message):
        y.eval(feed_dict={labels: 0})

  def testLabelsPlaceholderScalar(self):
    """This method is structured to be easily overridden by a child class."""
    self._testLabelsPlaceholderScalar(
        expected_error_message="labels must be 1-D")

  def testVector(self):
    loss = nn_ops.sparse_softmax_cross_entropy_with_logits_v2(
        labels=constant_op.constant(0), logits=constant_op.constant([1.0]))
    self.assertAllClose(0.0, loss)

  def testFloat(self):
    for label_dtype in np.int32, np.int64:
      self._testXent(
          np_labels=np.array([3, 0]).astype(label_dtype),
          np_logits=np.array([[1., 1., 1., 1.], [1., 2., 3.,
                                                 4.]]).astype(np.float32))

  def testDouble(self):
    for label_dtype in np.int32, np.int64:
      self._testXent(
          np_labels=np.array([0, 3]).astype(label_dtype),
          np_logits=np.array([[1., 1., 1., 1.], [1., 2., 3.,
                                                 4.]]).astype(np.float64))

  def testHalf(self):
    for label_dtype in np.int32, np.int64:
      self._testXent(
          np_labels=np.array([3, 0]).astype(label_dtype),
          np_logits=np.array([[1., 1., 1., 1.], [1., 2., 3.,
                                                 4.]]).astype(np.float16))

  def testEmpty(self):
    self._testXent(
        np_labels=np.zeros((0,), dtype=np.int32), np_logits=np.zeros((0, 3)))

  @test_util.run_in_graph_and_eager_modes()
  def testGradient(self):
    with self.session() as sess:
      labels = constant_op.constant([3, 0, 1], name="labels")
      logits = constant_op.constant(
          [0.1, 0.2, 0.3, 0.4, 0.1, 0.4, 0.9, 1.6, 0.1, 0.8, 2.7, 6.4],
          shape=[3, 4],
          dtype=dtypes.float64,
          name="logits")

      def xent(logits):
        # gradient_checker_v2.computee_gradient doesn't take int32/int64.
        # labels must be of type int32/int64, so passing them separately here.
        return nn_ops.sparse_softmax_cross_entropy_with_logits_v2(
            labels=labels, logits=logits, name="xent")

      analytical, numerical = gradient_checker_v2.compute_gradient(
          xent, [logits])

      if not context.executing_eagerly():
        # Check that no extra computation performed. When only first derivative
        # is requested, second derivative must not be computed. So when there is
        # no second derivative, there is no `BatchMatMul` op in the graph.
        op_names = [
            op.op_def.name for op in sess.graph.get_operations() if op.op_def
        ]
        self.assertNotIn("BatchMatMul", op_names)
        self.assertNotIn("BatchMatMulV2", op_names)

    tol = 5e-8
    self.assertAllClose(analytical, numerical, atol=tol, rtol=tol)

  @test_util.run_in_graph_and_eager_modes()
  def testSecondGradient(self):
    with self.session() as sess:
      labels = constant_op.constant([3, 0, 1], name="labels")
      logits = constant_op.constant(
          [0.3, 0.4, 0.1, 1.2, 0.1, 1.9, 0.1, 0.7, 0.8, 0.2, 1.3, 1.3],
          shape=[3, 4],
          dtype=dtypes.float64,
          name="logits")

      def xent_grad(logits):
        with backprop_lib.GradientTape() as tape:
          tape.watch(logits)
          return tape.gradient(
              nn_ops.sparse_softmax_cross_entropy_with_logits_v2(
                  labels=labels, logits=logits, name="xent"), [logits])[0]

      analytical, numerical = gradient_checker_v2.compute_gradient(
          xent_grad, [logits])

      if (not context.executing_eagerly() and
          not config.deterministic_ops_enabled()):
        # Check that second derivative is calculated.
        # (it is equivalent to being `BatchMatMul` op in the graph because of
        # implementation of xentropy grad)
        op_names = [
            op.op_def.name for op in sess.graph.get_operations() if op.op_def
        ]
        self.assertIn("BatchMatMulV2", op_names)

    tol = 5e-8
    self.assertAllClose(analytical, numerical, atol=tol, rtol=tol)

  @test_util.run_in_graph_and_eager_modes()
  def _testHighDim(self, labels, logits):
    np_loss, np_gradient = self._npXent(
        labels=np.array(labels), logits=np.array(logits))
    # manually reshape loss
    np_loss = np.reshape(np_loss, np.array(labels).shape)
    tf_loss = nn_ops.sparse_softmax_cross_entropy_with_logits_v2(
        labels=labels, logits=logits)
    with backprop_lib.GradientTape() as tape:
      logits = constant_op.constant(logits)
      tape.watch(logits)
      tf_gradient = tape.gradient(
          nn_ops.sparse_softmax_cross_entropy_with_logits_v2(
              labels=labels, logits=logits), [logits])[0]
      tf_gradient = array_ops.reshape(tf_gradient, np_gradient.shape)

    self.assertAllCloseAccordingToType(np_loss, tf_loss)
    self.assertAllCloseAccordingToType(np_gradient, tf_gradient)

  def testHighDim(self):
    labels = [[3], [0]]
    logits = [[[1., 1., 1., 1.]], [[1., 2., 3., 4.]]]
    self._testHighDim(labels, logits)

  def testHighDim2(self):
    labels = [[3, 2], [0, 3]]
    logits = [[[1., 1., 1., 1.], [2., 2., 2., 2.]],
              [[1., 2., 3., 4.], [5., 6., 7., 8.]]]
    self._testHighDim(labels, logits)

  def _testScalarHandling(self, expected_regex):
    with ops_lib.Graph().as_default(), self.session(use_gpu=False) as sess:
      with self.assertRaisesRegex(errors_impl.InvalidArgumentError,
                                  expected_regex):
        labels = array_ops.placeholder(dtypes.int32, shape=[None, 1])
        logits = array_ops.placeholder(dtypes.float32, shape=[None, 3])
        ce = nn_ops.sparse_softmax_cross_entropy_with_logits_v2(
            labels=array_ops.squeeze(labels), logits=logits)
        labels_v2 = np.zeros((1, 1), dtype=np.int32)
        logits_v2 = np.random.randn(1, 3)
        sess.run([ce], feed_dict={labels: labels_v2, logits: logits_v2})

  def testScalarHandling(self):
    """This method is structured to be easily overridden by a child class."""
    self._testScalarHandling(expected_regex=".*labels must be 1-D.*")

max_stars_repo_name: rsuderman/tensorflow

max_issues_repo_name: rsuderman/tensorflow

max_forks_repo_name: rsuderman/tensorflow


text:
"""
Main algorithm
"""

from __future__ import print_function

import sys

from fitness_functions import FitnessFunctions
from random import random, choice
import matplotlib.pyplot as plt
from statistics import mean
import math
import os
import pickle
import pandas as pd
import numpy as np

import neat
import visualize

#################### PARAMETERS   ####################
type_optimization="accuracy"
t=["accuracy", "timeO", "rotationO", "energyO"]
type_optimization=t[0]

USE_FIXED_TRAJECTORY=True
N_POINTS_TRAJECTORY_TRAINING=30 #Number of point for evaluate the quality of each network during training
num_generation=600
pop_size=200 #Remember that is defined in config file

type="rnn"  #rnn or feed

#################### PARAMETERS   ####################
fitnessFunctions = FitnessFunctions()
trajectory_paper = [[2.25, 1.1, 0.25],#
                    [0.9, 1.5, 0.25],
                    [-0.85, 1.14, 2.22],
                    [-1.8, 1.25, 1.17],
                    [1.8, 1.25, 1.17],
                    [-1.25, -1.1, 0.25],
                    [-2.25, -1.48, 0.25],
                    [0.45, -1.14, 2.22],
                    [0.8, -1.25, 2.35],
                    [0.8, -1.25, -1.35]]
fitnesses={"total":[],"rotation":[],"energy":[],"time":[],"accuracy":[]}

points=[]

if(USE_FIXED_TRAJECTORY==False):
    with open('points.csv', 'rb') as f:
        points = pickle.load(f)

def getMultiplierForNormalization(_min, _max):
    return (_max - _min)/1 + (_min)

# Use the RN network phenotype
def eval_genome(genome, config, verbose=False, fixed=-1, points_to_use=[]):

    if(type=="rnn"):
        net = neat.nn.RecurrentNetwork.create(genome, config)
    else:
        net = neat.nn.FeedForwardNetwork.create(genome, config)
    data={}

    fixed_data=USE_FIXED_TRAJECTORY
    if(fixed>-1):
        fixed_data=fixed==0 #fixed==0 used fix trajectory, any other number use the not fixed

    trajectory_points=[]

    if(fixed_data):
        trajectory_points = trajectory_paper
    else:
        trajectory_points = points_to_use

        #trajectory_points=trajectory_points+trajectory_paper

    outputs = []
    rescaling_factors=np.array([
        getMultiplierForNormalization(math.radians(-180), math.radians(180)),
        getMultiplierForNormalization(math.radians(-70), math.radians(70)),
        getMultiplierForNormalization(math.radians(-28), math.radians(105)),
        getMultiplierForNormalization(math.radians(-300), math.radians(300)),
        getMultiplierForNormalization(math.radians(-120), math.radians(120)),
        getMultiplierForNormalization(math.radians(-300), math.radians(300))
    ])

    trajectory_len=len(trajectory_points)

    for i in range(0,trajectory_len):
        if(type=="rnn"):
            output=net.activate(trajectory_points[i])
        else:
            #output=net.activate(trajectory_points[i]+trajectory_points[(i-1+trajectory_len)%trajectory_len])
            output=net.activate(trajectory_points[i])

        # RESCALING
        output=(np.array(output)*rescaling_factors).tolist()
        outputs.append(output)

    if verbose:
        np_outputs=np.array(outputs)
        for i in range(0,6):
            data["outputs_"+str(i)]=np_outputs[:,i].tolist()
        total_rotation, data["rotation"] = fitnessFunctions.evaluate_rotations(outputs, verbose)
        total_energy, data["energy"] = fitnessFunctions.evaluate_energy(outputs, verbose)
        total_operation_time, data["operation"] = fitnessFunctions.evaluate_operation_time(outputs, verbose)
        total_accuracy, data["accuracy"] = fitnessFunctions.evaluate_position_accuracy(outputs, trajectory_points, verbose)
        data["total_rotation"]=total_rotation
        data["total_energy"]=total_energy
        data["total_operation_time"]=total_operation_time
        data["total_accuracy"]=total_accuracy
    else:
        total_rotation = fitnessFunctions.evaluate_rotations(outputs)
        total_energy = fitnessFunctions.evaluate_energy(outputs)
        total_operation_time = fitnessFunctions.evaluate_operation_time(outputs)
        total_accuracy = fitnessFunctions.evaluate_position_accuracy(outputs, trajectory_points)

    if(type_optimization=="accuracy"):
        fitness = -(total_accuracy)#ACCURACY OPTIMAL
    elif(type_optimization=="timeO"):
        fitness = -(total_accuracy+20/150*total_operation_time)#TIME OPTIMAL
    elif(type_optimization=="rotationO"):
        fitness = -(total_accuracy+20/4000*total_rotation)#MINIMUM ROTATION
    elif(type_optimization=="energyO"):
        fitness = -(total_accuracy+20/20000*total_energy)#ENERGY OPTIMAL
    else:
        fitness = -(total_accuracy+5*total_energy+10*total_operation_time+5*total_rotation)#COMBINED CONTROL

    fitnesses["total"].append(-fitness)
    fitnesses["rotation"].append(total_rotation)
    fitnesses["energy"].append(total_energy)
    fitnesses["time"].append(total_operation_time)
    fitnesses["accuracy"].append(total_accuracy)
    if verbose:
        data["fitness"]=fitness
        df = pd.DataFrame(dict([ (k,pd.Series(v)) for k,v in data.items() ]))
        df.to_csv('./result_algorithm/tableResults_'+type_optimization+'.csv', index=False)
    return fitness

def eval_genomes(genomes, config):
    points_to_use=[]
    if(USE_FIXED_TRAJECTORY==False):
        for i in range(0, N_POINTS_TRAJECTORY_TRAINING):
            points_to_use.append(choice(points)[1])
    for genome_id, genome in genomes:
        genome.fitness = eval_genome(genome, config, points_to_use=points_to_use)

def run():

    # Load the config file
    local_dir = os.path.dirname(__file__)

    if(type=="rnn"):
        config_path = os.path.join(local_dir, 'config-ctrnn')
    else:
        config_path = os.path.join(local_dir, 'config-feed')

    config = neat.Config(neat.DefaultGenome, neat.DefaultReproduction,
                         neat.DefaultSpeciesSet, neat.DefaultStagnation,
                         config_path)

    pop = neat.Population(config)
    stats = neat.StatisticsReporter()
    pop.add_reporter(stats)
    pop.add_reporter(neat.StdOutReporter(True))

    if 1:
        winner = pop.run(eval_genomes, num_generation)
    else:
        pe = neat.ParallelEvaluator(4, eval_genome)
        winner = pop.run(pe.evaluate, num_generation)

    # Save the winner.
    with open('results/winner-rnn', 'wb') as f:
        pickle.dump(winner, f)

    eval_genome(winner, config, True, fixed=0)
    print(stats);

    #plt.figure("NEAT (Population's average/std. dev and best fitness)")
    #plt.rcParams["figure.figsize"] = [6,4]
    plt.rcParams.update({'font.size': 15})
    plt.figure("NEAT fitnesses")# (Population's average/std. dev and best fitness)")
    avg_fitnesses={"total":[],"rotation":[],"energy":[],"time":[],"accuracy":[]}
    for i in range(0, num_generation):
        avg_fitnesses["total"].append(np.mean(fitnesses["total"][i*pop_size:i*pop_size + pop_size-1]))
        avg_fitnesses["time"].append(np.mean(fitnesses["time"][i*pop_size:i*pop_size + pop_size-1]))
        avg_fitnesses["rotation"].append(np.mean(fitnesses["rotation"][i*pop_size:i*pop_size + pop_size-1]))
        avg_fitnesses["energy"].append(np.mean(fitnesses["energy"][i*pop_size:i*pop_size + pop_size-1]))
        avg_fitnesses["accuracy"].append(np.mean(fitnesses["accuracy"][i*pop_size:i*pop_size + pop_size-1]))
    #print(avg_fitnesses["accuracy"])
    #print(avg_fitnesses)
    #plt.plot([i for i in range(0, len(avg_fitnesses["total"]))], avg_fitnesses["total"], 'b-', label="fitness")
    plt.plot([i for i in range(0, len(avg_fitnesses["energy"]))], avg_fitnesses["energy"], color="#000000", label="energy", linestyle='-')
    #plt.plot(generation, avg_fitness - stdev_fitness, 'g-.', label="-1 sd")
    #plt.plot(generation, avg_fitness + stdev_fitness, 'g-.', label="+1 sd")
    #plt.plot(generation, best_fitness, 'r-', label="best")

    #plt.title("Population's average/std. dev and best fitness")
    plt.xlabel("Generations")
    plt.ylabel("Fitness", labelpad=0)
    plt.grid()
    plt.legend(loc="best")

    plt.savefig("energy_"+type_optimization+".png")
    view=True
    if view:
        plt.show()
        #plt.close()
        fig = None

    plt.plot([i for i in range(0, len(avg_fitnesses["accuracy"]))], avg_fitnesses["accuracy"], color="#808080", linestyle="-", label="accuracy",lw=0.9)
    plt.plot([i for i in range(0, len(avg_fitnesses["time"]))], avg_fitnesses["time"], color="#D3D3D3", linestyle="-", label="time",lw=0.9)
    plt.plot([i for i in range(0, len(avg_fitnesses["rotation"]))], avg_fitnesses["rotation"], color="#000000",linestyle="-", label="rotation",lw=0.9)

    plt.xlabel("Generations")
    plt.ylabel("Fitness")
    plt.grid()
    plt.legend(loc="best")

    plt.savefig("accuracy_time_rotation_"+type_optimization+".png")
    view=True
    if view:
        plt.show()
        fig = None

    if(1):
        visualize.plot_stats(stats, ylog=True, view=False, filename="results/rnn-fitness.svg")
        visualize.plot_species(stats, view=False, filename="results/rnn-speciation.svg")

        node_names = {0: 'omega', 1: 'theta', 2: 'psi', 3: 'fi', 4: 'ro', 5: 'epsilon', -3: 'x', -2: 'y', -1: 'z'}

        visualize.draw_net(config, winner, view=False, node_names=node_names,
                           filename="results/winner-rnn.gv")
        visualize.draw_net(config, winner, view=False, node_names=node_names,
                           filename="results/winner-rnn-enabled.gv", show_disabled=False)
        visualize.draw_net(config, winner, view=False, node_names=node_names,
                           filename="results/winner-rnn-enabled-pruned.gv", show_disabled=False, prune_unused=True)

if __name__ == '__main__':
    print(sys.argv)
    if(len(sys.argv)>1):
        type=sys.argv[1]
    print(type)
    run()


text:
import gym
from gym import spaces
from enum import IntEnum
import numpy as np
import math
import queue
import plotly.graph_objects as graph_object
from plotly.subplots import make_subplots

class Actions(IntEnum):
    DECREASE_NOZZLE_PRESSURE = 0
    INCREASE_NOZZLE_PRESSURE = 1
    DO_NOTHING = 2

class Constants:
    NOZZLE_PRESSURE_MAX = 700
    NOZZLE_PRESSURE_MIN = 0
    NOZZLE_PRESSURE_INIT = 350
    NOZZLE_PRESSURE_STEP_SIZE = 6
    NOZZLE_PRESSURE_TO_ZINC_SCRUB = 0.0009

    ZINC_BATH_COATING_MAX = 100
    ZINC_BATH_COATING_MIN = 75
    ZINC_BATH_COATING_INIT = 85
    ZINC_BATH_JITTER_STEP_SIZE = 1

    ZINC_COATING_DISCRETE_INTERVAL = 20
    ZINC_COATING_DISCRETE_STEP_SIZE = 1
    ZINC_COATING_DISCRETE_STEPS = 2 + \
        math.floor(ZINC_COATING_DISCRETE_INTERVAL /
                   ZINC_COATING_DISCRETE_STEP_SIZE)

    ZINC_COATING_TARGET = 50

class Plotter:
    def __init__(self):
        self.nozzle = []
        self.zinc_coating = []
        self.target_coating = []
        self.zinc_bath_coating = []
        self.reward = []

    def update(self, nozzle, thickness, target_thickness, bath, reward):
        self.nozzle.append(nozzle)
        self.zinc_coating.append(thickness)
        self.target_coating.append(target_thickness)
        self.zinc_bath_coating.append(bath)
        if (len(self.reward) > 0):
            self.reward.append(self.reward[-1] + reward)
        else:
            self.reward.append(reward)

    def show(self):
        plot = make_subplots(rows=3)
        plot.add_trace(graph_object.Scatter(
            y=self.zinc_bath_coating, mode="lines", name="zinc_bath_coating"), row=1, col=1)
        plot.add_trace(graph_object.Scatter(
            y=self.zinc_coating, mode="lines", name="zinc_coating"), row=1, col=1)
        plot.add_trace(graph_object.Scatter(
            y=self.target_coating, mode="lines", name="target_coating"), row=1, col=1)
        plot.add_trace(graph_object.Scatter(
            y=self.nozzle, mode="lines", name="nozzle_pressure"), row=2, col=1)
        plot.add_trace(graph_object.Scatter(
            y=self.reward, mode="lines", name="cum. reward"), row=3, col=1)
        plot.show()

class ZincCoatingV0(gym.Env):
    """Simple zinc coating environment"""
    metadata = {'render.modes': ['human']}

    def __init__(self, reward_offset=100):
        super(ZincCoatingV0, self).__init__()

        self.action_space = spaces.Discrete(len(Actions))
        self.observation_space = spaces.Discrete(
            Constants.ZINC_COATING_DISCRETE_STEPS)

        self.reward_offset = reward_offset
        self.reward_queue = queue.SimpleQueue()
        for i in range(self.reward_offset):
            self.reward_queue.put(0)

        self.plotter = Plotter()

    def step(self, action):
        # Simulating episodes if Monte Carlo Methods are used
        if (self.current_step >= 1000):
            self.done = True
        else:
            self.done = False

        self.reward = self._get_reward()
        self.reward_queue.put(self.reward)
        self._take_action(action)
        return self._next_observation(), self.reward_queue.get(), self.done, {}

    def reset(self):
        self.nozzle_pressure = Constants.NOZZLE_PRESSURE_INIT + \
            (np.random.rand() - 0.5) * 50
        self.zinc_bath_coating = Constants.ZINC_BATH_COATING_INIT + \
            (np.random.rand() - 0.5) * 10
        self.target_zinc_coating = Constants.ZINC_COATING_TARGET

        self.zinc_coating = self._get_zinc_coating()
        self.current_step = 0
        self.reward = 0

        self.plotter = Plotter()

        return self._next_observation()

    def render(self, mode='human', close=False):
        self.plotter.update(self.nozzle_pressure,
                            self.zinc_coating, self.target_zinc_coating, self.zinc_bath_coating, self.reward)
        if (self.done):
            self.plotter.show()

    def _get_reward(self):
        if self.zinc_coating < self.target_zinc_coating:
            reward = -100
        elif self.zinc_coating >= self.target_zinc_coating + Constants.ZINC_COATING_DISCRETE_INTERVAL:
            reward = -1
        else:
            reward = 1/(self.zinc_coating - self.target_zinc_coating + 1)

        return reward

    def _take_action(self, action):
        if(action == int(Actions.INCREASE_NOZZLE_PRESSURE)):
            self.nozzle_pressure = np.min([
                self.nozzle_pressure + Constants.NOZZLE_PRESSURE_STEP_SIZE, Constants.NOZZLE_PRESSURE_MAX])

        elif(action == int(Actions.DECREASE_NOZZLE_PRESSURE)):
            self.nozzle_pressure = np.max([
                self.nozzle_pressure - Constants.NOZZLE_PRESSURE_STEP_SIZE, Constants.NOZZLE_PRESSURE_MIN])

    def _next_observation(self):
        if(self.current_step % Constants.ZINC_BATH_JITTER_STEP_SIZE == 0):
            self.zinc_bath_coating = np.min([np.max([self.zinc_bath_coating +
                                                     (np.random.rand() - 0.5) * 5, Constants.ZINC_BATH_COATING_MIN]), Constants.ZINC_BATH_COATING_MAX])
        self.zinc_coating = self._get_zinc_coating()
        self.current_step += 1

        return self._get_state()

    def _get_zinc_coating(self):
        zinc_scrub = self.nozzle_pressure * Constants.NOZZLE_PRESSURE_TO_ZINC_SCRUB
        zinc_thickness = (1 - zinc_scrub) * self.zinc_bath_coating
        return zinc_thickness

    def _get_state(self):
        if(self.zinc_coating < self.target_zinc_coating):
            return 0
        elif self.zinc_coating >= self.target_zinc_coating + Constants.ZINC_COATING_DISCRETE_INTERVAL:
            return Constants.ZINC_COATING_DISCRETE_STEPS - 1
        else:
            return math.floor(
                (self.zinc_coating - self.target_zinc_coating) / Constants.ZINC_COATING_DISCRETE_STEP_SIZE) + 1


text:
#!/usr/bin/env python
# coding: utf-8

# [1]

from __future__ import  division
get_ipython().run_line_magic('qtconsole', '')

# [2]

import sympy as S
import sympy.stats as st

# [20]

u=S.symbols('u')
p=st.LogNormal('u',0,0.5)
pu=st.density(p)(u)
fx=S.lambdify(u,pu,'numpy')
x = np.linspace(0.001,4,100)
hist([st.sample(p) for i in range(1000)],bins=50,normed=1);
plot(x,fx(x),'r-',lw=3.)

# [162]

pw=[i*(S.Heaviside(u-j/2)-S.Heaviside(u-j/2-1/2)) for i,j in zip(S.symbols('a:8'),range(8))]
pwf = lambda i: sum(pw).subs(u,i)

# ## Rectangle Wedge Tail Decomposition

#
# $$ f(x) = \exp\left(-\frac{(x-1)^2}{2x} \right)(x+1)/12 $$
#
# where $x>0$

# [13]

x = np.linspace(0.001,10,100)
f= lambda x: (np.sqrt(1/x) + np.sqrt(x))/ (2.*pow(exp(1),pow(-np.sqrt(1/x) + np.sqrt(x),2)/ 2.)*np.sqrt(2*pi)*x)
fx = f(x)
plot(x,fx)

# [14]

u=S.symbols('u')
p=S.Piecewise(*[(f(j),(i<u<=j)) for i,j in zip(range(10),range(1,11))]+[(0,True)])
plot(x,[p.subs(u,i) for i in x],x,fx)

# [14]

# [ ]

max_stars_repo_name: tnakaicode/Python-for-Signal-Processing

max_issues_repo_name: tnakaicode/Python-for-Signal-Processing

max_forks_repo_name: tnakaicode/Python-for-Signal-Processing

reason: import sympy


text:
from siphon.radarserver import RadarServer
from datetime import datetime, timedelta
import matplotlib.pyplot as plt
import cartopy.crs as ccrs
import cartopy.feature as cfeature
import numpy as np
import metpy.plots as mpplots
import matplotlib
import io
import base64
matplotlib.use('Agg')
import warnings
warnings.filterwarnings("ignore")

def new_map(fig, lon, lat):
    # Create projection centered on the radar. This allows us to use x
    # and y relative to the radar.
    proj = ccrs.LambertConformal(central_longitude=lon, central_latitude=lat)

    # New axes with the specified projection
    ax = fig.add_axes([0.02, 0.02, 0.96, 0.96], projection=proj)

    # Add coastlines and states
    ax.add_feature(cfeature.COASTLINE.with_scale('50m'), linewidth=2)
    ax.add_feature(cfeature.STATES.with_scale('50m'))

    return ax

def raw_to_masked_float(var, data):
    # Values come back signed. If the _Unsigned attribute is set, we need to convert
    # from the range [-127, 128] to [0, 255].
    if var._Unsigned:
        data = data & 255

    # Mask missing points
    data = np.ma.array(data, mask=data==0)

    # Convert to float using the scale and offset
    return data * var.scale_factor + var.add_offset

def polar_to_cartesian(az, rng):
    az_rad = np.deg2rad(az)[:, None]
    x = rng * np.sin(az_rad)
    y = rng * np.cos(az_rad)
    return x, y

def get_image(station, year, month, date, time):
    try:
        dt = datetime(year, month, date, time)

        rs = RadarServer('http://tds-nexrad.scigw.unidata.ucar.edu/thredds/radarServer/nexrad/level2/S3/')
        query = rs.query()

        # Our specified time range
        query.stations(station).time(dt)

        if not rs.validate_query(query):
            return "invalid request"

        cat = rs.get_catalog(query)
        if len(cat.datasets) == 0:
            return None

        ref_norm, ref_cmap = mpplots.ctables.registry.get_with_steps('NWSReflectivity', 5, 5)

        ds = cat.datasets[0]
        data = ds.remote_access()
        fig = plt.figure(figsize=(10, 7.5))
        sLon, sLat = data.StationLongitude, data.StationLatitude
        ax = new_map(fig, sLon, sLat)

        # Set limits in lat/lon space
        ax.set_extent([sLon + 5, sLon - 5, sLat - 5, sLat + 5])

        ax.add_feature(cfeature.OCEAN.with_scale('50m'))
        ax.add_feature(cfeature.LAND.with_scale('50m'))

        for ds_name in cat.datasets:
            # After looping over the list of sorted datasets, pull the actual Dataset object out
            # of our list of items and access over CDMRemote
            data = cat.datasets[ds_name].remote_access()

            # Pull out the data of interest
            sweep = 0
            rng = data.variables['distanceR_HI'][:]
            az = data.variables['azimuthR_HI'][sweep]
            ref_var = data.variables['Reflectivity_HI']

            # Convert data to float and coordinates to Cartesian
            ref = raw_to_masked_float(ref_var, ref_var[sweep])
            x, y = polar_to_cartesian(az, rng)

            # Plot the data and the timestamp
            mesh = ax.pcolormesh(x, y, ref, cmap=ref_cmap, norm=ref_norm, zorder=0)
            text = ax.text(0.7, 0.02, data.time_coverage_start, transform=ax.transAxes,
                        fontdict={'size':16})

            #convert plot to base64
            s = io.BytesIO()
            plt.savefig(s, format='png', bbox_inches="tight")
            base64_str = base64.b64encode(s.getvalue()).decode("utf-8").replace("\n", "")
            return base64_str

            # Collect the things we've plotted so we can animate
            # meshes.append((mesh, text))

    except BaseException as error:
        print("*"*50)
        print(error)
        err_str = 'Error Occured while interating with AWS: ' +  str(error)
        raise Exception(err_str)

max_stars_repo_path: Data Ingestor/Nexrad_image.py

max_stars_repo_name: airavata-courses/DSDummies

max_issues_repo_path: Data Ingestor/Nexrad_image.py

max_issues_repo_name: airavata-courses/DSDummies

max_forks_repo_path: Data Ingestor/Nexrad_image.py

max_forks_repo_name: airavata-courses/DSDummies


text:
# -*- coding: utf-8 -*-
#
# Test Syncopy's plotting functionality
#

# Builtin/3rd party package imports
import pytest
import numpy as np
from copy import copy

# Local imports
from syncopy.tests.misc import generate_artificial_data, figs_equal
from syncopy.shared.errors import SPYValueError
from syncopy.plotting import singlepanelplot, multipanelplot
from syncopy import __plt__
if __plt__:
    import matplotlib.pyplot as plt
    import matplotlib as mpl

# If matplotlib is not available, this whole testing module is pointless
skip_without_plt = pytest.mark.skipif(not __plt__, reason="matplotlib not available")

@skip_without_plt
class TestAnalogDataPlotting():

    nChannels = 16
    nTrials = 8
    seed = 130810

    # To use `selectdata` w/``trials = None``-raw plotting, the trials must not
    # overlap - construct separate set of `raw*` AnalogData-objects for testing!
    dataReg = generate_artificial_data(nTrials=nTrials,
                                       nChannels=nChannels,
                                       seed=seed,
                                       equidistant=True,
                                       overlapping=True)
    dataInv = generate_artificial_data(nTrials=nTrials,
                                       nChannels=nChannels,
                                       seed=seed,
                                       equidistant=True,
                                       overlapping=True,
                                       dimord=dataReg.dimord[::-1])
    rawReg = generate_artificial_data(nTrials=nTrials,
                                      nChannels=nChannels,
                                      seed=seed,
                                      equidistant=True,
                                      overlapping=False)
    rawInv = generate_artificial_data(nTrials=nTrials,
                                      nChannels=nChannels,
                                      seed=seed,
                                      equidistant=True,
                                      overlapping=False,
                                      dimord=rawReg.dimord[::-1])

    trials = ["all", [4, 3, 2, 2, 7]]
    channels = ["all", [14, 13, 12, 12, 15]]
    toilim = [None, [1.9, 2.5], [2.1, np.inf]]

    # trlToilim = lambda self, trl, tlim: None if trl is None else tlim

    def test_singlepanelplot(self):

        # Lowest possible dpi setting permitting valid png comparisons in `figs_equal`
        mpl.rcParams["figure.dpi"] = 150

        # Test everything except "raw" plotting
        for trials in self.trials:
            for channels in self.channels:
                for toilim in self.toilim:
                    for avg_channels in [True, False]:

                        # Render figure using singlepanelplot mechanics, recreate w/`selectdata`,
                        # results must be identical
                        fig1 = self.dataReg.singlepanelplot(trials=trials,
                                                       channels=channels,
                                                       toilim=toilim,
                                                       avg_channels=avg_channels)
                        selected = self.dataReg.selectdata(trials=trials,
                                                           channels=channels,
                                                           toilim=toilim)
                        fig2 = selected.singlepanelplot(trials="all",
                                                   avg_channels=avg_channels)

                        # Recreate `fig1` and `fig2` in a single sweep by using
                        # `spy.singlepanelplot` w/multiple input objects
                        fig1a, fig2a = singlepanelplot(self.dataReg, self.dataInv,
                                                  trials=trials,
                                                  channels=channels,
                                                  toilim=toilim,
                                                  avg_channels=avg_channels,
                                                  overlay=False)

                        # `fig2a` is based on `dataInv` - be more lenient there
                        tol = None
                        if avg_channels:
                            tol = 1e-2
                        assert figs_equal(fig1, fig2)
                        assert figs_equal(fig1, fig1a)
                        assert figs_equal(fig2, fig2a, tol=tol)
                        assert figs_equal(fig1, fig2a, tol=tol)

                        # Create overlay figures: `fig3` combines `dataReg` and
                        # `dataInv` - must be identical to overlaying `fig1` w/`dataInv`
                        fig3 = singlepanelplot(self.dataReg, self.dataInv,
                                          trials=trials,
                                          channels=channels,
                                          toilim=toilim,
                                          avg_channels=avg_channels,
                                          overlay=True)
                        fig1 = singlepanelplot(self.dataInv,
                                          trials=trials,
                                          channels=channels,
                                          toilim=toilim,
                                          avg_channels=avg_channels,
                                          fig=fig1)
                        assert figs_equal(fig1, fig3)

                        # Close figures to avoid memory overflow
                        plt.close("all")

        # The `selectdata(trials="all")` part requires consecutive trials!
        for channels in self.channels:
            for avg_channels in [True, False]:
                fig1 = self.rawReg.singlepanelplot(trials=None,
                                              channels=channels,
                                              avg_channels=avg_channels)
                selected = self.rawReg.selectdata(trials="all",
                                                  channels=channels)
                fig2 = selected.singlepanelplot(trials=None, avg_channels=avg_channels)
                assert figs_equal(fig1, fig2)
                plt.close("all")

        # Do not allow selecting time-intervals w/o trial-specification
        with pytest.raises(SPYValueError):
            self.dataReg.singlepanelplot(trials=None, toilim=self.toilim[1])

        # Do not overlay multi-channel plot w/chan-average and unequal channel-count
        multiChannelFig = self.dataReg.singlepanelplot(avg_channels=False,
                                                  channels=self.channels[1])
        with pytest.raises(SPYValueError):
            self.dataReg.singlepanelplot(avg_channels=True, fig=multiChannelFig)
        with pytest.raises(SPYValueError):
            self.dataReg.singlepanelplot(channels="all", fig=multiChannelFig)

        # Do not overlay multi-panel plot w/figure produced by `singlepanelplot`
        multiChannelFig.nTrialsPanels = 99
        with pytest.raises(SPYValueError):
            self.dataReg.singlepanelplot(fig=multiChannelFig)
        multiChannelFig.nChanPanels = 99
        with pytest.raises(SPYValueError):
            self.dataReg.singlepanelplot(fig=multiChannelFig)

        # Ensure grid and title specifications are rendered correctly
        theTitle = "A title"
        gridFig = self.dataReg.singlepanelplot(grid=True)
        assert gridFig.axes[0].get_xgridlines()[0].get_visible() == True
        titleFig = self.dataReg.singlepanelplot(title=theTitle)
        assert titleFig.axes[0].get_title() == theTitle

        plt.close("all")

    def test_multipanelplot(self):

        # Lowest possible dpi setting permitting valid png comparisons in `figs_equal`
        mpl.rcParams["figure.dpi"] = 75

        # Test everything except "raw" plotting
        for trials in self.trials:
            for channels in self.channels:
                for toilim in self.toilim:
                    for avg_trials in [True, False]:
                        for avg_channels in [True, False]:

                            # ``avg_trials == avg_channels == True`` yields `SPYWarning` to
                            # use `singlepanelplot` -> no figs to compare here
                            if avg_trials is avg_channels is True:
                                continue

                            # Render figure using multipanelplot mechanics, recreate w/`selectdata`,
                            # results must be identical
                            fig1 = self.dataReg.multipanelplot(trials=trials,
                                                          channels=channels,
                                                          toilim=toilim,
                                                          avg_trials=avg_trials,
                                                          avg_channels=avg_channels)
                            selected = self.dataReg.selectdata(trials=trials,
                                                               channels=channels,
                                                               toilim=toilim)
                            fig2 = selected.multipanelplot(trials="all",
                                                      avg_trials=avg_trials,
                                                      avg_channels=avg_channels)

                            # Recreate `fig1` and `fig2` in a single sweep by using
                            # `spy.multipanelplot` w/multiple input objects
                            fig1a, fig2a = multipanelplot(self.dataReg, self.dataInv,
                                                     trials=trials,
                                                     channels=channels,
                                                     toilim=toilim,
                                                     avg_trials=avg_trials,
                                                     avg_channels=avg_channels,
                                                     overlay=False)

                            # `selectdata` preserves trial order but not numbering: ensure
                            # plot titles are correct, but then remove them to allow
                            # comparison of `selected` and `dataReg` figures
                            figTitleLists = []
                            if avg_trials is False:
                                if trials != "all":
                                    for fig in [fig1, fig1a]:
                                        titleList = []
                                        for ax in fig.axes:
                                            titleList.append(copy(ax.title))
                                            ax.set_title("")
                                        titles = [title.get_text() for title in titleList]
                                        assert titles == ["Trial #{}".format(trlno) for trlno in trials]
                                        figTitleLists.append(titleList)
                                    for fig in [fig2, fig2a]:
                                        for ax in fig.axes:
                                            ax.set_title("")

                            # After (potential) axes title removal, compare figures;
                            # `fig2a` is based on `dataInv` - be more lenient there
                            tol = None
                            if avg_channels:
                                tol = 1e-2
                            assert figs_equal(fig1, fig2)
                            assert figs_equal(fig1, fig1a)
                            assert figs_equal(fig2, fig2a, tol=tol)
                            assert figs_equal(fig1, fig2a, tol=tol)

                            # If necessary, restore axes title from `figTitleLists`
                            if figTitleLists:
                                for k, ax in enumerate(fig1.axes):
                                    ax.title = figTitleLists[0][k]

                            # Create overlay figures: `fig3` combines `dataReg` and
                            # `dataInv` - must be identical to overlaying `fig1` w/`dataInv`
                            fig3 = multipanelplot(self.dataReg, self.dataInv,
                                             trials=trials,
                                             channels=channels,
                                             toilim=toilim,
                                             avg_trials=avg_trials,
                                             avg_channels=avg_channels)
                            fig4 = multipanelplot(self.dataInv,
                                             trials=trials,
                                             channels=channels,
                                             toilim=toilim,
                                             avg_trials=avg_trials,
                                             avg_channels=avg_channels,
                                             fig=fig1)
                            assert figs_equal(fig3, fig4)

                            plt.close("all")

        # The `selectdata(trials="all")` part requires consecutive trials! Add'ly,
        # `avg_channels` must be `False`, otherwise single-panel plot warning is triggered
        for channels in self.channels:
                fig1 = self.rawReg.multipanelplot(trials=None,
                                             channels=channels,
                                             avg_channels=False,
                                             avg_trials=False)
                selected = self.rawReg.selectdata(trials="all",
                                                  channels=channels)
                fig2 = selected.multipanelplot(trials=None,
                                          avg_channels=False)
                assert figs_equal(fig1, fig2)
                plt.close("all")

        # Do not allow selecting time-intervals w/o trial-specification
        with pytest.raises(SPYValueError):
            self.dataReg.multipanelplot(trials=None, toilim=self.toilim[1])

        # Panels = trials, each panel shows single (averaged) channel
        multiTrialSingleChanFig = self.dataReg.multipanelplot(trials=self.trials[1],
                                                         avg_trials=False,
                                                         avg_channels=True)
        with pytest.raises(SPYValueError):  # trial-count does not match up
            self.dataReg.multipanelplot(trials="all",
                                   avg_trials=False,
                                   avg_channels=True,
                                   fig=multiTrialSingleChanFig)
        with pytest.raises(SPYValueError):  # multi-channel overlay
            self.dataReg.multipanelplot(trials=self.trials[1],
                                   avg_trials=False,
                                   avg_channels=False,
                                   fig=multiTrialSingleChanFig)

        # Panels = trials, each panel shows multiple channels
        multiTrialMultiChanFig = self.dataReg.multipanelplot(trials=self.trials[1],
                                                         avg_trials=False,
                                                         avg_channels=False)
        with pytest.raises(SPYValueError):  # no trial specification provided
            self.dataReg.multipanelplot(trials=None,
                                   avg_trials=False,
                                   avg_channels=False,
                                   fig=multiTrialMultiChanFig)
        with pytest.raises(SPYValueError):  # channel-count does not match up
            self.dataReg.multipanelplot(trials=self.trials[1],
                                   channels=self.channels[1],
                                   avg_trials=False,
                                   avg_channels=False,
                                   fig=multiTrialMultiChanFig)
        with pytest.raises(SPYValueError):  # single-channel overlay
            self.dataReg.multipanelplot(trials=self.trials[1],
                                   avg_trials=False,
                                   avg_channels=True,
                                   fig=multiTrialMultiChanFig)

        # Panels = channels
        multiChannelFig = self.dataReg.multipanelplot(trials=self.trials[1],
                                                 avg_trials=True,
                                                 avg_channels=False)
        with pytest.raises(SPYValueError):  # multi-trial overlay
            self.dataReg.multipanelplot(trials=self.trials[1],
                                   avg_trials=False,
                                   avg_channels=False,
                                   fig=multiChannelFig)
        with pytest.raises(SPYValueError):  # channel-count does not match up
            self.dataReg.multipanelplot(trials=self.trials[1],
                                   channels=self.channels[1],
                                   avg_trials=True,
                                   avg_channels=False,
                                   fig=multiChannelFig)

        # Do not overlay single-panel plot w/figure produced by `multipanelplot`
        singlePanelFig = self.dataReg.singlepanelplot()
        with pytest.raises(SPYValueError):
            self.dataReg.multipanelplot(fig=singlePanelFig)

        # Ensure grid and title specifications are rendered correctly
        theTitle = "A title"
        gridFig = self.dataReg.multipanelplot(avg_trials=False,
                                              avg_channels=True,
                                              grid=True)
        assert all([gridFig.axes[k].get_xgridlines()[0].get_visible() for k in range(self.nTrials)])
        titleFig = self.dataReg.multipanelplot(avg_trials=False,
                                          avg_channels=True,
                                          title=theTitle)
        assert titleFig._suptitle.get_text() == theTitle

max_stars_repo_name: KatharineShapcott/syncopy

max_issues_repo_name: KatharineShapcott/syncopy

max_forks_repo_name: KatharineShapcott/syncopy


text:
import errno
import os
import shutil

import numpy as np
import torch
from matplotlib import pyplot as plt
from torch import nn as nn
from torch.utils.data import Dataset

def mkdir_p(path):
    '''make dir if not exist'''
    try:
        os.makedirs(path)
    except OSError as exc:  # Python >2.5
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            pass
        else:
            raise

class AverageMeter:
    """Computes and stores the average and current value
       Imported from https://github.com/pytorch/examples/blob/master/imagenet/main.py#L247-L262
    """
    def __init__(self):
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count

def interleave_offsets(batch, nu):
    """
    Args:
        batch: batch_size
        nu: n 个 batch 减一

    Returns:

    """
    groups = [batch // (nu + 1)] * (nu + 1)
    for x in range(batch - sum(groups)):
        groups[-x - 1] += 1
    offsets = [0]
    for g in groups:
        offsets.append(offsets[-1] + g)
    assert offsets[-1] == batch
    return offsets

def interleave(xy, batch):
    """
    Args:
        xy: n 个 batch 的 list
        batch: batch_size

    Returns:

    例子：
    batch_size: 128
    len(xy) = 6, 即有 6 个 batch
    那么经过 interleave_offsets 处理得到：
      labeled:  [21, 21, 21, 21, 22, 22]
    unlabeled: [[21, 21, 21, 21, 22, 22]
                [21, 21, 21, 21, 22, 22]
                [21, 21, 21, 21, 22, 22]
                [21, 21, 21, 21, 22, 22]
                [21, 21, 21, 21, 22, 22]]
    如果直接将以上每一行作为一个 batch 输入到模型中，那么情况变成第一个 batch 全是标注数据，后面的数据全部都是无标注数据
    所以下面函数将对角线上分别与第一行同一列进行调换以使得每一个 batch 都存在一部分标注数据，每一个 batch 的数据分布也变得较为一致
    """
    nu = len(xy) - 1
    offsets = interleave_offsets(batch, nu)
    xy = [[v[offsets[p]:offsets[p + 1]] for p in range(nu + 1)] for v in xy]
    for i in range(1, nu + 1):
        xy[0][i], xy[i][i] = xy[i][i], xy[0][i]
    return [torch.cat(v, dim=0) for v in xy]

def savefig(fname, dpi=150):
    plt.savefig(fname, dpi=dpi)

def plot_overlap(logger, names=None):
    names = logger.names if names == None else names
    numbers = logger.numbers
    for _, name in enumerate(names):
        x = np.arange(len(numbers[name]))
        plt.plot(x, np.asarray(numbers[name]))
    return [logger.title + '(' + name + ')' for name in names]

class Logger:
    '''Save training process to log file with simple plot function.'''
    def __init__(self, fpath, title=None, resume=False):
        self.file = None
        self.resume = resume
        self.title = '' if title == None else title
        if fpath is not None:
            if resume:
                self.file = open(fpath, 'r')
                name = self.file.readline()
                self.names = name.rstrip().split('\t')
                self.numbers = {}
                for _, name in enumerate(self.names):
                    self.numbers[name] = []

                for numbers in self.file:
                    numbers = numbers.rstrip().split('\t')
                    for i in range(0, len(numbers)):
                        self.numbers[self.names[i]].append(numbers[i])
                self.file.close()
                self.file = open(fpath, 'a')
            else:
                self.file = open(fpath, 'w')

    def set_names(self, names):
        if self.resume:
            pass
        # initialize numbers as empty list
        self.numbers = {}
        self.names = names
        for _, name in enumerate(self.names):
            self.file.write(name)
            self.file.write('\t')
            self.numbers[name] = []
        self.file.write('\n')
        self.file.flush()

    def append(self, numbers):
        assert len(self.names) == len(numbers), 'Numbers do not match names'
        for index, num in enumerate(numbers):
            self.file.write("{0:.6f}".format(num))
            self.file.write('\t')
            self.numbers[self.names[index]].append(num)
        self.file.write('\n')
        self.file.flush()

    def plot(self, names=None):
        names = self.names if names == None else names
        numbers = self.numbers
        for _, name in enumerate(names):
            x = np.arange(len(numbers[name]))
            plt.plot(x, np.asarray(numbers[name]))
        plt.legend([self.title + '(' + name + ')' for name in names])
        plt.grid(True)

    def close(self):
        if self.file is not None:
            self.file.close()

class LoggerMonitor:
    '''Load and visualize multiple logs.'''
    def __init__ (self, paths):
        '''paths is a distionary with {name:filepath} pair'''
        self.loggers = []
        for title, path in paths.items():
            logger = Logger(path, title=title, resume=True)
            self.loggers.append(logger)

    def plot(self, names=None):
        plt.figure()
        plt.subplot(121)
        legend_text = []
        for logger in self.loggers:
            legend_text += plot_overlap(logger, names)
        plt.legend(legend_text, bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
        plt.grid(True)

def accuracy(output, target):
    """Computes the precision@k for the specified values of k"""
    _, pred = torch.max(output, dim=1)
    correct = pred.float().eq(target).float()
    return correct.mean()

class SemiLoss:
    def __call__(self, lambda_u, outputs_x, targets_x, outputs_u, targets_u, cur_epoch, total_epoch):
        probs_u = torch.softmax(outputs_u, dim=1)

        # 标注数据损失
        # Lx = -torch.mean(torch.sum(torch.log_softmax(outputs_x, dim=1) * targets_x, dim=1))
        Lx = nn.BCEWithLogitsLoss()(outputs_x, targets_x)
        # 无标注数据损失
        Lu = torch.mean((probs_u - targets_u) ** 2)
        return Lx, Lu, lambda_u * linear_rampup(cur_epoch, total_epoch)  # 线性增大 lambda_u

class WeightEMA:
    def __init__(self, model, ema_model, lr, alpha=0.999):
        self.model = model
        self.ema_model = ema_model
        self.alpha = alpha
        self.params = list(model.state_dict().values())
        self.ema_params = list(ema_model.state_dict().values())
        self.wd = 0.02 * lr

        for param, ema_param in zip(self.params, self.ema_params):
            param.data.copy_(ema_param.data)

    def step(self):
        one_minus_alpha = 1.0 - self.alpha
        for param, ema_param in zip(self.params, self.ema_params):
            if ema_param.dtype != torch.float32: continue
            ema_param.mul_(self.alpha)
            ema_param.add_(param * one_minus_alpha)
            # customized weight decay
            param.mul_(1 - self.wd)

def save_checkpoint(state, is_best, checkpoint, filename='checkpoint.pth.tar'):
    filepath = os.path.join(checkpoint, filename)
    torch.save(state, filepath)
    if is_best:
        shutil.copyfile(filepath, os.path.join(checkpoint, 'model_best.pth.tar'))

def linear_rampup(current, rampup_length):
    if rampup_length == 0:
        return 1.0
    else:
        current = np.clip(current / rampup_length, 0.0, 1.0)
        return float(current)

def truncated_padded(token_ids, pad_id, max_length):
    if len(token_ids) < max_length:
        token_ids.extend([pad_id] * (max_length - len(token_ids)))
    else:
        token_ids = token_ids[:max_length]
    return token_ids

class MyIMDB(Dataset):

    def __init__(self, imdb, text_vocab, label_vocab, max_length=512, unlabeled=False):
        super(MyIMDB, self).__init__()
        self.imdb = imdb
        self.text_vocab = text_vocab
        self.label_vovab = label_vocab
        self.unlabeled = unlabeled
        self.max_length = max_length

    def __getitem__(self, idx):
        if self.unlabeled:
            example_a = self.imdb[2*idx]
            example_b = self.imdb[2*idx+1]
            input_a = [self.text_vocab.stoi[token] for token in example_a.text]
            input_b = [self.text_vocab.stoi[token] for token in example_b.text]
            input_a = truncated_padded(input_a, self.text_vocab.stoi['<pad>'], self.max_length)
            input_b = truncated_padded(input_b, self.text_vocab.stoi['<pad>'], self.max_length)
            return torch.tensor(input_a, dtype=torch.long), torch.tensor(input_b, dtype=torch.long)
        else:
            example = self.imdb[idx]
            input = [self.text_vocab.stoi[token] for token in example.text]
            input = truncated_padded(input, self.text_vocab.stoi['<pad>'], self.max_length)
            target = self.label_vovab.stoi[example.label]
            return torch.tensor(input, dtype=torch.long), torch.tensor(target, dtype=torch.float)

    def __len__(self):
        return len(self.imdb) if not self.unlabeled else len(self.imdb) // 2

max_stars_repo_name: HuipengXu/MixMatch-PyTorch

max_issues_repo_name: HuipengXu/MixMatch-PyTorch

max_forks_repo_name: HuipengXu/MixMatch-PyTorch


text:
import query
import document_scoring
from sklearn.metrics.pairwise import cosine_similarity
import numpy as np

class Ranking:
    def __init__(self, q):
        self.query = query.Query(q)
        '''
        Query Object
        '''
        self.doc_score = document_scoring.DocScore()
        '''
        Document Scoring Object
        '''
        self.similarity_scores = []
        '''
        Document - Query Cosine Similarity Scores
        '''
        self.document_ranking = []
        '''
        List of Doc Ids based on similarity score ranking
        '''

    def get_similarity_scores(self):
        '''
        Returns list of Document - Query Cosine Similarity Scores
        '''
        return self.similarity_scores

    def compute_similarity_score(self):
        '''
        Computes Document - Query Cosine Similarity Scores
        '''
        self.query.compute_query_score()
        query_vector = self.query.get_query_vector()
        doc_vectors = self.query.get_doc_vectors()

        for i in range(len(doc_vectors)):
            doc_vector = doc_vectors[i]
            similarity = cosine_similarity([doc_vector], [query_vector])
            self.similarity_scores.append(similarity[0][0])

    def compute_document_ranking(self):
        '''
        Calculates Document Ranking based on Document - Query Cosine Similarity Scores
        '''
        s = np.array(self.similarity_scores)
        sort_document = np.argsort(s)
        sort_document = [x + 1 for x in sort_document]
        self.document_ranking = sort_document[::-1]

    def get_doc_by_ranking(self):
        '''
        Prints top 10 Relevant Documents
        '''
        self.compute_similarity_score()
        self.compute_document_ranking()
        corpus = self.query.get_doc_content()
        print(self.document_ranking)

        for i in range(10):
            print('\n Document No ',i+1,'\n')
            print(corpus[self.document_ranking[i]-1])

if __name__ == "__main__":
    ranking = Ranking(['India','girl','education'])
    ranking.get_doc_by_ranking()

max_stars_repo_name: nitishabharathi/Vertical-Search-Engine

max_issues_repo_name: nitishabharathi/Vertical-Search-Engine

max_forks_repo_name: nitishabharathi/Vertical-Search-Engine


text:
import os

import cv2 as cv
import numpy as np

WIDTH = 64
HEIGHT = 80

for root, _, files in os.walk("fingerprints"):
    for file in files:
        if file.endswith(".png"):
            path = os.path.join(root, file)
            image: np.ndarray = cv.imread(path, 0)
            if image.shape != (HEIGHT, WIDTH):
                cv.imwrite(path, image[0:HEIGHT, 0:WIDTH])


text:
import numpy as np
import os
label_dict = {"克林顿": 0, "奥巴马": 1, "特朗普": 2}

def get_filenames(file_dir, ratio):
    file_list = []
    label_list = []
    for label_dir in os.listdir(file_dir):
        for filename in os.listdir(os.path.join(file_dir, label_dir)):
            file_list.append(os.path.join(os.path.join(file_dir, label_dir, filename)))
            label_list.append(label_dict[label_dir])

    # 利用shuffle打乱顺序
    all_data = np.array([file_list, label_list]).transpose()
    np.random.shuffle(all_data)
    train_num = int(np.ceil(ratio * len(all_data)))
    train = all_data[0:train_num, :]
    test = all_data[train_num:-1, :]
    train_filename = list(train[:, 0])
    train_label = [int(i) for i in list(train[:, 1])]
    test_filename = list(test[:, 0])
    test_label = [int(i) for i in list(test[:, 1])]
    return train_filename, train_label, test_filename, test_label


text:
import numpy as np
from dataclasses import dataclass
from typing import Callable

from manifold.maths import ortho_normal_matrix

"""
    Basis claasses to create embeddings in R3 (Embedding) or RN (TwoStepsEmbedding)
"""

@dataclass
class Embedding:
    """
        Base embedding class for embeddings in three dimensions
    """

    name: str
    phi: Callable

    def __call__(self, *args):
        return self.phi(*args)

class TwoStepsEmbedding:
    def __init__(self, name, phi_1, scale=1, N=64):
        """
            Base embedding class for embedding in R^n.
            The embedding is done in two steps: phi_1 embeds onto
            R^3 and an orthonormal matrix does R^3 -> R^n.
        """
        self.N = N
        self.name = name
        self.mtx = ortho_normal_matrix(self.N, 3)
        self.phi_1 = phi_1
        self.scale = scale

    def __call__(self, p):
        embedded = self.mtx @ np.array(self.phi_1(p)) * self.scale
        return embedded

max_stars_repo_name: FedeClaudi/manyfolds

max_issues_repo_name: FedeClaudi/manyfolds

max_forks_repo_name: FedeClaudi/manyfolds