text:
import numpy as np

def longest_common_token_sequence(left: list, right: list) -> int:
    """
    Implementation of Longest Common Subsequence using lists of tokens
    instead of strings, hence Longest Common Token Sequence.

    Parameters
    ----------
    left: list
        First first of tokens
    right: list
        Second list of tokens
    i_left: int
        Index for iterating over left list
    i_right: int
        Index for iterating over right list

    Returns
    -------
    int
        Size of the Logest Common Token Sequence found.
    """

    n_left = len(left)
    n_right = len(right)

    L = [[None] * (n_right + 1) for i in range(n_left + 1)]

    for i in range(n_left + 1):
        for j in range(n_right + 1):
            if i == 0 or j == 0:
                L[i][j] = 0
            elif left[i - 1] == right[j - 1]:
                L[i][j] = L[i - 1][j - 1] + 1
            else:
                L[i][j] = max(L[i - 1][j], L[i][j - 1])

    return L[n_left][n_right]

def evaluate_dataset(
    pred_tokens: list, true_tokens: list, complete_metrics: bool = False
) -> (float, float):
    """
    Evaluate tokenizer's results, returning precision, recall, f-score and extra_metrics if
    `complete_metrics` is set to True.

    Parameters
    ----------
    pred_tokens: list
        List of predicted tokens.
    true_tokens: list
        List of true tokens.
    complete_metrics: bool
        If true, return aditional metrics.

    Returns
    -------
    ((float, float), (float, float), (float, float))
        Precision, Recall and F-score measurements (mean, std).
    """
    assert len(pred_tokens) == len(true_tokens)
    n_sents = len(pred_tokens)
    true_positives = 0
    false_positives = 0
    false_negatives = 0
    incorrect_sentences_ids = []
    precision_scores = []
    recall_scores = []
    f_scores = []

    for i in range(n_sents):
        tp, fp, fn = evaluate_sentence(pred_tokens[i], true_tokens[i])
        true_positives += tp
        false_positives += fp
        false_negatives += fn
        precision_scores.append(tp / (tp + fp + 1e-9))
        recall_scores.append(tp / (tp + fn + 1e-9))
        f_scores.append(
            2
            * precision_scores[-1]
            * recall_scores[-1]
            / (precision_scores[-1] + recall_scores[-1] + 1e-9)
        )

        if fp > 0 or fn > 0:
            incorrect_sentences_ids.append(i)

    precision = (np.mean(precision_scores), np.std(precision_scores))
    recall = (np.mean(recall_scores), np.std(recall_scores))
    f_score = (np.mean(f_scores), np.std(f_scores))

    if complete_metrics:
        additional_metrics = {
            "true_positives": true_positives,
            "false_positives": false_positives,
            "false_negatives": false_negatives,
            "incorrect_sentences_ids": incorrect_sentences_ids,
        }
        return precision, recall, f_score, additional_metrics

    return precision, recall, f_score

def evaluate_sentence(pred_tokens: list, true_tokens: list) -> (int, int):
    """
    Calculates precision and coverage of tokens.
    Based on the Longest Common Subsequence Algorithm
    to evaluate pred_tokens and true_tokens, with the difference of
    not considering spaced substrings as matches.

    Examples
    --------
    >>> evaluate_sentence(["Oi", ":)"], ["Oi", ":)"])
    (2, 0, 0)

    Parameters
    ----------
    pred_tokens: list
        List of tokens generated by the tokenizer
    true_tokens: list
        List of true tokens

    Returns
    -------
    (int, int, int)
        True positive, false positives, false negatives
    """

    true_positives = longest_common_token_sequence(pred_tokens, true_tokens)
    false_positives = len(pred_tokens) - true_positives
    false_negatives = len(true_tokens) - true_positives

    return true_positives, false_positives, false_negatives


text:
# -*- coding: utf-8 -*-
# # ResNet 1-channel

import os
import numpy as np
import torch
from torch import nn
from torch.utils.data import DataLoader
import torch.backends.cudnn as cudnn
from torch.optim.lr_scheduler import CosineAnnealingLR
import torchvision
from torchvision.transforms import transforms
import argparse

from atm.simclr import resnet
from torch.utils.tensorboard import SummaryWriter
#from tensorflow import summary
from atm.simclr.utils import save_config_file, accuracy, save_checkpoint

# def save_checkpoint(state, is_best, filename='checkpoint.pth.tar'):
#    torch.save(state, filename)
#    if is_best:
#        shutil.copyfile(filename, 'model_best.pth.tar')

args = argparse.Namespace()

args.data='./datasets'
args.dataset=['cifar10', 'stl10', 'nair'][0]
args.arch='resnet50'
args.workers=1
args.epochs=200
args.img_size =128
args.n_channels=1
args.weight_decay=0.0008

def get_num_correct(preds, labels):
    return preds.argmax(dim=1).eq(labels).sum().item()

# check if gpu training is available
if torch.cuda.is_available():
    args.device = torch.device('cuda')
    cudnn.deterministic = True
    cudnn.benchmark = True
else:
    args.device = torch.device('cpu')

print("Using...", args.device)

model = resnet.resnet50(pretrained=False, num_classes=10)
model.to(args.device)

# print(model)

if args.dataset == 'cifar10':
    img_size = 32
    args.batch_size = 256
    args.lr = 0.001
elif args.dataset == 'stl10':
    img_size = 96
    args.batch_size = 64
    args.lr = 0.001

transform_train = transforms.Compose(
    [transforms.RandomCrop(img_size),
     transforms.RandomHorizontalFlip(p=0.5),
     transforms.ToTensor(),
     transforms.Lambda(lambda x: x.mean(dim=0, keepdim=True)),
     transforms.Normalize((0.5), (0.2))])

transform_test = transforms.Compose(
    #[transforms.RandomHorizontalFlip(p=0.5),
     [transforms.ToTensor(),
     transforms.Lambda(lambda x: x.mean(dim=0, keepdim=True)),
     transforms.Normalize((0.5), (0.2))])

if args.dataset == 'cifar10':
    trainset = torchvision.datasets.CIFAR10(root='./data', train=True,
                                        download=True, transform=transform_train)

    testset = torchvision.datasets.CIFAR10(root='./data', train=False,
                                       download=True, transform=transform_test)
elif args.dataset == 'stl10':
    trainset = torchvision.datasets.STL10(root='./data', split='train',
                                        download=True, transform=transform_train)
    testset = torchvision.datasets.STL10(root='./data', split='test',
                                       download=True, transform=transform_test)

train_loader = torch.utils.data.DataLoader(trainset, batch_size=args.batch_size,
                                          shuffle=True, num_workers=2)

test_loader = torch.utils.data.DataLoader(testset, batch_size=args.batch_size,
                                         shuffle=False, num_workers=2)

checkpoint_name=f'Resnet_ch1_{args.dataset}_bn{args.batch_size}_{args.epochs}_'

import torch.optim as optim

class Config:
    def __init__(self, **kwargs):
        for key, value in kwargs.items():
            setattr(self, key, value)

config = Config(
    trainloader = train_loader,
    testloader = test_loader,
    model = model,
    device = args.device,
    optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay),
    criterion= nn.CrossEntropyLoss().to(args.device),
    globaliter = 0
)

# criterion = nn.CrossEntropyLoss()
# optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.9)

# do Training

import time
class train_test():
    def __init__(self, config):
        self.trainloader = config.trainloader
        self.testloader = config.testloader
        self.model = config.model
        self.device = config.device
        self.optimizer = config.optimizer
        self.criterion = config.criterion
        self.globaliter = config.globaliter
        timestr = time.strftime("%Y%m%d-%H%M%S")
        log_dir = './runs/'+timestr + f"_{args.dataset}_{args.arch}_{args.n_channels}_{args.batch_size}"
        self.writer = SummaryWriter(log_dir=log_dir)
        print("log dir:", self.writer.log_dir)

    def train(self, epochs, log_interval):
        self.model.train()
        for epoch in range(epochs):  # loop over the dataset multiple times
            running_loss = 0.0
            for i, data in enumerate(self.trainloader, 0):
                # get the inputs
                self.globaliter += 1
                inputs, labels = data
                inputs = inputs.to(self.device)
                labels = labels.to(self.device)

                # zero the parameter gradients
                self.optimizer.zero_grad()

                # forward + backward + optimize
                outputs = self.model(inputs)
                loss = self.criterion(outputs, labels)
                loss.backward()
                self.optimizer.step()
                running_loss += loss.item()

                if i % log_interval == log_interval -1 :
                    print('Train Epoch: {} [{}/{} ({:.0f}%)]\tlearningLoss: {:.6f}\twhole_loss: {:.6f} '.format(
                        epoch, i*len(inputs), len(self.trainloader.dataset),
                        100. * i*len(inputs) / len(self.trainloader.dataset),
                        running_loss / log_interval,
                        loss.item()))
                    running_loss = 0.0

                    #with train_summary_writer.as_default():
                    #    summary.scalar('train_loss', loss.item() , step = self.globaliter)
                self.writer.add_scalar('train_loss', loss.item() , global_step = self.globaliter)
            # evaluate
            with torch.no_grad():
                self.model.eval()
                correct = 0
                total = 0
                test_loss = 0
                acc = []
                for k, data in enumerate(self.testloader, 0):
                    images, labels = data
                    images = images.to(self.device)
                    labels = labels.to(self.device)
                    outputs = self.model(images)

                    _, predicted = torch.max(outputs.data, 1)
                    total += labels.size(0)
                    correct += (predicted == labels).sum().item()
                    test_loss += self.criterion(outputs, labels).item()
                    acc.append(100 * correct/total)

            print('\nTest set : Average loss:{:.4f}, Accuracy: {}/{}({:.5f}%)\n'.format(
                  test_loss, correct, total, 100 * correct/total))
            self.writer.add_scalar('test_loss', test_loss , global_step = self.globaliter)
            self.writer.add_scalar('accuracy', 100 * correct/total, global_step = self.globaliter)

            lr_sche.step()

        save_checkpoint({
        'epoch': args.epochs,
        'state_dict': model.state_dict(),
        'optimizer': config.optimizer.state_dict(),
        }, is_best=False, filename=os.path.join(self.writer.log_dir, checkpoint_name+f'{epoch}.pth'))
        self.writer.close()

ready_to_train=train_test(config)
lr_sche = optim.lr_scheduler.StepLR(config.optimizer, step_size=5000, gamma=0.5) # 20 step마다 lr조정
log_interval = 58

ready_to_train.train(args.epochs, log_interval)

ext: py

max_stars_repo_name: Hoseung/ATM

max_issues_repo_name: Hoseung/ATM

max_forks_repo_name: Hoseung/ATM

reason: import numpy


text:
import pandas as pd
import researchpy as rp
import seaborn as sns

import statsmodels.api as sm
from statsmodels.formula.api import ols
import statsmodels.stats.multicomp

def find_min_discm_each_model(df):
    x = df.sort_values("Discm_percent").groupby("Model-count", as_index=False).first()
    assert len(x) == 240
    return x

# df = pandas.read_csv("min_discm.csv")
# df = pandas.read_csv("removal.csv")
# df1 = pd.read_csv("adult_results_first120.csv")
# df2 = pd.read_csv("adult_results_last120.csv")
# df3 = pd.read_csv("adult_results_lastest80.csv")
# df = pd.concat([df1, df2, df3])
# if len(df) > 240:
#     df = find_min_discm_each_model(df)
# df.to_csv("min_discm.csv", index=False)
# exit(0)
df = pd.read_csv("min_discm.csv")
# import ipdb; ipdb.set_trace()
# print(rp.summary_cont(df['Discm']))
# print(rp.summary_cont(df.groupby(['Permutation']))['Discm'])
model = ols('Discm_percent ~ C(H1Units) + C(H2Units) + C(Batch) + C(DataSplit)', df).fit()
# model = ols('Removal ~ C(H1units) + C(H2units) + C(Batch) + C(Permutation)', df).fit()
# model = ols('Error ~ C(H1units) + C(H2units) + C(Batch) + C(Permutation)', df).fit()
print(f"Overall model F({model.df_model: .0f},{model.df_resid: .0f}) = {model.fvalue: .3f}, p = {model.f_pvalue: .4f}")
print(model.summary())
# print(rp.summary_cont(df.groupby(['H2units']))['Error'])
res = sm.stats.anova_lm(model, typ= 2)
print(res)

#
# from sklearn.linear_model import LinearRegression
# target_col = 'Discm'
# X = df.loc[:, df.columns != target_col]
# Y = df.loc[:, target_col]
# lr = LinearRegression()
# lr.fit(X, Y)
# print(lr.coef_)
# print(lr.intercept_)

max_stars_repo_name: vsahil/influence-duplicate

max_issues_repo_name: vsahil/influence-duplicate

max_forks_repo_name: vsahil/influence-duplicate

reason: import statsmodels,from statsmodels


text:
#
# For licensing see accompanying LICENSE file.
# Copyright (C) 2019 Apple Inc. All Rights Reserved.
#
'''Capsule in PyTorch
TBD
'''

import torch
import torch.nn as nn
import torch.nn.functional as F

import numpy as np
import math

from .bilinear_sparse_routing import BilinearSparseRouting, BilinearRouting, DynamicBilinearRouting
from .linformer import LinformerProjectionKernel, BilinearProjectionWithEmbeddings

#### Simple Backbone ####
class simple_backbone(nn.Module):
    def __init__(self, cl_input_channels,cl_num_filters,cl_filter_size,
                                  cl_stride,cl_padding):
        super(simple_backbone, self).__init__()
        self.pre_caps = nn.Sequential(
                    nn.Conv2d(in_channels=cl_input_channels,
                                    out_channels=cl_num_filters,
                                    kernel_size=cl_filter_size,
                                    stride=cl_stride,
                                    padding=cl_padding),
                    nn.ReLU(),
        )
    def forward(self, x):
        out = self.pre_caps(x) # x is an image
        return out

#### ResNet Backbone ####
class BasicBlock(nn.Module):
    expansion = 1

    def __init__(self, in_planes, planes, stride=1):
        super(BasicBlock, self).__init__()
        self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)
        self.bn1 = nn.BatchNorm2d(planes)
        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False)
        self.bn2 = nn.BatchNorm2d(planes)

        self.shortcut = nn.Sequential()
        if stride != 1 or in_planes != self.expansion*planes:
            self.shortcut = nn.Sequential(
                nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False),
                nn.BatchNorm2d(self.expansion*planes)
            )

    def forward(self, x):
        out = F.relu(self.bn1(self.conv1(x)))
        out = self.bn2(self.conv2(out))
        out += self.shortcut(x)
        out = F.relu(out)
        return out

class resnet_backbone_cifar(nn.Module):
    def __init__(self, cl_input_channels, cl_num_filters,
                 cl_stride):
        super(resnet_backbone_cifar, self).__init__()
        self.in_planes = 64
        def _make_layer(block, planes, num_blocks, stride):
            strides = [stride] + [1]*(num_blocks-1)
            layers = []
            for stride in strides:
                layers.append(block(self.in_planes, planes, stride))
                self.in_planes = planes * block.expansion
            return nn.Sequential(*layers)

        self.pre_caps = nn.Sequential(
            nn.Conv2d(in_channels=cl_input_channels,
                      out_channels=64,
                      kernel_size=3,
                      stride=1,
                      padding=1,
                      bias=False),
            nn.BatchNorm2d(64),
            nn.ReLU(),
            _make_layer(block=BasicBlock, planes=64, num_blocks=3, stride=1), # num_blocks=2 or 3
            _make_layer(block=BasicBlock, planes=cl_num_filters, num_blocks=4, stride=cl_stride), # num_blocks=2 or 4
        )
    def forward(self, x):
        out = self.pre_caps(x) # x is an image
        return out

#Imagenet backbone
class resnet_backbone_imagenet(nn.Module):
    def __init__(self, cl_input_channels, cl_num_filters,
                 cl_stride):
        super(resnet_backbone_imagenet, self).__init__()
        self.in_planes = 64
        def _make_layer(block, planes, num_blocks, stride):
            # strides = [stride] + [1]*(num_blocks-1)
            strides = [stride]*3 + [1]*(num_blocks-1)

            layers = []
            for stride in strides:
                layers.append(block(self.in_planes, planes, stride))
                self.in_planes = planes * block.expansion
            return nn.Sequential(*layers)

        self.pre_caps = nn.Sequential(
            nn.Conv2d(in_channels=cl_input_channels,
                      out_channels=64,
                      kernel_size=3,
                      stride=1,
                      padding=1,
                      bias=False),
            nn.BatchNorm2d(64),
            nn.ReLU(),
            _make_layer(block=BasicBlock, planes=64, num_blocks=3, stride=1), # num_blocks=2 or 3
            # _make_layer(block=BasicBlock, planes=128, num_blocks=4, stride=cl_stride), # num_blocks=2 or 4
            _make_layer(block=BasicBlock, planes=cl_num_filters, num_blocks=4, stride=cl_stride), # num_blocks=2 or 4
            # _make_layer(block=BasicBlock, planes=512, num_blocks=2, stride=cl_stride), # num_blocks=2 or 4
        )
    def forward(self, x):
        out = self.pre_caps(x) # x is an image
        # print("Resnet backbone shape: ", out.shape)
        return out

###
# Explained einsum

'''

https://stackoverflow.com/questions/26089893/understanding-numpys-einsum

torch.einsum('i,ij->i', A, B)

    1. A has one axis; we've labelled it i. And B has two axes;
    we've labelled axis 0 as i and axis 1 as j.
    2. By repeating the label i in both input arrays, we are telling
    einsum that these two axes should be multiplied together.
    In other words, we're multiplying array A with each column of array B,
    just like A[:, np.newaxis] * B does.
    3. Notice that j does not appear as a label in our desired output;
    we've just used i (we want to end up with a 1D array).
    By omitting the label, we're telling einsum to sum along this axis.
    In other words, we're summing the rows of the products, just like .sum(axis=1) does.

'''

#### Capsule Layer ####
class CapsuleFC(nn.Module):
    r"""Applies as a capsule fully-connected layer.
    TBD
    """

    '''
    Same as CapsuleConv
    except that kernal size=1 everywhere.
    '''

    def __init__(self, in_n_capsules, in_d_capsules, out_n_capsules, out_d_capsules, matrix_pose, dp):
        super(CapsuleFC, self).__init__()
        self.in_n_capsules = in_n_capsules
        self.in_d_capsules = in_d_capsules
        self.out_n_capsules = out_n_capsules
        self.out_d_capsules = out_d_capsules
        self.matrix_pose = matrix_pose

        # Matrix form of Hilton
        if matrix_pose:
            self.sqrt_d = int(np.sqrt(self.in_d_capsules))
            self.weight_init_const = np.sqrt(out_n_capsules/(self.sqrt_d*in_n_capsules))
            self.w = nn.Parameter(self.weight_init_const* \
                                          torch.randn(in_n_capsules, self.sqrt_d, self.sqrt_d, out_n_capsules))

        # Vector form of Hilton
        else:
            self.weight_init_const = np.sqrt(out_n_capsules/(in_d_capsules*in_n_capsules))
            self.w = nn.Parameter(self.weight_init_const* \
                                          torch.randn(in_n_capsules, in_d_capsules, out_n_capsules, out_d_capsules))

        self.dropout_rate = dp
        self.nonlinear_act = nn.LayerNorm(out_d_capsules)
        self.drop = nn.Dropout(self.dropout_rate)
        self.scale = 1. / (out_d_capsules ** 0.5)

    def extra_repr(self):
        return 'in_n_capsules={}, in_d_capsules={}, out_n_capsules={}, out_d_capsules={}, matrix_pose={}, \
            weight_init_const={}, dropout_rate={}'.format(
            self.in_n_capsules, self.in_d_capsules, self.out_n_capsules, self.out_d_capsules, self.matrix_pose,
            self.weight_init_const, self.dropout_rate
        )
    def forward(self, input, num_iter, next_capsule_value=None):
        # b: batch size
        # n: num of capsules in current layer
        # a: dim of capsules in current layer
        # m: num of capsules in next layer
        # d: dim of capsules in next layer
        if len(input.shape) == 5:
            input = input.permute(0, 4, 1, 2, 3)
            input = input.contiguous().view(input.shape[0], input.shape[1], -1)
            input = input.permute(0,2,1)

        if self.matrix_pose:
            w = self.w # nxdm
            _input = input.view(input.shape[0], input.shape[1], self.sqrt_d, self.sqrt_d) # bnax
        else:
            w = self.w

        if next_capsule_value is None:
            # next_capsule_vale=None at 1st Iteration
            # query key == r_{i,j} (routing probabilities)
            query_key = torch.zeros(self.in_n_capsules, self.out_n_capsules).type_as(input)
            query_key = F.softmax(query_key, dim=1)

            if self.matrix_pose:
                # Einsum: computing multilinear expressions (i.e. sums of products) using the Einstein summation convention.
                next_capsule_value = torch.einsum('nm, bnax, nxdm->bmad', query_key, _input, w)
            else:
                next_capsule_value = torch.einsum('nm, bna, namd->bmd', query_key, input, w)
        else:
            if self.matrix_pose:
                next_capsule_value = next_capsule_value.view(next_capsule_value.shape[0],
                                       next_capsule_value.shape[1], self.sqrt_d, self.sqrt_d)
                # _query_key == agreement vector ( a_{i,j})
                _query_key = torch.einsum('bnax, nxdm, bmad->bnm', _input, w, next_capsule_value)
            else:
                _query_key = torch.einsum('bna, namd, bmd->bnm', input, w, next_capsule_value)

            # New routing probabilities
            _query_key.mul_(self.scale)
            query_key = F.softmax(_query_key, dim=2)
            query_key = query_key / (torch.sum(query_key, dim=2, keepdim=True) + 1e-10)

            if self.matrix_pose:
                # Use new routing values, to update state of parent capsule
                next_capsule_value = torch.einsum('bnm, bnax, nxdm->bmad', query_key, _input,
                                                  w)
            else:
                next_capsule_value = torch.einsum('bnm, bna, namd->bmd', query_key, input,
                                                  w)

        # Apply dropout
        next_capsule_value = self.drop(next_capsule_value)
        if not next_capsule_value.shape[-1] == 1:
            if self.matrix_pose:
                next_capsule_value = next_capsule_value.view(next_capsule_value.shape[0],
                                       next_capsule_value.shape[1], self.out_d_capsules)
                # Apply layer Norm
                next_capsule_value = self.nonlinear_act(next_capsule_value)
            else:
                next_capsule_value = self.nonlinear_act(next_capsule_value)
        return next_capsule_value

#
class CapsuleCONV(nn.Module):
    r"""Applies as a capsule convolutional layer.
    TBD
    """
    def __init__(self, in_n_capsules, in_d_capsules, out_n_capsules, out_d_capsules,
                 kernel_size, stride, matrix_pose, dp, coordinate_add=False):
        super(CapsuleCONV, self).__init__()
        self.in_n_capsules = in_n_capsules
        self.in_d_capsules = in_d_capsules
        self.out_n_capsules = out_n_capsules
        self.out_d_capsules = out_d_capsules
        self.kernel_size = kernel_size
        self.stride = stride
        self.matrix_pose = matrix_pose
        self.coordinate_add = coordinate_add

        if matrix_pose:
            self.sqrt_d = int(np.sqrt(self.in_d_capsules))
            self.weight_init_const = np.sqrt(out_n_capsules/(self.sqrt_d*in_n_capsules*kernel_size*kernel_size))
            self.w = nn.Parameter(self.weight_init_const*torch.randn(kernel_size, kernel_size,
                                                     in_n_capsules, self.sqrt_d, self.sqrt_d, out_n_capsules))

        else:
            self.weight_init_const = np.sqrt(out_n_capsules/(in_d_capsules*in_n_capsules*kernel_size*kernel_size))
            self.w = nn.Parameter(self.weight_init_const*torch.randn(kernel_size, kernel_size,
                                                     in_n_capsules, in_d_capsules, out_n_capsules,
                                                     out_d_capsules))

        self.nonlinear_act = nn.LayerNorm(out_d_capsules)
        self.dropout_rate = dp
        self.drop = nn.Dropout(self.dropout_rate)
        self.scale = 1. / (out_d_capsules ** 0.5)

    def extra_repr(self):
        return 'in_n_capsules={}, in_d_capsules={}, out_n_capsules={}, out_d_capsules={}, \
                    kernel_size={}, stride={}, coordinate_add={}, matrix_pose={}, weight_init_const={}, \
                    dropout_rate={}'.format(
            self.in_n_capsules, self.in_d_capsules, self.out_n_capsules, self.out_d_capsules,
            self.kernel_size, self.stride, self.coordinate_add, self.matrix_pose, self.weight_init_const,
            self.dropout_rate
            )
    def input_expansion(self, input):
        # input has size [batch x num_of_capsule x height x width x capsule_dimension]

        # unfold(dimension, size, step) → Tensor: Unfold Extracts sliding local blocks along given dim
        # extracts kernel patches over complete height and width
        unfolded_input = input.unfold(2,size=self.kernel_size,step=self.stride).unfold(3,size=self.kernel_size,step=self.stride)
        unfolded_input = unfolded_input.permute([0,1,5,6,2,3,4])
        # output has size [batch x num_of_capsule x kernel_size x kernel_size x h_out x w_out x capsule_dimension]
        return unfolded_input

    def forward(self, input, num_iter, next_capsule_value=None):
        # k,l: kernel size
        # h,w: output width and length
        # b: batch size
        # n: num of capsules in current layer
        # a: dim of capsules in current layer
        # m: num of capsules in next layer
        # d: dim of capsules in next layer

        # This converts (b,32,14,14,16) --> (b,32,3,3,7,7,16) (3X3 patches, 7 in number along both height and width)
        inputs = self.input_expansion(input)

        # print("Expansion: ",input.shape, inputs.shape)

        if self.matrix_pose:
            # W is pose of capsules of layer L
            # Input is capsule of layer L (p_{L})
            w = self.w # klnxdm

            # Converts (b,32,3,3,7,7,16) --> (b,32,3,3,7,7,4,4)
            _inputs = inputs.view(inputs.shape[0], inputs.shape[1], inputs.shape[2], inputs.shape[3],\
                                  inputs.shape[4], inputs.shape[5], self.sqrt_d, self.sqrt_d) # bnklmhax
            # print(_inputs.shape)

        else:
            w = self.w

        if next_capsule_value is None:
            # Routing probabilities in 1st iteration

            query_key = torch.zeros(self.in_n_capsules, self.kernel_size, self.kernel_size,
                                                self.out_n_capsules).type_as(inputs) # nklm

            query_key = F.softmax(query_key, dim=3) # softmax on output number of capsules
            # print("Query :",query_key.shape)
            # print("w :",w.shape)
            # print("input :",_inputs.shape)

            if self.matrix_pose:
                # a,x are sqrt_d if matrix pose and not vector pose
                # This performs convolution as well and attention both

                # next capsule shape is (b,32,7,7,16) just like original input
                '''

                for all b:
                    for all m:
                        for all h:
                            for all w:
                                for all a:
                                    for all d:
                                        for all n: (summing over capsules of layer L)
                                            for all k:
                                                for all l: (over all patches)
                                                    multiply input pose [(a,x)==(4,4)] with w [(x,d)==(4,4)]
                '''
                next_capsule_value = torch.einsum('nklm, bnklhwax, klnxdm->bmhwad', query_key,
                                              _inputs, w)

            else:
                # Vectorised implementation
                next_capsule_value = torch.einsum('nklm, bnklhwa, klnamd->bmhwd', query_key,
                                              inputs, w)
        else:
            if self.matrix_pose:
                # break 16 to (4,4) pose
                next_capsule_value = next_capsule_value.view(next_capsule_value.shape[0],\
                                         next_capsule_value.shape[1], next_capsule_value.shape[2],\
                                         next_capsule_value.shape[3], self.sqrt_d, self.sqrt_d)

                # w=(3,3,32,4,4,m), _input=(b,32,3,3,7,7,4,4) , next_capsule_value= (b,m,7,7,4,4)
                _query_key = torch.einsum('bnklhwax, klnxdm, bmhwad->bnklmhw', _inputs, w,
                                     next_capsule_value)

            else:
                _query_key = torch.einsum('bnklhwa, klnamd, bmhwd->bnklmhw', inputs, w,
                                     next_capsule_value)

            # Compute new routing probabilities
            _query_key.mul_(self.scale)
            query_key = F.softmax(_query_key, dim=4)
            query_key = query_key / (torch.sum(query_key, dim=4, keepdim=True) + 1e-10)

            if self.matrix_pose:
                # Update parent parent using new routing probabilties
                next_capsule_value = torch.einsum('bnklmhw, bnklhwax, klnxdm->bmhwad', query_key,
                                              _inputs, w)
                # print("others iter : ", next_capsule_value.shape)
            else:
                next_capsule_value = torch.einsum('bnklmhw, bnklhwa, klnamd->bmhwd', query_key,
                                              inputs, w)

        next_capsule_value = self.drop(next_capsule_value)
        if not next_capsule_value.shape[-1] == 1:
            if self.matrix_pose:
                # Correct size of parent capsule
                next_capsule_value = next_capsule_value.view(next_capsule_value.shape[0],\
                                         next_capsule_value.shape[1], next_capsule_value.shape[2],\
                                         next_capsule_value.shape[3], self.out_d_capsules)
                # Layer Norm
                next_capsule_value = self.nonlinear_act(next_capsule_value)
            else:
                next_capsule_value = self.nonlinear_act(next_capsule_value)

        return next_capsule_value

#### Capsule Layers with the proposed bilinear sparse routing ####
class SACapsuleFC(nn.Module):
    r"""Applies as a capsule fully-connected layer.
    TBD
    """
    def __init__(self, in_n_capsules, in_d_capsules, out_n_capsules, out_d_capsules, matrix_pose, dp):
        super(SACapsuleFC, self).__init__()
        self.in_n_capsules = in_n_capsules # This is n_caps * h_in * w_in
        self.in_d_capsules = in_d_capsules
        self.out_n_capsules = out_n_capsules
        self.out_d_capsules = out_d_capsules
        self.matrix_pose = matrix_pose
        self.dropout_rate = dp
        self.nonlinear_act = nn.LayerNorm(out_d_capsules)
        self.drop = nn.Dropout(self.dropout_rate)
        self.scale = 1. / (out_d_capsules ** 0.5)

        self.sinhkorn_caps_attn = BilinearSparseRouting(next_bucket_size=self.out_n_capsules, in_n_capsules=in_n_capsules, in_d_capsules=in_d_capsules, out_n_capsules=out_n_capsules,
                                                     out_d_capsules=out_d_capsules, matrix_pose=self.matrix_pose, layer_type='FC', kernel_size=1,
                                                     temperature = 0.75,
                                                    non_permutative = True, sinkhorn_iter = 7, n_sortcut = 2, dropout = 0., current_bucket_size = self.in_n_capsules//8,
                                                    use_simple_sort_net = False)

    def extra_repr(self):
        return 'in_n_capsules={}, in_d_capsules={}, out_n_capsules={}, out_d_capsules={}, matrix_pose={}, \
            dropout_rate={}'.format(
            self.in_n_capsules, self.in_d_capsules, self.out_n_capsules, self.out_d_capsules, self.matrix_pose,
            self.dropout_rate
        )
    def forward(self, input, num_iter, next_capsule_value=None):
        # b: batch size
        # n: num of capsules in current layer
        # a: dim of capsules in current layer
        # m: num of capsules in next layer
        # d: dim of capsules in next layer
        print("Input ", input.shape)
        if len(input.shape) == 5:
            input = input.permute(0, 4, 1, 2, 3)
            input = input.contiguous().view(input.shape[0], input.shape[1], -1)
            input = input.permute(0,2,1)

        print("Transformed ", input.shape)
        batch_size = input.shape[0]
        next_capsule_value = self.sinhkorn_caps_attn(current_pose=input, h_out=1, w_out=1, next_pose=next_capsule_value)
        next_capsule_value = self.drop(next_capsule_value)
        if not next_capsule_value.shape[-1] == 1:
            next_capsule_value = self.nonlinear_act(next_capsule_value)
        return next_capsule_value

class SACapsuleCONV(nn.Module):
    r"""Applies as a capsule convolutional layer.
    TBD
    """
    def __init__(self, in_n_capsules, in_d_capsules, out_n_capsules, out_d_capsules,
                 kernel_size, stride, matrix_pose, dp, padding=None, coordinate_add=False):
        super(SACapsuleCONV, self).__init__()
        self.in_n_capsules = in_n_capsules
        self.in_d_capsules = in_d_capsules
        self.out_n_capsules = out_n_capsules
        self.out_d_capsules = out_d_capsules
        self.kernel_size = kernel_size
        self.stride = stride
        self.matrix_pose = matrix_pose
        self.coordinate_add = coordinate_add
        self.padding = padding

        self.nonlinear_act = nn.LayerNorm(out_d_capsules)
        self.dropout_rate = dp
        self.drop = nn.Dropout(self.dropout_rate)

        self.sinhkorn_caps_attn = BilinearSparseRouting(next_bucket_size=self.out_n_capsules, in_n_capsules=in_n_capsules, in_d_capsules=in_d_capsules, out_n_capsules=out_n_capsules,
                                                     out_d_capsules=out_d_capsules, matrix_pose=self.matrix_pose, layer_type='conv', kernel_size=kernel_size,
                                                     temperature = 0.75,
                                                        non_permutative = True, sinkhorn_iter = 7, n_sortcut = 1, dropout = 0., current_bucket_size = self.in_n_capsules,
                                                        use_simple_sort_net = False)

    def extra_repr(self):
        return 'in_n_capsules={}, in_d_capsules={}, out_n_capsules={}, out_d_capsules={}, \
                    kernel_size={}, stride={}, coordinate_add={}, matrix_pose={},  \
                    dropout_rate={}'.format(
            self.in_n_capsules, self.in_d_capsules, self.out_n_capsules, self.out_d_capsules,
            self.kernel_size, self.stride, self.coordinate_add, self.matrix_pose,
            self.dropout_rate
            )

    def input_expansion(self, input):
        # input has size [batch x num_of_capsule x height x width x  x capsule_dimension]
        if self.padding:
            input = input.permute([0,1,4,2,3]) #For padding h,w
            if not self.padding%1:
                input = F.pad(input, [self.padding, self.padding, self.padding, self.padding]) #TODO: Padding to maintain same size, change so that caps dim not padded
            else:
                input = F.pad(input, [math.ceil(self.padding), math.floor(self.padding), math.ceil(self.padding), math.floor(self.padding)]) #TODO: Padding to maintain same size, change so that caps dim not padded
            input = input.permute([0,1,3,4,2])
        unfolded_input = input.unfold(2,size=self.kernel_size,step=self.stride).unfold(3,size=self.kernel_size,step=self.stride)
        unfolded_input = unfolded_input.permute([0,1,5,6,2,3,4])
        # output has size [batch x num_of_capsule x kernel_size x kernel_size x h_out x w_out x capsule_dimension]
        return unfolded_input

    def forward(self, input, num_iter, next_capsule_value=None):
        # k,l: kernel size
        # h,w: output width and length
        inputs = self.input_expansion(input)
        batch_size = inputs.shape[0]
        h_out = inputs.shape[4]
        w_out = inputs.shape[5]
        next_capsule_value = self.sinhkorn_caps_attn(current_pose=inputs, h_out=h_out, w_out=w_out, next_pose=next_capsule_value)
        next_capsule_value = self.drop(next_capsule_value)
        if not next_capsule_value.shape[-1] == 1:
            next_capsule_value = self.nonlinear_act(next_capsule_value)
        return next_capsule_value

#### Capsule Layers with the proposed bilinear routing without sinkhorn - Ablation study ####
class BACapsuleFC(nn.Module):
    r"""Applies as a capsule fully-connected layer.
    TBD
    """
    def __init__(self, in_n_capsules, in_d_capsules, out_n_capsules, out_d_capsules, matrix_pose, dp):
        super(BACapsuleFC, self).__init__()
        self.in_n_capsules = in_n_capsules
        self.in_d_capsules = in_d_capsules
        self.out_n_capsules = out_n_capsules
        self.out_d_capsules = out_d_capsules
        self.matrix_pose = matrix_pose
        self.dropout_rate = dp
        self.nonlinear_act = nn.LayerNorm(out_d_capsules)
        self.drop = nn.Dropout(self.dropout_rate)
        self.scale = 1. / (out_d_capsules ** 0.5)

        self.bilinear_attn = BilinearRouting(next_bucket_size=self.out_n_capsules, in_n_capsules=in_n_capsules, in_d_capsules=in_d_capsules, out_n_capsules=out_n_capsules,
                                                     out_d_capsules=out_d_capsules, matrix_pose=self.matrix_pose, layer_type='FC', kernel_size=1,
                                                     temperature = 0.75,
                                                    non_permutative = True, sinkhorn_iter = 7, n_sortcut = 2, dropout = 0., current_bucket_size = self.in_n_capsules//8,
                                                    use_simple_sort_net = False)

    def extra_repr(self):
        return 'in_n_capsules={}, in_d_capsules={}, out_n_capsules={}, out_d_capsules={}, matrix_pose={}, \
            dropout_rate={}'.format(
            self.in_n_capsules, self.in_d_capsules, self.out_n_capsules, self.out_d_capsules, self.matrix_pose,
            self.dropout_rate
        )
    def forward(self, input, num_iter, next_capsule_value=None):
        # b: batch size
        # n: num of capsules in current layer
        # a: dim of capsules in current layer
        # m: num of capsules in next layer
        # d: dim of capsules in next layer
        if len(input.shape) == 5:
            input = input.permute(0, 4, 1, 2, 3)
            input = input.contiguous().view(input.shape[0], input.shape[1], -1)
            input = input.permute(0,2,1)

        batch_size = input.shape[0]
        next_capsule_value = self.bilinear_attn(current_pose=input, h_out=1, w_out=1, next_pose=next_capsule_value)
        next_capsule_value = self.drop(next_capsule_value)
        if not next_capsule_value.shape[-1] == 1:
            next_capsule_value = self.nonlinear_act(next_capsule_value)
        return next_capsule_value

class BACapsuleCONV(nn.Module):
    r"""Applies as a capsule convolutional layer.
    TBD
    """
    def __init__(self, in_n_capsules, in_d_capsules, out_n_capsules, out_d_capsules,
                 kernel_size, stride, matrix_pose, dp, padding=None, coordinate_add=False):
        super(BACapsuleCONV, self).__init__()
        self.in_n_capsules = in_n_capsules
        self.in_d_capsules = in_d_capsules
        self.out_n_capsules = out_n_capsules
        self.out_d_capsules = out_d_capsules
        self.kernel_size = kernel_size
        self.stride = stride
        self.matrix_pose = matrix_pose
        self.coordinate_add = coordinate_add
        self.padding = padding

        self.nonlinear_act = nn.LayerNorm(out_d_capsules)
        self.dropout_rate = dp
        self.drop = nn.Dropout(self.dropout_rate)

        self.bilinear_attn = BilinearRouting(next_bucket_size=self.out_n_capsules, in_n_capsules=in_n_capsules, in_d_capsules=in_d_capsules, out_n_capsules=out_n_capsules,
                                                     out_d_capsules=out_d_capsules, matrix_pose=self.matrix_pose, layer_type='conv', kernel_size=kernel_size,
                                                     temperature = 0.75,
                                                        non_permutative = True, sinkhorn_iter = 7, n_sortcut = 1, dropout = 0., current_bucket_size = self.in_n_capsules,
                                                        use_simple_sort_net = False)

    def extra_repr(self):
        return 'in_n_capsules={}, in_d_capsules={}, out_n_capsules={}, out_d_capsules={}, \
                    kernel_size={}, stride={}, coordinate_add={}, matrix_pose={},  \
                    dropout_rate={}'.format(
            self.in_n_capsules, self.in_d_capsules, self.out_n_capsules, self.out_d_capsules,
            self.kernel_size, self.stride, self.coordinate_add, self.matrix_pose,
            self.dropout_rate
            )

    def input_expansion(self, input):
        # input has size [batch x num_of_capsule x height x width x  x capsule_dimension]
        if self.padding:
            input = input.permute([0,1,4,2,3]) #For padding h,w
            if not self.padding%1:
                input = F.pad(input, [self.padding, self.padding, self.padding, self.padding]) #TODO: Padding to maintain same size, change so that caps dim not padded
            else:
                input = F.pad(input, [math.ceil(self.padding), math.floor(self.padding), math.ceil(self.padding), math.floor(self.padding)]) #TODO: Padding to maintain same size, change so that caps dim not padded
            input = input.permute([0,1,3,4,2])
        unfolded_input = input.unfold(2,size=self.kernel_size,step=self.stride).unfold(3,size=self.kernel_size,step=self.stride)
        unfolded_input = unfolded_input.permute([0,1,5,6,2,3,4])
        # output has size [batch x num_of_capsule x kernel_size x kernel_size x h_out x w_out x capsule_dimension]
        return unfolded_input

    def forward(self, input, num_iter, next_capsule_value=None):
        # k,l: kernel size
        # h,w: output width and length
        inputs = self.input_expansion(input)
        batch_size = inputs.shape[0]
        h_out = inputs.shape[4]
        w_out = inputs.shape[5]
        next_capsule_value = self.bilinear_attn(current_pose=inputs, h_out=h_out, w_out=w_out, next_pose=next_capsule_value)
        next_capsule_value = self.drop(next_capsule_value)
        if not next_capsule_value.shape[-1] == 1:
            next_capsule_value = self.nonlinear_act(next_capsule_value)
        return next_capsule_value

#### Capsule Layers with the linformer projections and unfold operstions
class LACapsuleFC(nn.Module):
    r"""Applies as a capsule fully-connected layer.
    TBD
    """
    def __init__(self, in_n_capsules, in_d_capsules, out_n_capsules, out_d_capsules, input_img_size, output_img_size, hidden_dim, matrix_pose, dp):
        super(LACapsuleFC, self).__init__()
        self.in_n_capsules = in_n_capsules # This is n_caps * h_in * w_in
        self.in_d_capsules = in_d_capsules
        self.out_n_capsules = out_n_capsules
        self.out_d_capsules = out_d_capsules
        self.input_img_size = input_img_size
        self.output_img_size = output_img_size
        self.matrix_pose = matrix_pose
        self.dropout_rate = dp
        self.nonlinear_act = nn.LayerNorm(out_d_capsules)
        self.drop = nn.Dropout(self.dropout_rate)
        self.scale = 1. / (out_d_capsules ** 0.5)

        self.linformer_attention = LinformerProjectionKernel(in_n_capsules=in_n_capsules, in_d_capsules=in_d_capsules, out_n_capsules=out_n_capsules,
                                                     out_d_capsules=out_d_capsules, matrix_pose=self.matrix_pose, layer_type='FC', input_img_size = input_img_size, output_img_size = output_img_size,
                                                    hidden_dim = hidden_dim, kernel_size=1, dropout = 0.)

    def extra_repr(self):
        return 'in_n_capsules={}, in_d_capsules={}, out_n_capsules={}, out_d_capsules={}, matrix_pose={}, \
            dropout_rate={}'.format(
            self.in_n_capsules, self.in_d_capsules, self.out_n_capsules, self.out_d_capsules, self.matrix_pose,
            self.dropout_rate
        )
    def forward(self, input, num_iter, next_capsule_value=None):
        # b: batch size
        # n: num of capsules in current layer
        # a: dim of capsules in current layer
        # m: num of capsules in next layer
        # d: dim of capsules in next layer
        # print("Input ", input.shape)
        if len(input.shape) == 5:
            input = input.permute(0, 4, 1, 2, 3)
            input = input.contiguous().view(input.shape[0], input.shape[1], -1)
            input = input.permute(0,2,1)

        # print("Transformed ", input.shape)
        batch_size = input.shape[0]
        next_capsule_value = self.linformer_attention(current_pose=input, h_out=1, w_out=1, next_pose=next_capsule_value)
        next_capsule_value = self.drop(next_capsule_value)
        if not next_capsule_value.shape[-1] == 1:
            next_capsule_value = self.nonlinear_act(next_capsule_value)
        return next_capsule_value

class LACapsuleCONV(nn.Module):
    r"""Applies as a capsule convolutional layer.
    TBD
    """
    def __init__(self, in_n_capsules, in_d_capsules, out_n_capsules, out_d_capsules,
                 kernel_size, stride, input_img_size, output_img_size, hidden_dim,matrix_pose, dp, padding=None, coordinate_add=False):
        super(LACapsuleCONV, self).__init__()
        self.in_n_capsules = in_n_capsules
        self.in_d_capsules = in_d_capsules
        self.out_n_capsules = out_n_capsules
        self.out_d_capsules = out_d_capsules
        self.input_img_size = input_img_size
        self.output_img_size = output_img_size
        self.kernel_size = kernel_size
        self.stride = stride
        self.matrix_pose = matrix_pose
        self.coordinate_add = coordinate_add
        self.padding = padding

        self.nonlinear_act = nn.LayerNorm(out_d_capsules)
        self.dropout_rate = dp
        self.drop = nn.Dropout(self.dropout_rate)

        self.linformer_attention = LinformerProjectionKernel(in_n_capsules=in_n_capsules, in_d_capsules=in_d_capsules, out_n_capsules=out_n_capsules,
                                                     out_d_capsules=out_d_capsules, matrix_pose=self.matrix_pose, layer_type='conv', input_img_size = input_img_size, output_img_size = output_img_size,
                                                     hidden_dim=hidden_dim, kernel_size=kernel_size, dropout = 0.)

    def extra_repr(self):
        return 'in_n_capsules={}, in_d_capsules={}, out_n_capsules={}, out_d_capsules={}, \
                    kernel_size={}, stride={}, coordinate_add={}, matrix_pose={},  \
                    dropout_rate={}'.format(
            self.in_n_capsules, self.in_d_capsules, self.out_n_capsules, self.out_d_capsules,
            self.kernel_size, self.stride, self.coordinate_add, self.matrix_pose,
            self.dropout_rate
            )

    def input_expansion(self, input):
        # input has size [batch x num_of_capsule x height x width x  x capsule_dimension]
        if self.padding:
            input = input.permute([0,1,4,2,3]) #For padding h,w
            if not self.padding%1:
                input = F.pad(input, [self.padding, self.padding, self.padding, self.padding]) #TODO: Padding to maintain same size, change so that caps dim not padded
            else:
                input = F.pad(input, [math.ceil(self.padding), math.floor(self.padding), math.ceil(self.padding), math.floor(self.padding)]) #TODO: Padding to maintain same size, change so that caps dim not padded
            input = input.permute([0,1,3,4,2])
        unfolded_input = input.unfold(2,size=self.kernel_size,step=self.stride).unfold(3,size=self.kernel_size,step=self.stride)
        unfolded_input = unfolded_input.permute([0,1,5,6,2,3,4])
        # output has size [batch x num_of_capsule x kernel_size x kernel_size x h_out x w_out x capsule_dimension]
        return unfolded_input

    def forward(self, input, num_iter, next_capsule_value=None):
        # k,l: kernel size
        # h,w: output width and length
        inputs = self.input_expansion(input)
        batch_size = inputs.shape[0]
        h_out = inputs.shape[4]
        w_out = inputs.shape[5]
        next_capsule_value = self.linformer_attention(current_pose=inputs, h_out=h_out, w_out=w_out, next_pose=next_capsule_value)
        next_capsule_value = self.drop(next_capsule_value)
        if not next_capsule_value.shape[-1] == 1:
            next_capsule_value = self.nonlinear_act(next_capsule_value)
        return next_capsule_value

max_stars_repo_name: Sharut/Bilinear-Linformer

max_issues_repo_name: Sharut/Bilinear-Linformer

max_forks_repo_name: Sharut/Bilinear-Linformer


text:
# -*- coding: utf-8 -*-
"""
Created on Wed Apr 25 13:40:13 2018

@author: stevenconnorg
"""

#
import os
import pandas
import arcpy
import numpy

xlsx = r"C:\Users\stevenconnorg\Documents\knight-federal-solutions\GDB_DataReview\GDB_DataReview\dat\OSD RPI Site (For Components) FOUO.xlsx"
gdb = r"C:\Users\stevenconnorg\Documents\knight-federal-solutions\GeoBASE_3101_CIP_FINAL_20180502\GeoBASE_3101_CIP_FINAL_20180502.gdb"

def get_field_names(table):
    """
    Get a list of field names not inclusive of the geometry and object id fields.

    Parameters
    ----------
    table: Table readable by ArcGIS
    Returns
    -------
    List of field names.
    """
    # list to store values
    field_list = []

    # iterate the fields
    for field in arcpy.ListFields(table):

        # if the field is not geometry nor object id, add it as is
        if field.type != 'Geometry' and field.type != 'OID':
            field_list.append(field.name)

        # if geomtery is present, add both shape x and y for the centroid
        elif field.type == 'Geometry':
            field_list.append('SHAPE@XY')

    # return the field list
    return field_list

# to convert arcgis table to pandas dataframe
def table_to_pandas_dataframe(table, field_names=None):
    """
    Load data into a Pandas Data Frame from esri geodatabase table for subsequent analysis.

    Parameters
    ----------
    table = Table readable by ArcGIS.
    field_names: List of fields.
    Returns
    -------
    Pandas DataFrame object.

    """
    # if field names are not specified
    if not field_names:

        # get a list of field names
        field_names = get_field_names(table)

    # create a pandas data frame
    dataframe = pandas.DataFrame(columns=field_names)

    # use a search cursor to iterate rows
    with arcpy.da.SearchCursor(table, field_names) as search_cursor:

        # iterate the rows
        for row in search_cursor:

            # combine the field names and row items together, and append them
            dataframe = dataframe.append(
                dict(zip(field_names, row)),
                ignore_index=True
            )

    # return the pandas data frame
    return dataframe

# to get a pandas dataframe into a arcgis table
def pandas_to_table(pddf,tablename):
    '''
    Parameters
    ----------
    pddf = pandas dataframe
    tablename = output table name to 'installGDB'

    Returns
    -------
    a geodatabase table from pandas dataframe inside 'installGDB' geodatabase object (string to .gdb path)
    '''
    x = numpy.array(numpy.rec.fromrecords(pddf))
    names = pddf.dtypes.index.tolist()
    for i,item in enumerate(names):
        newName = (item,str(item).replace(' ', '_'))[-1]
        names[i] = newName
    x.dtype.names = tuple(names)
    gdbTbl = os.path.join(arcpy.env.workspace,tablename)
    if arcpy.Exists(gdbTbl):
        arcpy.Delete_management(gdbTbl)
    arcpy.da.NumPyArrayToTable(x, gdbTbl)

def feature_class_to_pandas_data_frame(feature_class, field_list):
    """
    Load data into a Pandas Data Frame for subsequent analysis.
    :param feature_class: Input ArcGIS Feature Class.
    :param field_list: Fields for input.
    :return: Pandas DataFrame object.
    """
    return pandas.DataFrame(
        arcpy.da.FeatureClassToNumPyArray(
            in_table=feature_class,
            field_names=field_list,
            skip_nulls=False,
            null_value=-99999
        )
    )

fds = "Cadastre"
fClass = "Site_A"
fcPath = os.path.join(gdb,fds,fClass)

arcpy.env.workspace =gdb

fcFields = []
for fld in arcpy.ListFields(fcPath):
    fcFields.append(fld.name)

iDat = pandas.DataFrame( [row for row in arcpy.da.SearchCursor(fcPath, fcFields) ] )

dDat = pandas.read_excel(xlsx)

disDat = dDat.loc[dDat['RPAD Submitter'] == 'AF']

# =============================================================================
# g = disDat.columns.to_series().groupby(disDat.dtypes).groups
# g
# =============================================================================

dRPSUID = disDat.RPSUID.unique()
dRPSUID = list(dRPSUID)

iDat.columns = fcFields
iRPSUID = iDat.realPropertySiteUniqueID.unique()
iRPSUID = list(iRPSUID)

iRPSUID = [str(x) for x in iRPSUID]
dRPSUID = [str(x) for x in dRPSUID]

# =============================================================================
# missing = set(dRPSUID) - set(iRPSUID)
# mRPSUIDs = [ x for x in missing ]
#
# extra = set(iRPSUID) - set(dRPSUID)
# exRPSUIDs = [ x for x in extra ]
#
# =============================================================================

missing = list(numpy.setdiff1d(dRPSUID,iRPSUID))
extra = list(numpy.setdiff1d(iRPSUID,dRPSUID))
isIn=[]
for val in iRPSUID:
    if val in dRPSUID:
        isIn.append(val)

#newDf = disDat
#pandas.options.mode.chained_assignment = None  # default='warn'
#disDat = disDat.assign(missingRPSUID=None)

##
list(disDat)
newNames = []
for n in list(disDat):
    newName = (n,n.replace(' ', '_'))[-1]
    newNames.append(newName)

disDat.columns = newNames

# disDat_M = disDat[disDat['RPSUID'].isin(missing)]
missingIndices = disDat['RPSUID'].isin(missing)
missingIndices = disDat[missingIndices]
missingIndices
# disDat_M = disDat[disDat['RPSUID'].isin(missing)]
extraIndices = disDat['RPSUID'].isin(extra)
extraIndices = disDat[extraIndices]
extraIndices

# disDat_M = disDat[disDat['RPSUID'].isin(missing)]
inIndices = disDat['RPSUID'].isin(isIn)
inIndices = disDat[inIndices]
inIndices

disDat.Component_Comments.loc[missingIndices.index] = "Missing in Geodatabase"
disDat.Component_Comments.loc[extraIndices.index] = "Non-SDS"
disDat.Component_Comments.loc[inIndices.index] = "Not Missing"

outName = (xlsx,xlsx.replace(' ', '_'))[-1]
writer = pandas.ExcelWriter(outName)
disDat.to_excel(writer)
writer.save()


text:
from twisted.internet import reactor, task, defer, threads
import heapq
import time
import copy
from twisted.internet.defer import DeferredQueue, DeferredLock
import weakref
import csv
import copy
import os
import threading
import numpy as np

class ProcessQueue():

	def __init__(self):
		self.queue = []
		self.consumers = []

		self.target = 0.5

		self.Kp = 3.0 #2
		self.Ki = 0.0 #1
		self.Kd = 15.0 #5

		self.drop = 0
		self.sum_Error = []
		self.timings = 0.0
		self.prev_Error = 0.0

		self.lock = threading.Lock()
		self.logs = []

	def __contains__(self, key):
		return key in self.queue

	def put(self, obj):
		insert_t = time.time()
		self.queue.append((insert_t, obj))

		self._process()

	def get(self):
		d = defer.Deferred()
		self.consumers += [d]

		reactor.callLater(0.0, self._process)
		return d

	def _process(self):
		try:
			while self.consumers != [] and self.queue != []:
				d = self.consumers.pop(0)
				obj = self.queue.pop(0)
				dt = threads.deferToThread(self._process_in_thread, d, obj)
				#reactor.callInThread(self._process_in_thread, d, obj)
		except Exception, e:
			print str(e)

	def _process_in_thread(self, d, obj):

		inserted_time, message = obj
		start_time = time.time()
		d.callback(message)
		end_time = time.time()

		# self.timings = 0 * self.timings + 1 * (start_time - inserted_time)

		# the proportional term produces an output value that is proportional to the current error value
		# P = self.timings - self.target
		with self.lock:
			P = (start_time - inserted_time) - self.target

		# the contribution from the integral term is proportional to both the magnitude of the error and the duration of the error
		# I = 0.8 * self.sum_Error + 0.2 * P # the integral in a PID controller is the sum of the instantaneous error over time and gives the accumulated offset that should have been corrected previously
			self.sum_Error.append(P)
			self.sum_Error = self.sum_Error[-1000:]
			I  = sum(self.sum_Error)

		# Derivative action predicts system behavior and thus improves settling time and stability of the system
		#D = P - I # the derivative of the process error is calculated by determining the slope of the error over time
			D = P - self.prev_Error

			self.prev_Error = P

			self.drop += self.Kp*P + self.Ki*I + self.Kd*D
			drop_tmp = self.drop
			# save_drop = self.drop

			self.drop = max(0.0, self.drop)

			tmp = np.random.poisson(self.drop)
			q_len = len(self.queue)
			del self.queue[:int(tmp)]

		# print "===== Delay: %.2f ==== Latency: %.2f ===== Estimate: %.2f =====" % (start_time - inserted_time, end_time - start_time, self.timings)
		# print "====Before queue len: %.2f ==== Queue Len: %.2f ==== Drop Len: %.2f ======" % (q_len, len(self.queue), self.drop)

			dataTmp = [tmp, P, I, D, q_len, start_time - inserted_time]
			self.log(dataTmp)

	def log(self, data):
		self.logs.append(data)
		if len(self.logs) > 1000:
			with open('PIDcontrolVal.csv', 'ab') as outfile:
				csvW = csv.writer(outfile, delimiter=',')
				csvW.writerows(self.logs)
			self.logs = []

max_stars_repo_name: yijiull/loopix

max_issues_repo_name: yijiull/loopix

max_forks_repo_name: yijiull/loopix


text:
import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np

from layers import Linear, LengthRegulator, PostNet
import hparams
import utils

device = torch.device('cuda'if torch.cuda.is_available()else 'cpu')

class LightSpeech(nn.Module):
    """ LightSpeech """

    def __init__(self):
        super(LightSpeech, self).__init__()
        self.embeddings = nn.Embedding(hparams.embedding_size+1,
                                       hparams.embedding_dim,
                                       padding_idx=0)

        self.pre_gru = nn.GRU(hparams.pre_gru_in_dim,
                              int(hparams.pre_gru_out_dim / 2),
                              num_layers=hparams.pre_gru_layer_size,
                              batch_first=True,
                              bidirectional=True)
        self.pre_linear = Linear(hparams.pre_gru_out_dim,
                                 hparams.post_gru_in_dim)
        self.LR = LengthRegulator()

        self.post_gru = nn.GRU(hparams.post_gru_in_dim,
                               int(hparams.post_gru_out_dim / 2),
                               num_layers=hparams.post_gru_layer_size,
                               batch_first=True,
                               bidirectional=True)
        self.post_linear = Linear(hparams.post_gru_out_dim,
                                  hparams.n_mel_channels)

        # self.postnet = PostNet()

    def pad_again(self, x, max_len):
        return F.pad(x, (0, 0, 0, max_len-x.size(1)))

    def mask(self, x, predicted, cemb, input_lengths, output_lengths, max_c_len, max_mel_len):
        x_mask = ~utils.get_mask_from_lengths(output_lengths, max_mel_len)
        x_mask = x_mask.expand(hparams.n_mel_channels,
                               x_mask.size(0), x_mask.size(1))
        x_mask = x_mask.permute(1, 2, 0)

        c_mask = ~utils.get_mask_from_lengths(input_lengths, max_c_len)
        c_mask = c_mask.expand(hparams.pre_gru_out_dim,
                               c_mask.size(0), c_mask.size(1))
        c_mask = c_mask.permute(1, 2, 0)

        x.data.masked_fill_(x_mask, 0.0)
        cemb.data.masked_fill_(c_mask, 0.0)
        predicted.data.masked_fill_(c_mask[:, :, 0], 0.0)

        return x, predicted, cemb

    def forward(self, character, length_c, length_mel, max_c_len, max_mel_len, D=None):
        x = self.embeddings(character)

        length_c_copy = length_c.cpu().numpy()
        x = nn.utils.rnn.pack_padded_sequence(
            x, length_c_copy, batch_first=True)
        self.pre_gru.flatten_parameters()
        x, _ = self.pre_gru(x)
        x, _ = nn.utils.rnn.pad_packed_sequence(x, batch_first=True)
        x = self.pad_again(x, max_c_len)

        cemb = self.pre_linear(x)

        x, predicted = self.LR(cemb, target=D, mel_max_length=max_mel_len)

        sorted_length_index = np.argsort(-length_mel.cpu().numpy())
        sorted_length_mel = np.array(list())
        x_sorted = list()

        for ind in sorted_length_index:
            x_sorted.append(x[ind])
            sorted_length_mel = np.append(
                sorted_length_mel, length_mel[ind].item())

        x_sorted = torch.stack(x_sorted).to(device)
        x_sorted = nn.utils.rnn.pack_padded_sequence(x_sorted,
                                                     sorted_length_mel,
                                                     batch_first=True)
        self.post_gru.flatten_parameters()
        x_sorted, _ = self.post_gru(x_sorted)
        x_sorted, _ = nn.utils.rnn.pad_packed_sequence(
            x_sorted, batch_first=True)
        x_sorted = self.pad_again(x_sorted, max_mel_len)
        x = list([0 for _ in range(x_sorted.size(0))])

        for i, ind in enumerate(sorted_length_index):
            x[ind] = x_sorted[i]
        x = torch.stack(x).to(device)

        x = self.post_linear(x)
        mel, predicted, cemb = self.mask(
            x, predicted, cemb, length_c, length_mel, max_c_len, max_mel_len)

        # mel_postnet_1, mel_postnet_2 = self.postnet(
        #     mel, length_mel, max_mel_len)

        # return [mel, mel_postnet_1, mel_postnet_2], predicted, cemb
        return mel, predicted, cemb

    def inference(self, character, alpha=1.0):
        x = self.embeddings(character)

        self.pre_gru.flatten_parameters()
        x, _ = self.pre_gru(x)

        x = self.pre_linear(x)
        x = self.LR(x, alpha=alpha)

        self.post_gru.flatten_parameters()
        x, _ = self.post_gru(x)
        mel = self.post_linear(x)
        # mel_postnet_1, mel_postnet_2 = self.postnet.inference(mel)

        # return mel, mel_postnet_1, mel_postnet_2
        return mel

if __name__ == "__main__":
    # Test
    num_1 = utils.get_param_num(LightSpeech())
    print(num_1)

    model = utils.get_Tacotron2()
    num_2 = utils.get_param_num(model)
    print(num_2 / num_1)

max_stars_repo_name: entn-at/LightSpeech

max_issues_repo_name: entn-at/LightSpeech

max_forks_repo_name: entn-at/LightSpeech


text:
#!/usr/bin/env python
# -*- coding: utf-8 -*-

from ..plotting import Plotter, InvalidArgumentsException
from datetime import datetime
from numpy import array
import pytest

def test_lineplot_construction():
    """Test that instances for lineplots get constructed correctly."""
    test_times = array((datetime(2018, 1, 1, 12, 0, 0),
                        datetime(2018, 1, 1, 12, 1, 0),
                        datetime(2018, 1, 1, 12, 2, 0),
                        datetime(2018, 1, 1, 12, 3, 0),
                        datetime(2018, 1, 1, 12, 4, 0),
                        datetime(2018, 1, 1, 12, 5, 0),
                        datetime(2018, 1, 1, 12, 6, 0),
                        datetime(2018, 1, 1, 12, 7, 0),
                        datetime(2018, 1, 1, 12, 8, 0),
                        datetime(2018, 1, 1, 12, 9, 0)))
    test_values = array((1, 2, 3, 4, 5, 6, 7, 8, 9, 10))
    plotter = Plotter('test plot', timestamps=test_times, values=test_values)
    assert plotter._plotfunc == plotter._line_plot
    assert (plotter._timestamps == test_times).all()
    assert (plotter._values == test_values).all()
    assert plotter._type == 'test plot'

def test_histogram_construction():
    """Test that instances for histogram plots get constructed correctly."""
    test_times = array((datetime(2018, 1, 1, 12, 0, 0),
                        datetime(2018, 1, 1, 12, 1, 0),
                        datetime(2018, 1, 1, 12, 2, 0),
                        datetime(2018, 1, 1, 12, 3, 0),
                        datetime(2018, 1, 1, 12, 4, 0),
                        datetime(2018, 1, 1, 12, 5, 0),
                        datetime(2018, 1, 1, 12, 6, 0),
                        datetime(2018, 1, 1, 12, 7, 0),
                        datetime(2018, 1, 1, 12, 8, 0),
                        datetime(2018, 1, 1, 12, 9, 0)))
    test_bins = array((1, 2, 3, 4, 5))
    test_histogram = array([[1, 2, 3, 4, 5],
                            [1, 2, 3, 4, 5],
                            [1, 2, 3, 4, 5],
                            [1, 2, 3, 4, 5],
                            [1, 2, 3, 4, 5],
                            [1, 2, 3, 4, 5],
                            [1, 2, 3, 4, 5],
                            [1, 2, 3, 4, 5],
                            [1, 2, 3, 4, 5],
                            [1, 2, 3, 4, 5]])
    plotter = Plotter('test plot', timestamps=test_times, bins=test_bins,
                      histogram=test_histogram)
    assert plotter._plotfunc == plotter._hist_plot
    assert (plotter._timestamps == test_times).all()
    assert (plotter._bins == test_bins).all()
    assert (plotter._histogram == test_histogram).all()
    assert plotter._type == 'test plot'

def test_invalid_construction():
    """Test that instantiation with incorrect arguments raise an exception."""
    with pytest.raises(InvalidArgumentsException):
        plotter = Plotter('test plot', foo=1, bar=1)

max_stars_repo_name: aniloutdo/Fitness-Gadgets

reason: from numpy


text:
import starepandas
import shapely
import numpy
import glob
import re

# Dask helpers
def slam(client, action, data, partition_factor=1.5, dbg=0):
    np = sum(client.nthreads().values())
    if dbg > 0:
        print('slam: np = %i' % np)
    shard_bounds = [int(i * len(data) / (1.0 * partition_factor * np)) for i in range(int(partition_factor * np))]
    if shard_bounds[-1] != len(data):
        if dbg > 0:
            print('a sb[-1]: ', shard_bounds[-1], len(data))
        shard_bounds = shard_bounds + [len(data)]
    if dbg > 0:
        print('sb: ', shard_bounds)
    data_shards = [data[shard_bounds[i]:shard_bounds[i + 1]] for i in range(len(shard_bounds) - 1)]
    if dbg > 0:
        print('ds len:        ', len(data_shards))
        print('ds item len:   ', len(data_shards[0]))
        print('ds type:       ', type(data_shards[0]))
        try:
            print('ds dtype:      ', data_shards[0].dtype)
        except:
            pass
    big_future = client.scatter(data_shards)
    results = client.map(action, big_future)
    return results

def make_row(granule_path, add_sf=False):
    granule = starepandas.io.granules.granule_factory(granule_path)

    granule.sidecar_path = granule.guess_sidecar_path()
    if not granule.sidecar_path:
        print('no sidecar found for {}'.format(granule_path))
        return None
    granule.read_sidecar_cover()

    row = {}
    row['granule_path'] = granule_path
    row['sidecar_path'] = granule.sidecar_path
    row['stare_cover'] = granule.stare_cover

    granule.read_timestamps()
    row['begining'] = granule.ts_start
    row['ending'] = granule.ts_end

    if add_sf:
        row['geom'] = get_sf_cover(granule_path)
    return row

def get_sf_cover(granule_path):
    hdf = starepandas.io.s3.sd_wrapper(granule_path)
    lon1 = hdf.select('Longitude').get()[0:-1, 0].astype(numpy.double)
    lat1 = hdf.select('Latitude').get()[0:-1, 0].astype(numpy.double)

    lon2 = hdf.select('Longitude').get()[-1, 0:-1].astype(numpy.double)
    lat2 = hdf.select('Latitude').get()[-1, 0:-1].astype(numpy.double)

    lon3 = hdf.select('Longitude').get()[0:-1, -1].astype(numpy.double)
    lat3 = hdf.select('Latitude').get()[0:-1, -1].astype(numpy.double)
    lon3 = numpy.flip(lon3)
    lat3 = numpy.flip(lat3)

    lon4 = hdf.select('Longitude').get()[0, 0:-1].astype(numpy.double)
    lat4 = hdf.select('Latitude').get()[0, 0:-1].astype(numpy.double)
    lon4 = numpy.flip(lon4)
    lat4 = numpy.flip(lat4)

    lon = numpy.concatenate((lon1, lon2, lon3, lon4))
    lat = numpy.concatenate((lat1, lat2, lat3, lat4))
    return shapely.geometry.Polygon(zip(lon, lat))

def folder2catalog(path, granule_trunk='', granule_extension='*', add_sf=False, client=None):
    """ Reads a folder of granules into a STAREDataFrame catalog

    :param path: Path of the folder containing granules
    :type path: str
    :param granule_trunk: Granule identifier (e.g. MOD09)
    :type granule_trunk: str
    :param granule_extension: Extension of the granule (e.g. hdf, nc, HDF5)
    :type granule_extension: str
    :param add_sf: toggle creating simple feature representation of the iFOVs
    :type add_sf: bool
    :param client:
    :type client:
    :return: catalog
    :rtype: starepandas.STAREDataFrame
    """
    term = '{path}/{granule_trunk}*.{ext}'.format(path=path, granule_trunk=granule_trunk, ext=granule_extension)
    s3 = None
    if path[0:5] != 's3://':
        granule_paths = glob.glob(term)
    else:
        granule_paths, s3 = starepandas.io.s3.s3_glob(path, '.*\.{ext}$'.format(ext=granule_extension))
    if not granule_paths:
        print('no granules in folder')
        return None

    pattern = '.*[^_stare]\.(nc|hdf|HDF5)'
    granule_paths = list(filter(re.compile(pattern).match, granule_paths))

    df = starepandas.STAREDataFrame()
    if client is None:
        for granule_path in granule_paths:
            if s3 is not None:
                granule_url = 's3://{bucket_name}/{granule}'.format(bucket_name=s3[0]['bucket_name'],
                                                                    granule=granule_path)
            else:
                granule_url = granule_path
            row = make_row(granule_url, add_sf)
            df = df.append(row, ignore_index=True)
    else:
        pass
        # client=Client()
        # client.close()
    df.set_sids('stare_cover', inplace=True)
    if add_sf:
        df.set_geometry('geom', inplace=True)
    return df

max_stars_repo_name: SpatioTemporal/STAREPandas

max_issues_repo_name: SpatioTemporal/STAREPandas

max_forks_repo_name: SpatioTemporal/STAREPandas