text:
# coding=UTF-8
import torch
import torchvision
import numpy as np
import torch.nn as nn
import torch.nn.functional as F
from math import *
import os
import numpy as np
import torch
import argparse
import torch.nn as nn
import torch.nn.functional as F

class Flatten(nn.Module):
    def forward(self, input):
        return input.view(input.size(0), -1)

class Block(nn.Module):
    def __init__(self, n_input, n_out):
        super(Block, self).__init__()

        self.network = nn.Sequential(nn.Conv2d(in_channels=n_input, out_channels=n_input // 2, kernel_size=1),
                                     nn.Conv2d(in_channels=n_input // 2, out_channels=int(3 * n_input / 4),
                                               kernel_size=1),
                                     nn.Conv2d(in_channels=int(3 * n_input / 4), out_channels=int(3 * n_input / 4),
                                               kernel_size=3, groups=int(3 * n_input / 4), padding=1),
                                     nn.Conv2d(in_channels=int(3 * n_input / 4), out_channels=n_input // 2,
                                               kernel_size=1),
                                     nn.Conv2d(in_channels=n_input // 2, out_channels=n_out, kernel_size=1))

    def forward(self, x):
        return self.network(x)

class CovidNet(nn.Module):
    def __init__(self, bnd=False, bna=False, label_smoothing=False,
                 n_classes=2, hidden_size=1024, emmbedding_size=128):
        super(CovidNet, self).__init__()
        filters = {
            'pexp1_1': [64, 256],
            'pexp1_2': [256, 256],
            'pexp1_3': [256, 256],
            'pexp2_1': [256, 512],
            'pexp2_2': [512, 512],
            'pexp2_3': [512, 512],
            'pexp2_4': [512, 512],
            'pexp3_1': [512, 1024],
            'pexp3_2': [1024, 1024],
            'pexp3_3': [1024, 1024],
            'pexp3_4': [1024, 1024],
            'pexp3_5': [1024, 1024],
            'pexp3_6': [1024, 1024],
            'pexp4_1': [1024, 2048],
            'pexp4_2': [2048, 2048],
            'pexp4_3': [2048, 2048],
        }

        self.label_smoothing = label_smoothing
        self.bnd = bnd
        self.bna = bna
        if bnd:
            self.add_module('conv1', nn.Conv2d(in_channels=3, out_channels=64, kernel_size=7, stride=2, padding=3))
            self.add_module('conv1_1x1', nn.Conv2d(in_channels=64, out_channels=256, kernel_size=1))
            self.add_module('conv2_1x1', nn.Conv2d(in_channels=256, out_channels=512, kernel_size=1))
            self.add_module('conv3_1x1', nn.Conv2d(in_channels=512, out_channels=1024, kernel_size=1))
            self.add_module('conv4_1x1', nn.Conv2d(in_channels=1024, out_channels=2048, kernel_size=1))

            # bn for ucsd data
            self.add_module('bn0', nn.Sequential(nn.BatchNorm2d(64), nn.ReLU()))
            self.add_module('bn1', nn.Sequential(nn.BatchNorm2d(256), nn.ReLU()))
            self.add_module('bn2', nn.Sequential(nn.BatchNorm2d(512), nn.ReLU()))
            self.add_module('bn3', nn.Sequential(nn.BatchNorm2d(1024), nn.ReLU()))
            self.add_module('bn4', nn.Sequential(nn.BatchNorm2d(2048), nn.ReLU()))
            # bn for new data
            self.add_module('bn0new', nn.Sequential(nn.BatchNorm2d(64), nn.ReLU()))
            self.add_module('bn1new', nn.Sequential(nn.BatchNorm2d(256), nn.ReLU()))
            self.add_module('bn2new', nn.Sequential(nn.BatchNorm2d(512), nn.ReLU()))
            self.add_module('bn3new', nn.Sequential(nn.BatchNorm2d(1024), nn.ReLU()))
            self.add_module('bn4new', nn.Sequential(nn.BatchNorm2d(2048), nn.ReLU()))
        else:
            self.add_module('conv1', nn.Conv2d(in_channels=3, out_channels=64, kernel_size=7, stride=2, padding=3))
            self.add_module('conv1_1x1', nn.Conv2d(in_channels=64, out_channels=256, kernel_size=1))
            self.add_module('conv2_1x1', nn.Conv2d(in_channels=256, out_channels=512, kernel_size=1))
            self.add_module('conv3_1x1', nn.Conv2d(in_channels=512, out_channels=1024, kernel_size=1))
            self.add_module('conv4_1x1', nn.Conv2d(in_channels=1024, out_channels=2048, kernel_size=1))

        for key in filters.keys():
            self.add_module(key, Block(filters[key][0], filters[key][1]))

        self.add_module('avg', nn.AdaptiveAvgPool2d(output_size=(2, 2)))

        self.add_module('flat', Flatten())

        self.add_module('fc1', nn.Linear(2048 * 2 * 2, 1024))
        self.add_module('fc2', nn.Linear(1024, 256))

        # bn
        if bna:
            # for ucsd
            self.add_module('bn5', nn.Sequential(nn.BatchNorm1d(1024), nn.ReLU()))
            self.add_module('bn6', nn.Sequential(nn.BatchNorm1d(256), nn.ReLU()))
            # for new
            self.add_module('bn5new', nn.Sequential(nn.BatchNorm1d(1024), nn.ReLU()))
            self.add_module('bn6new', nn.Sequential(nn.BatchNorm1d(256), nn.ReLU()))

        # contrastive feature
        self.hidden_size = hidden_size
        self.emmbedding_size = emmbedding_size
        self.add_module('feature1', nn.Linear(2048 * 2 * 2, self.hidden_size))
        self.add_module('feature2', nn.Linear(self.hidden_size, self.emmbedding_size))

        # bn
        if bna:
            # for ucsd
            self.add_module('fbn5', nn.Sequential(nn.BatchNorm1d(self.hidden_size), nn.ReLU()))
            self.add_module('fbn6', nn.Sequential(nn.BatchNorm1d(self.emmbedding_size), nn.ReLU()))
            # for new
            self.add_module('fbn5new', nn.Sequential(nn.BatchNorm1d(self.hidden_size), nn.ReLU()))
            self.add_module('fbn6new', nn.Sequential(nn.BatchNorm1d(self.emmbedding_size), nn.ReLU()))

        classifier_list = [
            nn.Linear(256, n_classes),
            nn.Sigmoid()
        ]
        self.classifier = nn.Sequential(*classifier_list)

    def forward(self, x, site):
        conv1 = self.conv1(x)

        if self.bnd:
            if site == 'ucsd':
                conv1 = self.bn0(conv1)
            else:
                conv1 = self.bn0new(conv1)
        x = F.max_pool2d(conv1, 2)

        out_conv1_1x1 = self.conv1_1x1(x)

        if self.bnd:
            if site == 'ucsd':
                out_conv1_1x1 = self.bn1(out_conv1_1x1)
            else:
                out_conv1_1x1 = self.bn1new(out_conv1_1x1)

        pepx11 = self.pexp1_1(x)
        pepx12 = self.pexp1_2(pepx11 + out_conv1_1x1)
        pepx13 = self.pexp1_3(pepx12 + pepx11 + out_conv1_1x1)

        conv2 = self.conv2_1x1(pepx12 + pepx11 + pepx13 + out_conv1_1x1)

        if self.bnd:
            if site == 'ucsd':
                conv2 = self.bn2(conv2)
            else:
                conv2 = self.bn2new(conv2)
        out_conv2_1x1 = F.max_pool2d(conv2, 2)

        pepx21 = self.pexp2_1(
            F.max_pool2d(pepx13, 2) + F.max_pool2d(pepx11, 2) + F.max_pool2d(pepx12, 2) + F.max_pool2d(out_conv1_1x1,
                                                                                2))
        pepx22 = self.pexp2_2(pepx21 + out_conv2_1x1)
        pepx23 = self.pexp2_3(pepx22 + pepx21 + out_conv2_1x1)
        pepx24 = self.pexp2_4(pepx23 + pepx21 + pepx22 + out_conv2_1x1)

        conv3 = self.conv3_1x1(pepx22 + pepx21 + pepx23 + pepx24 + out_conv2_1x1)

        if self.bnd:
            if site == 'ucsd':
                conv3 = self.bn3(conv3)
            else:
                conv3 = self.bn3new(conv3)
        out_conv3_1x1 = F.max_pool2d(conv3, 2)

        pepx31 = self.pexp3_1(
            F.max_pool2d(pepx24, 2) + F.max_pool2d(pepx21, 2) + F.max_pool2d(pepx22, 2) + F.max_pool2d(pepx23,
                                                                                2) + F.max_pool2d(
                out_conv2_1x1, 2))
        pepx32 = self.pexp3_2(pepx31 + out_conv3_1x1)
        pepx33 = self.pexp3_3(pepx31 + pepx32 + out_conv3_1x1)
        pepx34 = self.pexp3_4(pepx31 + pepx32 + pepx33 + out_conv3_1x1)
        pepx35 = self.pexp3_5(pepx31 + pepx32 + pepx33 + pepx34 + out_conv3_1x1)
        pepx36 = self.pexp3_6(pepx31 + pepx32 + pepx33 + pepx34 + pepx35 + out_conv3_1x1)

        conv4 = self.conv4_1x1(pepx31 + pepx32 + pepx33 + pepx34 + pepx35 + pepx36 + out_conv3_1x1)

        if self.bnd:
            if site == 'ucsd':
                conv4 = self.bn4(conv4)
            else:
                conv4 = self.bn4new(conv4)
        out_conv4_1x1 = F.max_pool2d(conv4, 2)

        pepx41 = self.pexp4_1(
            F.max_pool2d(pepx31, 2) + F.max_pool2d(pepx32, 2) + F.max_pool2d(pepx32, 2) + F.max_pool2d(pepx34,
                                                                                2) + F.max_pool2d(
                pepx35, 2) + F.max_pool2d(pepx36, 2) + F.max_pool2d(out_conv3_1x1, 2))
        pepx42 = self.pexp4_2(pepx41 + out_conv4_1x1)
        pepx43 = self.pexp4_3(pepx41 + pepx42 + out_conv4_1x1)

        avg = self.avg(pepx43)

        flat = self.flat(avg)

        fc1 = self.fc1(flat)

        if self.bna:
            if site == 'ucsd':
                fc1 = self.bn5(fc1)
            else:
                fc1 = self.bn5new(fc1)
        fc2 = self.fc2(fc1)

        if self.bna:
            if site == 'ucsd':
                fc2 = self.bn6(fc2)
            else:
                fc2 = self.bn6new(fc2)

        logits = self.classifier(fc2)

        # contrastive loss
        feature1 = self.feature1(flat)
        if self.bna:
            if site == 'ucsd':
                feature1 = self.fbn5(feature1)
            else:
                feature1 = self.fbn5new(feature1)
        feature2 = self.feature2(feature1)
        if self.bna:
            if site == 'ucsd':
                feature2 = self.fbn6(feature2)
            else:
                feature2 = self.fbn6new(feature2)

        return logits, feature2


text:
import cv2
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F

from registry import Registries
from utils import norm

from .base_score import BaseScore

@Registries.score.register("smd2")
class SMD2Score(BaseScore):
    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        self.smd2_conv = SMD2_Conv2d()

    def _score_for_one_group(self, frame_group)->list:
        scores = []
        for frame in frame_group:
            frame = torch.Tensor(frame/255.0)
            score = self.smd2_conv(frame)
            scores.append(float(score))
        return norm(scores)

    def score_frame(self,
                    video_cap: cv2.VideoCapture,
                    group_size: int=2,
                    transforms=None,
                    **kwargs) -> list:
        assert group_size > 1
        scores = []
        frame_count = int(video_cap.get(cv2.CAP_PROP_FRAME_COUNT))
        for i_frame in range(frame_count):
            success, frame = video_cap.read()
            if transforms is not None:
                frame = transforms(frame)
            if i_frame == 0:
                group = []
            elif i_frame%group_size == 0:
                ssim_scores = self._score_for_one_group(group)
                scores.extend(ssim_scores)
                group = []
            group.append(frame)
        datas = [{"index":i_frame,
                  "score":scores[i_frame]} for i_frame in range(len(scores))]
        datas = self._sort_score(datas)
        return datas

"""
    calculate the smd according to its definition
    very slow
"""
def smd2_defination(img, **kwargs):
    f = np.matrix(img) / 255.0  # 返回矩阵
    x, y = f.shape
    smd2 = 0
    for i in range(x - 1):
        for j in range(y - 1):
            smd2 += np.abs(f[i, j]-f[i+1, j])*np.abs(f[i, j]-f[i, j+1])
    return smd2

"""
    Calculate smd2 with convolution
    quicker than the defination one more than 10 times
"""
class SMD2_Conv2d(nn.Module):
    def __init__(self):
        super().__init__()
        kernel_x = np.array([[[[1.0, -1.0], # shape: (1, 1, 2, 2)
                            [0.0, 0.0]]]])
        kernel_y = np.array([[[[1.0, 0.0],
                            [-1.0, 0.0]]]])
        self.kernel_x = nn.Parameter(torch.tensor(kernel_x, dtype=torch.float))
        self.kernel_y = nn.Parameter(torch.tensor(kernel_y, dtype=torch.float))
        self.bias = nn.Parameter(torch.zeros((1)))

    def forward(self, x):
        x = x.view(-1, 1, *x.shape)
        out_x = F.conv2d(x, self.kernel_x, self.bias)
        out_y = F.conv2d(x, self.kernel_x, self.bias)
        out_x = torch.abs(out_x)
        out_y = torch.abs(out_y)
        # out = torch.mul(out_x, out_y)
        out = out_x * out_y
        return torch.sum(out).detach().cpu()

@torch.no_grad()
def smd2_conv(gray, CNN: nn.Module, device: torch.device):
    gray = cv2.resize(gray, SET_IMG_SHAPE, interpolation=cv2.INTER_CUBIC)
    gray = gray / 255.0
    gray = torch.tensor(gray, dtype=torch.float32)
    gray = gray.to(device)
    smd2 = CNN(gray)
    return smd2

def smd2(img, **kwargs):
    score = smd2_conv(img, **kwargs)
    return float(score)

ext: py

max_stars_repo_name: JasonZuu/Frame-Selection

max_issues_repo_name: JasonZuu/Frame-Selection

max_forks_repo_name: JasonZuu/Frame-Selection

reason: import numpy


text:
#!/usr/bin/python

import sys
import pandas as pd
import numpy as np
import re
import string
import numpy as np
import matplotlib.pyplot as plt
from pylab import rcParams
import json

def main():
    # print command line arguments
    #for arg in sys.argv[1:]:
    #print sys.argv[1]

    #if len(sys.argv) < 4:
    #    print('usage: python token_generator.py <number of files> <file base name>'

    entries = dict()
    total_files = sys.argv[1]
    filename = sys.argv[3]

    for i in range(total_files):
        with open(filename + '-diffs-' + str(i) + '.txt') as f:
            count = 0
            is_rev = False
            is_msg = False
            is_diff = False
            msg_tokens = list()
            diff_tokens = list()

            revision = ''

            for line in f:
                #if count > 5:
                 #   break

                if line == '\n':
                    continue

                if 'revi####commit##gen####vt####se###spring####2018' in line:

                    if is_diff == True:
                        entries[count] = (revision, len(msg_tokens), len(diff_tokens))
                        #print(revision, len(msg_tokens), len(diff_tokens))

                        msg_tokens = list()
                        diff_tokens = list()
                        revision = ''

                        count += 1

                    is_rev = True
                    is_msg = False
                    is_diff = False

                    continue

                if 'comm####commit##gen####vt####se###spring####2018' in line:
                    is_rev = False
                    is_msg = True
                    is_diff = False
                    continue

                if 'diff####commit##gen####vt####se###spring####2018' in line:
                    is_rev = False
                    is_msg = False
                    is_diff = True
                    continue

                if is_rev == True:
                    revision = line.strip()
                elif is_msg == True:
                    msg_tokens.extend(re.findall(r"[^,.:;' ]+|[,.:;']", line.strip()))
                else:
                    diff_tokens.extend(re.findall(r"[^,.:;' ]+|[,.:;']", line.strip()))

    with open(filename + '-tokens.txt', 'w') as file:
        file.write(json.dumps(entries)) # use `json.loads` to do the reverse

if __name__ == "__main__":
    main()

max_stars_repo_name: salmanyam/CommitGen

max_issues_repo_name: salmanyam/CommitGen

max_forks_repo_name: salmanyam/CommitGen


text:
import os
import gym
import torch
import torch.nn.functional as F
import torch.distributed as dist
from torch.autograd import Variable
import numpy as np

# https://github.com/ikostrikov/pytorch-ddpg-naf/blob/master/ddpg.py#L11
def soft_update(target, source, tau):
    """
    Perform DDPG soft update (move target params toward source based on weight
    factor tau)
    Inputs:
        target (torch.nn.Module): Net to copy parameters to
        source (torch.nn.Module): Net whose parameters to copy
        tau (float, 0 < x < 1): Weight factor for update
    """
    for target_param, param in zip(target.parameters(), source.parameters()):
        target_param.data.copy_(target_param.data * (1.0 - tau) + param.data * tau)

# https://github.com/ikostrikov/pytorch-ddpg-naf/blob/master/ddpg.py#L15
def hard_update(target, source):
    """
    Copy network parameters from source to target
    Inputs:
        target (torch.nn.Module): Net to copy parameters to
        source (torch.nn.Module): Net whose parameters to copy
    """
    for target_param, param in zip(target.parameters(), source.parameters()):
        target_param.data.copy_(param.data)

# https://github.com/seba-1511/dist_tuto.pth/blob/gh-pages/train_dist.py
def average_gradients(model):
    """ Gradient averaging. """
    size = float(dist.get_world_size())
    for param in model.parameters():
        dist.all_reduce(param.grad.data, op=dist.reduce_op.SUM, group=0)
        param.grad.data /= size

# https://github.com/seba-1511/dist_tuto.pth/blob/gh-pages/train_dist.py
def init_processes(rank, size, fn, backend='gloo'):
    """ Initialize the distributed environment. """
    os.environ['MASTER_ADDR'] = '127.0.0.1'
    os.environ['MASTER_PORT'] = '29500'
    dist.init_process_group(backend, rank=rank, world_size=size)
    fn(rank, size)

def onehot_from_logits(logits, eps=0.0):
    """
    Given batch of logits, return one-hot sample using epsilon greedy strategy
    (based on given epsilon)
    """
    # get best (according to current policy) actions in one-hot form
    argmax_acs = (logits == logits.max(1, keepdim=True)[0]).float()
    if eps == 0.0:
        return argmax_acs
    # get random actions in one-hot form
    rand_acs = Variable(torch.eye(logits.shape[1])[[np.random.choice(
        range(logits.shape[1]), size=logits.shape[0])]], requires_grad=False)
    # chooses between best and random actions using epsilon greedy
    return torch.stack([argmax_acs[i] if r > eps else rand_acs[i] for i, r in
                        enumerate(torch.rand(logits.shape[0]))])

# modified for PyTorch from https://github.com/ericjang/gumbel-softmax/blob/master/Categorical%20VAE.ipynb
def sample_gumbel(shape, eps=1e-20, tens_type=torch.FloatTensor):
    """Sample from Gumbel(0, 1)"""
    U = Variable(tens_type(*shape).uniform_(), requires_grad=False)
    return -torch.log(-torch.log(U + eps) + eps)

# modified for PyTorch from https://github.com/ericjang/gumbel-softmax/blob/master/Categorical%20VAE.ipynb
def gumbel_softmax_sample(logits, temperature):
    """ Draw a sample from the Gumbel-Softmax distribution"""
    y = logits + sample_gumbel(logits.shape, tens_type=type(logits.data))
    return F.softmax(y / temperature, dim=1)

# modified for PyTorch from https://github.com/ericjang/gumbel-softmax/blob/master/Categorical%20VAE.ipynb
def gumbel_softmax(logits, temperature=1.0, hard=False):
    """Sample from the Gumbel-Softmax distribution and optionally discretize.
    Args:
      logits: [batch_size, n_class] unnormalized log-probs
      temperature: non-negative scalar
      hard: if True, take argmax, but differentiate w.r.t. soft sample y
    Returns:
      [batch_size, n_class] sample from the Gumbel-Softmax distribution.
      If hard=True, then the returned sample will be one-hot, otherwise it will
      be a probabilitiy distribution that sums to 1 across classes
    """
    y = gumbel_softmax_sample(logits, temperature)
    if hard:
        y_hard = onehot_from_logits(y)
        y = (y_hard - y).detach() + y
    return y

def discount(x, gamma, axis=0):
    """ discount on rewards/values sequence, axis is time dimension in x
        reference: https://stackoverflow.com/questions/47970683/vectorize-a-numpy-discount-calculation
        reference2: https://docs.scipy.org/doc/scipy/reference/generated/scipy.signal.lfilter.html
    """
    y = scipy.signal.lfilter([1], [1, -gamma], x[::-1], axis=axis)
    return y[::-1]

def compute_advantages(
        last_r,
        rewards,
        vf_preds,
        gamma=0.9,
        lambda_=1.0,
        use_gae=True,
        use_critic=True,
        norm_advantages=False
    ):
    """ normal advantage or generalized advantage estiamate
        reference: https://arxiv.org/pdf/1506.02438.pdf
        reference2: https://ray.readthedocs.io/en/latest/_modules/ray/rllib/evaluation/postprocessing.html
    - TODO: need to add time limit and termination masks (following)
        reference3: https://github.com/ikostrikov/pytorch-a2c-ppo-acktr-gail/blob/master/a2c_ppo_acktr/storage.py
    Arguments:
        - last_r: (B,1)
        - rewards: (B,T,1)
        - vf_preds: (B,T,1)
    Returns:
        - advantages: (B,T,1)
        - value_targets: (B,T,1)
    """
    if use_gae:
        # use generalized advantage estimator
        vpred_t = torch.cat([vf_preds.squeeze(), last_r], -1) # (B,T+1)
        delta_t = rewards.squeeze() + gamma * vpred_t[:,1:] - vpred_t[:,:-1] # (B,T)

        advantages = discount(delta_t.numpy(), gamma * lambda_, -1) # (B,T)
        advantages = torch.from_numpy(advantages).unsqueeze(-1) # (B,T,1)
        value_targets = advantages + vf_preds
    else:
        # use rewards + value prediction (last step)
        rewards_plus_v = torch.cat([rewards.squeeze(), last_r], -1) # (B,T+1)
        discounted_returns = discount(rewards_plus_v.numpy(), gamma, -1)[:-1] # (B,T)
        discounted_returns = torch.from_numpy(discounted_returns).unsqueeze(-1) # (B,T,1)

        if use_critic:
            advantages = discounted_returns - vf_preds  # (B,T,1)
            value_targets = discounted_returns
        else:
            advantages = discounted_returns
            value_targets = torch.zeros_like(advantages)

    # if to normalize advantages
    if norm_advantages:
        advantages = (advantages - advantages.mean()) / (advantages.std() + 1e-5)

    return advantages, value_targets

def explained_variance_torch(y, pred):
    y_var = torch.pow(y.std(0), 0.5)
    diff_var = torch.pow((y - pred).std(0), 0.5)
    var = max(-1.0, 1.0 - (diff_var / y_var).data)
    return var

max_stars_repo_name: Justin-Yuan/learn-to-interact

max_issues_repo_name: Justin-Yuan/learn-to-interact

max_forks_repo_name: Justin-Yuan/learn-to-interact


text:
# -*- coding: utf-8 -*-
import unittest
try:
    import torch
except ImportError:
    TORCH_AVAILABLE = False
else:
    TORCH_AVAILABLE = True
    from dival.util.torch_utility import (
        load_state_dict_convert_data_parallel, TOMOSIPO_AVAILABLE,
        TorchRayTrafoParallel2DModule, TorchRayTrafoParallel2DAdjointModule)
import numpy as np
from dival import get_standard_dataset
try:
    import astra
except ImportError:
    ASTRA_CUDA_AVAILABLE = False
else:
    ASTRA_CUDA_AVAILABLE = astra.use_cuda()

@unittest.skipUnless(
    TORCH_AVAILABLE and TOMOSIPO_AVAILABLE and ASTRA_CUDA_AVAILABLE,
    'PyTorch or tomosipo or ASTRA+CUDA not available')
class TestTorchRayTrafoParallel2DModule(unittest.TestCase):
    def test(self):
        dataset = get_standard_dataset('ellipses', fixed_seeds=True,
                                       impl='astra_cuda')
        ray_trafo = dataset.get_ray_trafo(impl='astra_cuda')
        module = TorchRayTrafoParallel2DModule(ray_trafo)
        for batch_size, channels in [(1, 3),
                                     (5, 1),
                                     (2, 3)]:
            test_data = dataset.get_data_pairs(part='train',
                                               n=batch_size * channels)
            torch_in = (torch.from_numpy(np.asarray(test_data.ground_truth))
                        .view(batch_size, channels, *dataset.shape[1]))
            torch_out = module(torch_in).view(-1, *dataset.shape[0])
            for i, odl_in in enumerate(test_data.ground_truth):
                odl_out = ray_trafo(odl_in)
                self.assertTrue(np.allclose(
                    torch_out[i].detach().cpu().numpy(), odl_out, rtol=1.e-2))

    def testGradient(self):
        dataset = get_standard_dataset('ellipses', fixed_seeds=True,
                                       impl='astra_cuda')
        ray_trafo = dataset.get_ray_trafo(impl='astra_cuda')
        module = TorchRayTrafoParallel2DModule(ray_trafo)
        batch_size, channels = 2, 3
        torch_in_ = torch.ones(
            (batch_size * channels,) + dataset.shape[1],
            requires_grad=True)
        torch_in = torch_in_.view(batch_size, channels, *dataset.shape[1])
        torch_out = module(torch_in).view(-1, *dataset.shape[0])
        for i in range(batch_size * channels):
            for j in range(0, dataset.shape[0][0], 3):  # angle
                for k in range(0, dataset.shape[0][1], 12):  # detector pos
                    odl_value_in = np.zeros(dataset.shape[0])
                    odl_value_in[j, k] = 1.
                    odl_grad = ray_trafo.adjoint(odl_value_in)
                    torch_in_.grad = None
                    torch_out[i, j, k].backward(retain_graph=True)
                    torch_grad_np = (
                        torch_in_.grad[i].detach().cpu().numpy())
                    # very rough check for maximum error
                    self.assertTrue(np.allclose(torch_grad_np, odl_grad,
                        rtol=1.))
                    non_zero = np.nonzero(np.asarray(odl_grad))
                    if np.any(odl_grad):  # there seem to be cases where
                                          # a pixel has no influence on the
                                          # gradient
                        # tighter check for mean error
                        self.assertLess(np.mean(np.abs(
                            torch_grad_np[non_zero] - odl_grad[non_zero])
                            / np.abs(odl_grad[non_zero])), 1e-3)

@unittest.skipUnless(
    TORCH_AVAILABLE and TOMOSIPO_AVAILABLE and ASTRA_CUDA_AVAILABLE,
    'PyTorch or tomosipo or ASTRA+CUDA not available')
class TestTorchRayTrafoParallel2DAdjointModule(unittest.TestCase):
    def test(self):
        dataset = get_standard_dataset('ellipses', fixed_seeds=True,
                                       impl='astra_cuda')
        ray_trafo = dataset.get_ray_trafo(impl='astra_cuda')
        module = TorchRayTrafoParallel2DAdjointModule(ray_trafo)
        for batch_size, channels in [(1, 3),
                                     (5, 1),
                                     (2, 3)]:
            test_data = dataset.get_data_pairs(part='train',
                                               n=batch_size * channels)
            torch_in = (torch.from_numpy(np.asarray(test_data.observations))
                        .view(batch_size, channels, *dataset.shape[0]))
            torch_out = module(torch_in).view(-1, *dataset.shape[1])
            for i, odl_in in enumerate(test_data.observations):
                odl_out = ray_trafo.adjoint(odl_in)
                self.assertTrue(np.allclose(
                    torch_out[i].detach().cpu().numpy(), odl_out, rtol=1.e-4))

    def testGradient(self):
        dataset = get_standard_dataset('ellipses', fixed_seeds=True,
                                       impl='astra_cuda')
        ray_trafo = dataset.get_ray_trafo(impl='astra_cuda')
        module = TorchRayTrafoParallel2DAdjointModule(ray_trafo)
        batch_size, channels = 2, 2
        torch_in_ = torch.ones(
            (batch_size * channels,) + dataset.shape[0],
            requires_grad=True)
        torch_in = torch_in_.view(batch_size, channels, *dataset.shape[0])
        torch_out = module(torch_in).view(-1, *dataset.shape[1])
        for i in range(batch_size * channels):
            for j in range(0, dataset.shape[1][0], 9):  # x
                for k in range(0, dataset.shape[1][1], 9):  # y
                    odl_value_in = np.zeros(dataset.shape[1])
                    odl_value_in[j, k] = 1.
                    odl_grad = ray_trafo.adjoint.adjoint(odl_value_in)
                    torch_in_.grad = None
                    torch_out[i, j, k].backward(retain_graph=True)
                    torch_grad_np = (
                        torch_in_.grad[i].detach().cpu().numpy())
                    # very rough check for maximum error
                    self.assertTrue(np.allclose(torch_grad_np, odl_grad,
                        rtol=1.))
                    non_zero = np.nonzero(np.asarray(odl_grad))
                    if np.any(odl_grad):  # there seem to be cases where
                                          # a pixel has no influence on the
                                          # gradient
                        # tighter check for mean error
                        self.assertLess(np.mean(np.abs(
                            torch_grad_np[non_zero] - odl_grad[non_zero])
                            / np.abs(odl_grad[non_zero])), 1e-2)

@unittest.skipUnless(
    TORCH_AVAILABLE and TOMOSIPO_AVAILABLE and ASTRA_CUDA_AVAILABLE,
    'PyTorch or tomosipo or ASTRA+CUDA not available')
class TestGetTorchRayTrafoParallel2d(unittest.TestCase):
    def test(self):
        pass

@unittest.skipUnless(
    TORCH_AVAILABLE and TOMOSIPO_AVAILABLE and ASTRA_CUDA_AVAILABLE,
    'PyTorch or tomosipo or ASTRA+CUDA not available')
class TestGetTorchRayTrafoParallel2dAdjoint(unittest.TestCase):
    def test(self):
        pass

@unittest.skipUnless(TORCH_AVAILABLE, 'PyTorch not available')
class TestLoadStateDictConvertDataParallel(unittest.TestCase):
    def setUp(self):
        self.model = torch.nn.Sequential(
            torch.nn.Conv2d(1, 5, 3),
            torch.nn.Conv2d(5, 1, 3))
        self.model_parallel = torch.nn.DataParallel(self.model)

    def test(self):
        state_dict = self.model.state_dict()
        state_dict_parallel = self.model_parallel.state_dict()
        try:
            load_state_dict_convert_data_parallel(self.model,
                                                  state_dict)
        except RuntimeError:
            self.fail()
        try:
            load_state_dict_convert_data_parallel(self.model,
                                                  state_dict_parallel)
        except RuntimeError:
            self.fail()
        try:
            load_state_dict_convert_data_parallel(self.model_parallel,
                                                  state_dict)
        except RuntimeError:
            self.fail()
        try:
            load_state_dict_convert_data_parallel(self.model_parallel,
                                                  state_dict_parallel)
        except RuntimeError:
            self.fail()
        state_dict_missing = {
            k: v for i, (k, v) in enumerate(state_dict.items()) if i >= 1}
        state_dict_parallel_missing = {
            k: v for i, (k, v) in enumerate(state_dict_parallel.items())
            if i >= 1}
        with self.assertRaises(RuntimeError):
            load_state_dict_convert_data_parallel(
                self.model, state_dict_missing)
        with self.assertRaises(RuntimeError):
            load_state_dict_convert_data_parallel(
                self.model, state_dict_parallel_missing)
        with self.assertRaises(RuntimeError):
            load_state_dict_convert_data_parallel(
                self.model_parallel, state_dict_missing)
        with self.assertRaises(RuntimeError):
            load_state_dict_convert_data_parallel(
                self.model_parallel, state_dict_parallel_missing)
        state_dict_unexpected = {
            '{}_unexpected'.format(k): v for k, v in state_dict.items()}
        state_dict_parallel_unexpected = {
            '{}_unexpected'.format(k): v for k, v in
            state_dict_parallel.items()}
        with self.assertRaises(RuntimeError):
            load_state_dict_convert_data_parallel(
                self.model, state_dict_unexpected)
        with self.assertRaises(RuntimeError):
            load_state_dict_convert_data_parallel(
                self.model, state_dict_parallel_unexpected)
        with self.assertRaises(RuntimeError):
            load_state_dict_convert_data_parallel(
                self.model_parallel, state_dict_unexpected)
        with self.assertRaises(RuntimeError):
            load_state_dict_convert_data_parallel(
                self.model_parallel, state_dict_parallel_unexpected)

if __name__ == '__main__':
    unittest.main()

max_stars_repo_name: MBaltz/dival

max_issues_repo_name: MBaltz/dival

max_forks_repo_name: MBaltz/dival


text:
#!/usr/bin/env python3
import numpy as np
import pickle
import json
import argparse
from typing import List, Tuple, Optional
from collections import namedtuple

DataSet = namedtuple(
    "DataSet",
    [
        "train_data",
        "valid_data",
        "test_data",
        "train_labels",
        "valid_labels",
        "test_labels",
    ],
)

def main():
    p = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    p.add_argument("--lr", type=float, help="learning rate", default=0.1)
    p.add_argument("--beta-min", type=float, help="min beta", default=0.1)
    p.add_argument("--beta-max", type=float, help="max beta", default=0.1)
    p.add_argument(
        "--beta-num",
        type=int,
        help="number of betas to test (linear between min and max)",
        default=1,
    )
    p.add_argument("--num-chains", type=int, help="number of chains", default=10)
    p.add_argument(
        "--num-indices", type=int, help="number of indices to select", default=20
    )
    p.add_argument(
        "--num-inner-steps", type=int, help="number of steps per chain", default=50
    )
    p.add_argument(
        "--lr-div-factor",
        type=float,
        default=4,
        help="fisher chain error is (lr/lr_div_factor) * np.linalg.norm(grad - v_grad)**2)",
    )
    p.add_argument(
        "--null",
        action="store_true",
        default=False,
        help="whether there should be 2 true predictors included in the dataset",
    )
    p.add_argument(
        "--output-results",
        default="freedman-results.jsonl",
        help="Path to save results",
    )
    args = p.parse_args()

    indices_set = []

    #
    # create_dataset will dump dataset generated
    # for experiment to freedman-data-set.pkl
    # in the directory that this script is called from
    #
    data_set = create_dataset(args.null)
    trial_betas = np.linspace(args.beta_min, args.beta_max, num=args.beta_num)

    with open(args.output_results, "w") as fp:
        json_dump = (
            json.dumps(
                {
                    "lr": args.lr,
                    "lr_div_factor": args.lr_div_factor,
                    "trial_betas": trial_betas.tolist(),
                    "num_chains": args.num_chains,
                    "num_inner_steps": args.num_inner_steps,
                },
                sort_keys=True,
            )
            + "\n"
        )
        fp.write("# " + json_dump)

    for i in range(args.num_indices):
        print("Forward selection iteration %d..." % i)
        indices_set = forward_selection(
            current_indices_set=indices_set,
            data_set=data_set,
            lr=args.lr,
            lr_div_factor=args.lr_div_factor,
            betas=trial_betas,
            num_chains=args.num_chains,
            num_inner_steps=args.num_inner_steps,
            output_results=args.output_results,
        )

def create_dataset(null: bool = False) -> DataSet:
    """Create the Freedman's paradox dataset

    There are 500 points and 500 features. If null == True
    the features are all normally distributed random values,
    as are the labels. If null == False, the first two features
    are valid predictors and the others features are as before.

    Dumps dataset to freedman-data-set.pkl which is then used by
    freedman_naive.py to do feature selection on an identical data
    set without regularization.
    """
    random = np.random.RandomState(0)
    num_points = 500
    num_preds = 500
    split = 250
    data = random.normal(0, 1, (num_points, num_preds))
    if null:
        labels = random.normal(0, 1, (num_points,))
    else:
        labels = data[:, 0] + data[:, 1] + random.normal(0, 2, (num_points,))
        labels = labels / np.sqrt(6)

    train_data = data[0:split, :]
    train_labels = labels[0:split]

    valid_data = data[split:, :]
    valid_labels = labels[split:]

    test_data = random.normal(0, 1, (num_points, num_preds))
    test_labels = test_data[:, 0] + test_data[:, 1] + random.normal(0, 2, (num_points,))
    test_labels = test_labels / np.sqrt(6)

    data_set = DataSet(
        train_data, valid_data, test_data, train_labels, valid_labels, test_labels
    )

    with open("freedman-data-set.pkl", "wb") as f:
        pickle.dump(data_set, f, protocol=pickle.HIGHEST_PROTOCOL)
    return data_set

def forward_selection(
    current_indices_set: List[int],
    data_set: DataSet,
    lr: float,
    lr_div_factor: float,
    betas: List[float],
    num_chains: int,
    num_inner_steps: int,
    output_results: str,
) -> List[int]:
    """Perform one step of forward selection to determine the additional feature
    to add to the model. This function screens each of the remaining features
    not currently included in current_indices_set and greedily selects one additional
    feature.

    Args:
        current_indices_set (List[int]) the features currently included in the model
        data_set (DataSet) training and validation data
        lr (float) learning rate
        lr_div_factor (float) scaling factor that determines the regularization penalty
        beta (float) parameter of the sampled Gibbs posterior
        num_chains (int) the number of chains to run
        num_inner_steps (int) the number of steps of inner optimization to run

    Returns:
        new_indices_set (List[int]) the new set of included features, equal to current_indices_set
            plus one new one
    """
    top_objective = float("inf")
    top_index_set: Optional[List[int]] = None
    top_error: Optional[float] = None
    top_r2: Optional[float] = None
    top_beta: Optional[float] = None
    top_objective_list: List[float] = []

    remaining_indices = [
        i for i in range(data_set.train_data.shape[1]) if i not in current_indices_set
    ]

    for beta in betas:
        for index in remaining_indices:
            trial_index_set = current_indices_set + [index]
            objective, error, r2, objective_list = run_chains(
                lr=lr,
                lr_div_factor=lr_div_factor,
                beta=beta,
                data_set=data_set,
                index_set=trial_index_set,
                num_chains=num_chains,
                num_inner_steps=num_inner_steps,
            )
            if objective < top_objective:
                top_objective = objective
                top_index_set = trial_index_set
                top_error = error
                top_r2 = r2
                top_beta = beta
                top_objective_list = objective_list
    assert top_index_set is not None
    assert top_error is not None
    assert top_r2 is not None
    assert top_beta is not None

    coefs, _resids, _rank, _S = np.linalg.lstsq(
        data_set.train_data[:, top_index_set], data_set.train_labels, rcond=None
    )
    true_r2 = get_r2(coefs, data_set.test_data[:, top_index_set], data_set.test_labels)
    valid_r2 = get_r2(
        coefs, data_set.valid_data[:, top_index_set], data_set.valid_labels
    )

    bound_r2 = 1 - (len(data_set.valid_labels) * top_objective) / (
        len(data_set.valid_labels) * np.var(data_set.valid_labels)
    )

    print("Forward selection iteration complete")
    print(f"Top Objective: {top_objective}")
    print(f"Top Index Set: {top_index_set}")
    print(f"Top Error: {top_error}")
    print(f"Beta {top_beta}")
    print(f"Langevin R-squared: {top_r2}")
    print(f"Valid R-squared: {valid_r2}")
    print(f"Bound R-squared: {bound_r2}")
    print(f"True R-squared: {true_r2}")
    print()

    with open(output_results, "a") as fp:
        fp.write(
            json.dumps(
                {
                    "num_selected": len(top_index_set),
                    "top_r2": top_r2,
                    "top_objective": top_objective,
                    "top_error": top_error,
                    "top_index_set": top_index_set,
                    "top_beta": top_beta,
                    "top_objective_list": top_objective_list,
                },
                sort_keys=True,
            )
            + "\n"
        )

    return top_index_set

def run_chains(
    lr: float,
    lr_div_factor: float,
    beta: float,
    data_set: DataSet,
    index_set: List[int],
    num_chains: int,
    num_inner_steps: int,
) -> Tuple[float, float, float, List[float]]:
    """Runs multiple chains of optimization to estimate the objective
    derived in Eq. 7. This function runs Langevin Dynamics specified by the
    input parameters to fit the model whose features are determined by
    'index_set' to the training data set.

    Args:
        lr (float) learning rate
        lr_div_factor (float) scaling factor that determines the regularization penalty
        beta (float) parameter of the sampled Gibbs posterior
        data_set (DataSet) training and validation data
        index_set (List[int]) the to include in the model
        num_chains (int) the number of chains to run
        num_inner_steps (int) the number of steps of inner optimization to run

    Returns:
        objective (float) average objective over the chains
        error (float) mean squared error averaged over the chains
        r2 (float) average r^2 over the chains
        chain_objectives (List[float]) objective of each individual chain
    """

    random = np.random.RandomState(0)

    X = data_set.train_data[:, index_set]
    vX = data_set.valid_data[:, index_set]

    chain_objective = 0.0
    chains = [random.normal(size=(len(index_set), 1)) for i in range(num_chains)]
    chain_objectives = [0.0 for i in range(num_chains)]
    for _step in range(num_inner_steps):
        fisher_chains = []
        for chain in range(num_chains):
            grad = gradient(chains[chain], X, data_set.train_labels)
            v_grad = gradient(chains[chain], vX, data_set.valid_labels)
            noise = random.normal(
                scale=np.sqrt(2 * lr / (beta * X.shape[0])), size=grad.shape
            )
            chains[chain] = chains[chain] - lr * grad + noise
            fisher_chains.append(
                beta * (lr / lr_div_factor) * np.linalg.norm(grad - v_grad) ** 2
            )
            chain_objectives[chain] += fisher_chains[-1]
        chain_objective += np.mean(fisher_chains)

    chain_objective = np.sqrt(chain_objective)
    chain_objectives = [np.sqrt(co) for co in chain_objectives]
    val_errors = []
    val_r2s = []
    avg_error = 0.0
    avg_r2 = 0.0
    for idx, chain in enumerate(chains):
        val_error = get_error(chain, vX, data_set.valid_labels)
        chain_objectives[idx] += val_error
        val_r2 = get_r2(chain, vX, data_set.valid_labels)
        val_errors.append(val_error)
        val_r2s.append(val_r2)

    avg_error = np.mean(val_errors)
    avg_r2 = np.mean(val_r2s)

    chain_objective += avg_error
    return chain_objective, avg_error, avg_r2, chain_objectives

def get_error(coefs, X, labels):
    """Compute the mean squared error"""
    preds = np.dot(X, coefs.reshape(-1, 1)).reshape(-1)
    MSE = np.mean((preds - labels) ** 2)
    return MSE

def get_r2(coefs, X, labels):
    """Compute the R^2 statistic"""
    preds = np.dot(X, coefs.reshape(-1, 1)).reshape(-1)
    SSE = np.sum((preds - labels) ** 2)
    r_sq = 1 - SSE / (len(labels) * np.var(labels))
    return r_sq

def gradient(coefs, X, labels, clip=np.inf):
    """Compute the deriatives of the mean squared error with respect
    to the coefficients"""
    gradient = X.T @ X @ coefs.reshape(-1, 1) - X.T @ labels.reshape(-1, 1)
    gradient = gradient / len(labels)
    if np.linalg.norm(gradient) > clip:
        gradient = gradient / (np.linalg.norm(gradient) / clip)
    return gradient

if __name__ == "__main__":
    main()

max_stars_repo_name: DEShawResearch/PACBayesHyperOpt

max_issues_repo_name: DEShawResearch/PACBayesHyperOpt

max_forks_repo_name: DEShawResearch/PACBayesHyperOpt


text:
# -*- coding: utf-8 -*-

"""
contain class for targets, observations, drz, flt, and sources.
"""

import os
import sys
import copy
import shutil
import numpy as np
from astropy import wcs
from astropy.io import fits
import astropy.table as at
import gzip
import jinja2 as jin
import numbers

def make_roadmap(fp_tab_target, fp_tab_observations, fp_tab_sources, source_names, observation_names=None, fp='roadmap.json', key='obj_name', dir_data='./', dir_local='./'):
	""" make roadmap json file from input tables

	Note:
		each table has to have the same number of rows. Each table has to have a key column with name, e.g., 'obj_name', as set by args 'key', for unique identification of targets.

		All drz and flt fitz file should locate in one single directory dir_data.

	Args:
		fp_tab_target (str):
			path to input target table. Columns: 'ra', 'dec'.
		fp_tab_observations (list of str):
			list of paths to input observation tables. Columns: 'camera', 'filter', 'fn_drzs'.
		fp_tab_sources (list of str):
			list of paths to input source tables. Columns: 'ra', 'dec', 'spectral_form', 'spectral_type'
		source_names (list of str): list of source names, same length as fp_tab_sources.
		observation_names=None (list of str, optional):
			list of observation names, same length as fp_tab_observations.
			Default: 'obs0', 'obs1', etc.
		fp='roadmap.json' (str, optional): path to output json file.
		key='obj_name' (str): column name of the key column.
		dir_data='./' (str): directory where fits files are taken from.
		dir_local='./' (str):
			dummy directory for parent dir of targets that will not affect the result.

	Return:
		(bool): True if succes
	"""
	# setting
	tab_target = at.Table.read(fp_tab_target)
	tab_observations = [at.Table.read(f) for f in fp_tab_observations]
	tab_sources = [at.Table.read(f) for f in fp_tab_sources]

	if observation_names is None:
		observation_names = ['obs'+str(i) for i in range(len(fp_tab_observations))]

	# make sure each table has corresponding rows.
	tab_target.sort(key)
	lst_target = list(tab_target[key])
	for tab_observation in tab_observations:
		tab_observation.sort(key)
		assert list(tab_observation[key]) == lst_target

	for tab_observation in tab_observations:
		tab_observation.sort(key)
		assert list(tab_observation[key]) == lst_target

	for tab_source in tab_sources:
		tab_source.sort(key)
		assert list(tab_source[key]) == lst_target

	# run
	aux_cols_target = tab_target.colnames
	[aux_cols_target.remove(col) for col in [key, 'ra', 'dec']]

	targets = []
	for i, row_target in enumerate(tab_target):
		aux = {col: row_target[col] for col in aux_cols_target}
		target = Target(name=row_target[key], ra=row_target['ra'], dec=row_target['dec'], dir_parent=dir_local, **aux)
		sources = {source_names[j]: Source(name=source_names[j], ra=tab_source[i]['ra'],
					dec=tab_source[i]['dec'],
					spectrum_form=tab_source[i]['spectrum_form'],
					spectrum_type=tab_source[i]['spectrum_type']
					) for j, tab_source in enumerate(tab_sources)}

		observations = []
		for k, tab_observation in enumerate(tab_observations):
			observation = Observation(name=observation_names[k],
									camera=tab_observation[i]['camera'],
									filter=tab_observation[i]['filter'],
									dir_parent=target.directory)

			fn_drzs = tab_observation[i]['fn_drzs'].split(',')
			drz_names = ['drz'+str(l) for l in range(len(fn_drzs))]
			drzs = [DRZ(name=drz_names[l], fp=dir_data+fn_drzs[l], sources=sources, dir_parent=observation.directory) for l in range(len(fn_drzs))]
			observation.drzs = drzs
			observations = observations+[observation]
		target.observations = observations
		target.sources = sources
		targets = targets +[target]

	render_roadmap(fp, targets)
	return os.path.isfile(fp)

def render_roadmap(fp, targets):
	"""
	write targets as json file to fp
	"""
	fp_template = 'roadmap_template.json'
	env = jin.Environment(loader=jin.FileSystemLoader(getlocalpath()))
	env.filters['quote'] = quote_jfilter
	temp = env.get_template(fp_template)
	targets_json = temp.render(targets=targets)

	with open(fp, 'w') as file:
		file.write(targets_json)

def getlocalpath():
	"""
	return path to filter direcotry
	"""
	path = os.path.dirname(sys.modules[__name__].__file__)
	if path == '': path ='.'
	return path+'/'

def quote_jfilter(string):
	if isinstance(string, numbers.Number):
		return string
	else:
		try:
			return "\""+string+"\""
		except:
			return "null"

class Target(object):
	"""Target that has all info for processing psf-subtraction of an target

	Attributes:
		name (str): name of the source.
		directory (str): path to the working directory
		ra (float): ra
		dec (float): dec
		reference (str): sample source of the target, could be None.
		sources (list of :obj: Source):
			list of sources associated with the target with attributes ra, dec, spectrum_form, spectrum_type.
		observations (list of :obj: Observation):
			list of observations of the target.
		roadmap (json, optional): json representation of the object.
	"""
	def __init__(self, dir_parent='./', **kwargs):
		""" initializing

		Note:
			attribute `directory` is set to dir_parent+name+'/'

		Args:
			dir_parent='./' (str): path to the parent directory
			either:
				roadmap (json): json representation of the object containing the following.
			or:
				name (str): name of the source.
				ra (float): ra
				dec (float): dec
				reference (str): sample source
				sources (list of :obj: Source):
					list of sources associated with the target with attributes ra, dec, spectrum_form, spectrum_type.
				observations (list of :obj: Observation):
					list of observations of the target.
		"""
		if 'roadmap' in kwargs:
			roadmap = kwargs.pop('roadmap', None)
			self.roadmap = copy.copy(roadmap)
			self.name = roadmap.pop('name', None)
			self.directory = dir_parent+self.name+'/'
			self.ra = roadmap.pop('ra', None)
			self.dec = roadmap.pop('dec', None)
			self.reference = roadmap.pop('reference', None)
			self.sources = get_dict_Sources_from_roadmap(roadmap=roadmap.pop('sources', []), dir_parent=self.directory)
			self.observations = [Observation(dir_parent=self.directory, roadmap=r) for r in roadmap.pop('observations', [])]
		else:
			self.name = kwargs.pop('name', None)
			self.directory = dir_parent+self.name+'/'
			self.ra = kwargs.pop('ra', None)
			self.dec = kwargs.pop('dec', None)
			self.reference = kwargs.pop('reference', None)
			self.sources = kwargs.pop('sources', {})
			self.observations = kwargs.pop('observations', [])

class Observation(object):
	""" an observation that has name, camera, filter, drzs

	Attributes:
		name (str): name of the observation.
		directory (str): path to the working directory
		camera (str): the camera.
		filter (str): the filter.
		drzs (list of :obj: DRZ): the drz files associated with the observation.
		roadmap (json, optional): json representation of the object.
	"""
	def __init__(self, dir_parent='./', **kwargs):
		""" initializing

		Note:
			attribute `directory` is set to dir_parent+name+'/'

		Args:
			dir_parent='./' (str): path to the parent directory
			either:
				roadmap (json): json representation of the object containing the following.
			or:
				name (str): name of the observation.
				camera (str): Following tinytim convention should be one of the following:
					'wfpc1', 'wfpc1_planetary', 'wfpc1_foc_f48', 'wfpc1_foc_f48', 'wfpc2', 'wfpc2_planetary', 'wfpc2_foc_f48', 'wfpc2_foc_f48', 'nicmos1_precryo', 'nicmos2_precryo', 'nicmos3_precryo', 'stis_ccd', 'stis_nuv', 'stis_fuv', 'acs_widefield', 'acs_highres', 'acs_coronoffspot', 'acs_solarblind', 'nicmos1_cryo', 'nicmos2_cryo', 'nicmos3_cryo', 'wfc3_uvis', 'wfc3_ir',

				filter (str): the filter, e.g., 'f160w', see tinytim.
				drzs (list of :obj: DRZ): the drz files associated with the observation.
		"""
		if 'roadmap' in kwargs:
			roadmap = kwargs.pop('roadmap', None)
			self.roadmap = copy.copy(roadmap)
			self.name = roadmap.pop('name', None)
			self.directory = dir_parent+self.name+'/'
			self.camera = roadmap.pop('camera', None)
			self.filter = roadmap.pop('filter', None)
			self.drzs = [DRZ(dir_parent=self.directory, roadmap=r) for r in roadmap.pop('drzs', [])]
		else:
			self.name = kwargs.pop('name', None)
			self.directory = dir_parent+self.name+'/'
			self.camera = kwargs.pop('camera', None)
			self.filter = kwargs.pop('filter', None)
			self.drzs = kwargs.pop('drzs', [])

class DRZ(object):
	"""a hst drz file that has name, fp, and a list of flt files.

	Attributes:
		name (str): name of the drz file, can be arbitrary.
		directory (str): path to the working directory
		fp (str): file name (including path) to the drz file.
		fp_local (str): file path in local data tree, set to direcotry+'drz.fits'.
		wcs (:obj: 'astropy.wcs.WCS')
		flts (:obj: 'list' of :obj: Flt)
		sources (list of :obj: FITSource, optional)
		roadmap (json, optional): json representation of the object.

	Methods:
		copyfile()
	"""
	def __init__(self, dir_parent='./', **kwargs):
		""" Initialize drz object.

		Note:
			attribute `directory` is set to dir_parent+name+'/'

			wcs is read from the header.

			The FLT files will be automatically identified from drz header if not specified. For automated identification, the flt files should locate in the same directory as the drz file. The flts are named as flt0, flt1, ..., etc.

			If fp does not exist, it will attemp to unzip a file named fp+'.gz'.

		Args:
			dir_parent='./' (str): path to the parent directory
			either:
				roadmap (json):
					json representation of the object containing the following. If it does not contain flts info, it will be automatically extracted from the header of the drz file.
			or:
				name (str): name of the drz file, can be arbitrary.
				fp (str): file name (including path) to the drz file.
				sources = {} (list of :obj: Source or FITSource, optional)
				flts = [] (:obj: 'list' of :obj: Flt, optional):
					If not specified, it will be automatically extracted from the header of the drz file.
		"""
		if 'roadmap' in kwargs:
			roadmap = kwargs.pop('roadmap', None)
			self.roadmap = copy.copy(roadmap)
			self.name = roadmap.pop('name', None)
			self.directory = dir_parent+self.name+'/'
			self.fp = roadmap.pop('fp', None)
			self.fp_local = self.directory+'drz.fits'
			self._set_wcs()

			self.flts = [FLT(dir_parent=self.directory, roadmap=r) for r in roadmap.pop('flts', [])]
			self.sources = get_dict_FITSources_from_roadmap(roadmap=roadmap.pop('sources', {}), wcs=self.wcs, dir_parent=self.directory)

		else:
			self.name = kwargs.pop('name', None)
			self.directory = dir_parent+self.name+'/'
			self.fp = kwargs.pop('fp', None)
			self.fp_local = self.directory+'drz.fits'
			self._set_wcs()
			# if not os.path.isfile(self.fp):
			# 	if os.path.isfile(self.fp+'.gz'):
			# 		unzip_gz(self.fp+'.gz', self.fp)
			# self.wcs = wcs.WCS(fits.getheader(self.fp, ext=1), fits.open(self.fp))
			self.flts = kwargs.pop('flts', [])
			sources = kwargs.pop('sources', {})
			self.set_fitsources(sources)
			# self.sources = {name: FITSource(wcs=self.wcs, name=name, ra=source.ra, dec=source.dec, ) for name, source in sources.items()}

		# automated identification of the flt files.
		if self.flts == []:
			self._initialize_flts_from_header()

	def _set_wcs(self):
		""" set wcs using the header of self.fp if it exist, if not try unzip corresponding .gz, if not from self.fp_local. """
		if os.path.isfile(self.fp):
			fp_4_wcs = self.fp
		else:
			if os.path.isfile(self.fp+'.gz'):
				# unzip fp
				unzip_gz(self.fp+'.gz', self.fp)
				fp_4_wcs = self.fp
			else:
				fp_4_wcs = self.fp_local
		self.wcs = wcs.WCS(fits.getheader(fp_4_wcs, ext=1), fits.open(fp_4_wcs))

	def set_fitsources(self, sources):
		""" setting attribute sources for drz and all the flts from argument and enforcing that x, y are calculated from the corresponding wcs. Auxiliary attributes of sources are also propagated over.
		"""
		# setting list of additional auxiliary attributes from an arbitrary source
		if len(sources) == 0:
			self.sources={}
		else:
			aux_attr = list(next(iter(sources.values())).__dict__.keys())
			[aux_attr.remove(key) for key in ['name', 'ra', 'dec', 'directory']]

			# self.sources = {name: FITSource(name=name, ra=source.ra, dec=source.dec, dir_parent=self.directory, wcs=self.wcs, ) for name, source in sources.items()}
			self.sources = {name: FITSource(name=name, ra=source.ra, dec=source.dec, dir_parent=self.directory, wcs=self.wcs, **{a: getattr(source, a) for a in aux_attr}) for name, source in sources.items()}

		for flt in self.flts:
			flt.set_fitsources(sources)

	def copyfile(self):
		""" copy fits file from fp to local location fp_local.
		"""
		if self.fp_local != self.fp:
			if not os.path.isdir(self.directory):
				os.makedirs(self.directory)
			shutil.copy(self.fp, self.fp_local)

	def _initialize_flts_from_header(self, dir_flt=None):
		""" inititalize flts from the header of the drz file

		Args:
			dir_flt (str, optional):
				path of the flt parent directory. Default is set to the parent directory of the drz file.
		"""
		header = fits.getheader(self.fp)
		nflt = header['NDRIZIM']
		flt_fns = [header['D00{}DATA'.format(str(i+1))].split('[')[0] for i in range(nflt)]

		if dir_flt is None:
			dir_flt = os.path.dirname(self.fp)+'/'

		self.flts = [FLT(name='flt{}'.format(str(i)), fp=dir_flt+flt_fns[i], sources=self.sources, dir_parent=self.directory) for i in range(nflt)]

class FLT(object):
	"""an hst flt file that has name, fp (filepath), and optionally a list of sources.

	Attributes:
		name (str): name of the flt file, can be arbitrary.
		directory (str): path to the working directory
		fp (str): file name (including path) to the flt file.
		fp_local (str): file path in local data tree, set to direcotry+'flt.fits'.
		wcs (:obj: 'astropy.wcs.WCS')
		sources (list of :obj: FITSource, optional)
		roadmap (json, optional): json representation of the object.

	Methods:
		copyfile()
	"""
	def __init__(self, dir_parent='./', **kwargs):
		""" Initialize flt object.

		Note:
			attribute `directory` is set to dir_parent+name+'/'
			The wcs is read from the header.
			If fp does not exist, it will attemp to unzip a file named fp+'.gz'.

		Args:
			dir_parent='./' (str): path to the parent directory
			either:
				roadmap (json): json representation of the object containing the following.
			or:
				name (str): name of the flt file, can be arbitrary.
				fp (str): file name (including path) to the flt file.
				sources = {} (list of :obj: Source or FITSource, optional)
		"""
		if 'roadmap' in kwargs:
			roadmap = kwargs.pop('roadmap', None)
			self.roadmap = copy.copy(roadmap)
			self.name = roadmap.pop('name', None)
			self.directory = dir_parent+self.name+'/'
			self.fp = roadmap.pop('fp', None)
			self.fp_local = self.directory+'flt.fits'
			self._set_wcs()
			# # unzip fp
			# if not os.path.isfile(self.fp):
			# 	if os.path.isfile(self.fp+'.gz'):
			# 		unzip_gz(self.fp+'.gz', self.fp)
			# self.wcs = wcs.WCS(fits.getheader(self.fp, ext=1), fits.open(self.fp))
			self.sources = get_dict_FITSources_from_roadmap(roadmap=roadmap.pop('sources', {}), wcs=self.wcs, dir_parent=self.directory)
		else:
			self.name = kwargs.pop('name', None)
			self.directory = dir_parent+self.name+'/'
			self.fp = kwargs.pop('fp', None)
			self.fp_local = self.directory+'flt.fits'
			self._set_wcs()
			# # unzip fp
			# if not os.path.isfile(self.fp):
			# 	if os.path.isfile(self.fp+'.gz'):
			# 		unzip_gz(self.fp+'.gz', self.fp)
			# self.wcs = wcs.WCS(fits.getheader(self.fp, ext=1), fits.open(self.fp))
			sources = kwargs.pop('sources', {})
			self.set_fitsources(sources)
			# self.sources = {name: FITSource(name=name, ra=source.ra, dec=source.dec, wcs=self.wcs, ) for name, source in sources.items()}

		self.fp_skysub = self.directory+'flt_skysub.fits'

	def _set_wcs(self):
		""" set wcs using the header of self.fp if it exist, if not try unzip corresponding .gz, if not from self.fp_local. """
		if os.path.isfile(self.fp):
			fp_4_wcs = self.fp
		else:
			if os.path.isfile(self.fp+'.gz'):
				# unzip fp
				unzip_gz(self.fp+'.gz', self.fp)
				fp_4_wcs = self.fp
			else:
				fp_4_wcs = self.fp_local
		self.wcs = wcs.WCS(fits.getheader(fp_4_wcs, ext=1), fits.open(fp_4_wcs))

	def make_flt_skysub(self):
		"""produce skysubtracted flt file 'flt_skysub.fits'. constant sky of value 'MDRIZSKY' in header is taken out.
		"""
		hdus = fits.open(self.fp_local)

		midrizsky = hdus[1].header['MDRIZSKY']
		data = hdus[1].data
		data_skysub = data - midrizsky

		hdus[1].data = data_skysub
		hdus[1].header['MDRIZSKY'] = 0.
		hdus[1].header['HISTORY'] = 'Constant sky of level {} is subtracted'.format(str(midrizsky))

		hdus.writeto(self.fp_skysub, overwrite=True)

	def set_fitsources(self, sources):
		""" setting attribute sources from argument and enforcing that x, y are calculated from self.wcs. Auxiliary attributes of sources are also propagated over.
		"""
		# setting list of additional auxiliary attributes from an arbitrary source
		if len(sources) == 0:
			self.sources={}
		else:
			aux_attr = list(next(iter(sources.values())).__dict__.keys())
			[aux_attr.remove(key) for key in ['name', 'ra', 'dec', 'directory']]

			# self.sources = {name: FITSource(name=name, ra=source.ra, dec=source.dec, dir_parent=self.directory, wcs=self.wcs, ) for name, source in sources.items()}
			self.sources = {name: FITSource(name=name, ra=source.ra, dec=source.dec, dir_parent=self.directory, wcs=self.wcs, **{a: getattr(source, a) for a in aux_attr}) for name, source in sources.items()}

	def copyfile(self):
		""" copy fits file from fp to local location fp_local.
		"""
		if self.fp_local != self.fp:
			if not os.path.isdir(self.directory):
				os.makedirs(self.directory)
			shutil.copy(self.fp, self.fp_local)

class Source(object):
	""" a source that has name, ra, dec, and optionally other attributes.

	Attributes:
		name (str): name of the source.
		# directory (str): path to the working directory
		ra (float): ra
		dec (float): dec
		dir_parent='./' (str): path to the parent directory
	"""
	def __init__(self, **kwargs):
		""" initializing
		Args:
			name (str): name of the source.
			ra (float): ra
			dec (float): dec
			# dir_parent='./' (str): path to the parent directory
			**kwargs: 'key' 'value' pairs that will be set as attributes.
		"""
		self.name = kwargs.pop('name', None)
		self.directory = kwargs.pop('dir_parent', './')+self.name+'/'
		self.ra = kwargs.pop('ra', None)
		self.dec = kwargs.pop('dec', None)
		for key, value in kwargs.items():
			setattr(self, key, value)

class FITSource(Source):
	"""a source that has name, ra, dec and corresponding x, y position in the fits file

	Note:
		x, y is automatically inferred from wcs and ra, dec and if x, y were input as arguments it will be overwritten.

	Attributes:
		name (str): name of the source.
		# directory (str): path to the working directory
		ra (float): ra
		dec (float): dec
		x (int): x position in fits file
		y (int): y position in fits file
	"""
	def __init__(self, wcs, **kwargs):
		""" initializing
		Args:
			wcs (:obj: 'astropy.wcs.WCS')
			# dir_parent='./' (str): path to the parent directory
			name (str): name of the source.
			ra (float): ra
			dec (float): dec
			x (int): x position in fits file
			y (int): y position in fits file
		"""

		super(self.__class__, self).__init__(**kwargs)

		# set origin to 0 to comply with numpy convention
		x, y = np.round(wcs.all_world2pix(np.array([[self.ra, self.dec]]), 0, maxiter=100)[0])
		self.x, self.y = int(x), int(y)

def get_dict_FITSources_from_roadmap(roadmap, wcs, **kwargs):
	"""
	translate roadmap of sources to dictionary of Source objects.

	Args:
		roadmap (dict): For example:
			{
				"qso":{
					"ra": 128.000833,
					"dec": 16.250111,
					"spectrum_form": "powerlaw_nu",
					"spectrum_type": -1.33441542369
				},
				"star0":{
					"ra": 128.01769053,
					"dec": 16.24563831,
					"spectrum_form": "stellar",
					"spectrum_type": "g8v"
				},
			}
		wcs (:obj: 'astropy.wcs.WCS')
		** kwargs: additional arguments for FITSource

	Return:
		{dict of :obj: 'FITSource'}
	"""
	return {name: FITSource(wcs=wcs, name=name, **value, **kwargs) for name, value in roadmap.items()}

def get_dict_Sources_from_roadmap(roadmap, **kwargs):
	"""
	translate roadmap of sources to dictionary of Source objects.

	Args:
		roadmap (dict): For example:
			{
				"qso":{
					"ra": 128.000833,
					"dec": 16.250111,
					"spectrum_form": "powerlaw_nu",
					"spectrum_type": -1.33441542369
				},
				"star0":{
					"ra": 128.01769053,
					"dec": 16.24563831,
					"spectrum_form": "stellar",
					"spectrum_type": "g8v"
				},
			}
		** kwargs: additional arguments for FITSource

	Return:
		{dict of :obj: 'Source'}
	"""
	return {name: Source(name=name, **value, **kwargs) for name, value in roadmap.items()}

def unzip_gz(fp_in, fp_out):
	""" unzip gz file fp_in to fp_out """
	print('unzipping', fp_in)
	with gzip.open(fp_in, 'rb') as f_in:
		with open(fp_out, 'wb') as f_out:
			shutil.copyfileobj(f_in, f_out)

max_stars_repo_name: aileisun/tinyfit

max_issues_repo_name: aileisun/tinyfit

max_forks_repo_name: aileisun/tinyfit

reason: import numpy,import astropy,from astropy


text:
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

titanic = pd.read_excel('titanic.xls')
from sklearn.neighbors import KNeighborsClassifier


text:
from unittest.mock import MagicMock

import pytest
from eliot import MemoryLogger
from eliot.testing import check_for_errors, swap_logger
from networkx import DiGraph

@pytest.fixture(autouse=True)
def logger():
    test_logger = MemoryLogger()
    swap_logger(test_logger)
    yield test_logger
    check_for_errors(test_logger)

@pytest.fixture
def bootsteps_graph():
    mocked_graph = MagicMock(spec_set=DiGraph)
    mocked_graph.__iter__.return_value = []

    return mocked_graph

max_stars_repo_name: celery/bootsteps

max_issues_repo_name: celery/bootsteps

max_forks_repo_name: celery/bootsteps

reason: from networkx


text:
"""
Functions for generating and managing embeddings.
"""

import numpy as np

def load_embeddings(embeddings_path, network):
    """ Builds a numpy matrix for a node embedding encoded in the embeddingf ile
    passed in. Row indices are given by protein_to_node dictionary passed in.
    @param embedding_filename (string)
    @param network (Network)
    @returns embeddings (np.array)
    """
    with open(embeddings_path) as embedding_file:
        # get
        line = embedding_file.readline()
        n_nodes, n_dim = map(int, line.split(" "))
        embeddings = np.empty((len(network), n_dim))

        for index in range(n_nodes):
            line = embedding_file.readline()
            line_elements = line.split(" ")
            protein = int(line_elements[0])

            if protein not in network.protein_to_node:
                continue

            node_embedding = list(map(float, line_elements[1:]))
            embeddings[network.get_node(protein), :] = node_embedding

    return embeddings

def write_embeddings(embeddings, embeddings_path, network):
    """
    @param embeddings (np.array)
    @param embeddings_path (string)
    @param network (Network)
    """
    with open(embeddings_path, 'w') as embedding_file:
        n_nodes, n_dim = embeddings.shape
        embedding_file.write(f"{n_nodes} {n_dim}\n")
        for node in range(n_nodes):
            protein = network.get_protein(node)
            embedding = " ".join(map(str, list(embeddings[node, :])))
            line = f"{protein} {embedding}\n"
            embedding_file.write(line)

max_stars_repo_name: seyuboglu/milieu

max_issues_repo_name: seyuboglu/milieu

max_forks_repo_name: seyuboglu/milieu


text:
'''
Evaluate finetuned GPT3 on the in-domain test set of existing benchmarks: MPE, ADEPT.
End-of-prompt separator:\n\n###\n\n
Start-of-completion separator:a whitespace
End token:\n, ### or any other special token which doesn't appear within any completion.
'''
import csv
import json
import numpy as np
import openai

import os
import sys
os.chdir("../../../..")
root_dir = os.getcwd()

# config
from configuration import Config
config_path = (f'source/Qa/finetune_on_benchmark/gpt3/config.json')
config = Config.from_json_file(config_path)

sys.path.append(f"{root_dir}/source")
from Qa.utils import compute_scores, n_classes_dict_benchmark, task_fieldnames, query

# name of our finetuned models for each RNPC task
finetuned_model_dict = {
	"MPTE":{
		"ada":"ada:ft-ccb-lab-members-2021-08-12-01-09-11",
		"curie":"curie:ft-ccb-lab-members-2021-08-12-02-24-03"
	},
	"EPC":{
		"ada":"ada:ft-ccb-lab-members-2021-08-13-00-40-39",
		"curie":"curie:ft-ccb-lab-members-2021-08-13-02-39-17",
	}
}

if __name__ == "__main__":

	# config
	RNPC_task = eval(config.task)

	# not supporting the corresponding benchmarks for SPTE, since the finetuned GPT3 models on SNLI/MNLI are not publicly available
	if RNPC_task not in ["MPTE", "EPC"]:
		raise ValueError(f"Unspported task: {RNPC_task}. Please choose from 'MPTE', and 'EPC'.")

	benchmark_dict = {"MPTE": "MPE",
	                  "EPC": "ADEPT"}
	benchmark = benchmark_dict[RNPC_task]

	model = eval(config.gpt3_model)
	trial = config.trial

	# dirs
	jsonl_dir = f"data/existing_benchmarks/{benchmark}/jsonl"
	output_dir = f"output_dir/benchmarks/{benchmark}"
	if not os.path.isdir(output_dir):
		os.makedirs(output_dir)
	if trial:
		frn = f"{jsonl_dir}/test_sample.jsonl"
	else:
		frn = f"{jsonl_dir}/test.jsonl"
	fwn = f"{output_dir}/gpt3-{model}_test.tsv"

	gold_labels, pred_labels = [], []

	# For SPTE, the finetuning datasets (MNLI, SNLI) are too large.
	# OpenAI doesn't allow datasets of such sizes at the time of the work.

	# Evaluate on MPE
	if benchmark == "MPE":
		with open(frn, 'r') as fr, open(fwn, 'w') as fw:
			writer = csv.DictWriter(fw, delimiter="\t", fieldnames=task_fieldnames[RNPC_task])
			writer.writeheader()

			for row_id, row_obj in enumerate(fr.readlines()):
				if row_id % 50 == 0:
					print(f"{row_id} examples finished.")

				row = json.loads(row_obj)
				prompt, completion = row["prompt"], row["completion"]
				gold_label = int(completion.strip())

				pred_label, label_probs = query(finetuned_model_dict[RNPC_task][model], prompt, n_classes_dict_benchmark[RNPC_task])
				pred_prob = label_probs[pred_label] / sum(label_probs.values())

				gold_labels.append(gold_label)
				pred_labels.append(pred_label)

				cols = prompt.split("\n")
				premises, hypothesis = cols[0], cols[1]
				premises = premises[len("Premises: "):]
				hypothesis = hypothesis[len("Hypothesis: "):]

				new_row = {}
				new_row["premise"] = premises
				new_row["hypothesis"] = hypothesis
				new_row["gold label"] = gold_label
				new_row["pred label"] = pred_label
				new_row["confidence"] = pred_prob
				writer.writerow(new_row)

	# Evaluate on ADEPT
	elif benchmark == "ADEPT":

		with open(frn, 'r') as fr, open(fwn, 'a') as fw:
			writer = csv.DictWriter(fw, delimiter="\t", fieldnames=task_fieldnames[RNPC_task])
			writer.writeheader()

			for row_id, row_obj in enumerate(fr.readlines()):

				if row_id % 50 == 0:
					print(f"{row_id} examples finished.")

				row = json.loads(row_obj)
				prompt, completion = row["prompt"], row["completion"]
				gold_label = int(completion.strip())

				pred_label, label_probs = query(finetuned_model_dict[RNPC_task][model], prompt, n_classes_dict_benchmark[RNPC_task])
				pred_prob = label_probs[pred_label] / sum(label_probs.values())

				gold_labels.append(gold_label)
				pred_labels.append(pred_label)

				cols = prompt.split("\n")
				event1, event2 = cols[0], cols[1]
				event1 = event1[len("Event 1: "):]
				event2 = event2[len("Event 2: "):]

				new_row = {}
				new_row["first_event"] = event1
				new_row["second_event"] = event2
				new_row["gold label"] = gold_label
				new_row["pred label"] = pred_label
				new_row["confidence"] = pred_prob
				writer.writerow(new_row)

	scores =compute_scores(n_classes_dict_benchmark[RNPC_task], gold_labels, pred_labels)
	print(scores)


text:
from abc import ABC, abstractmethod
import os
import numpy as np
import torch
import gym
from torch.utils.tensorboard import SummaryWriter
import matplotlib.pyplot as plt
from collections import deque
from scipy.stats import bernoulli
import json

import os
import yaml
import argparse
from datetime import datetime
from collections import deque
import gym
import random
import torch
import pandas as pd
from PPO import PPO

def exploit(actor, state):
    # Act without randomness.
    state = torch.FloatTensor(
        state[None, ...]).float()  # /255.
    with torch.no_grad():
        action = actor.eval().act(state)
    return action.item()

def EvalActor(path, env, actor, patientNo):
    # para is a string denotes which policy to use
    policy = torch.load(path + patientNo +"_policy.pth")
    actor.load_state_dict(policy)
    actor.eval()

    max_episode_steps = 120
    state_arr = deque()
    action_arr = deque()
    reward_arr = []
    dose_arr = []
    fea, state = env.reset()
    state_arr.append(state)
    episode_steps = 0
    episode_return = 0.0
    done = False
    while (not done) and episode_steps <= max_episode_steps:
        action = actor.act_exploit(fea)
        fea, next_state, reward, done, infos = env.step(action)
        episode_steps += 1
        episode_return += reward
        state = next_state
        action_arr.append(action)
        state_arr.append(state)
        reward_arr.append(reward)
        dose_arr.append(infos["dose"])
    return np.array(action_arr).reshape(-1), np.array(state_arr).reshape(-1,3), np.array(reward_arr).reshape(-1), np.array(dose_arr).reshape(-1,2)

def MakePlot(path, para, env, actor,  plotStyle, patientNo, metasPar):
    actSeq, stateSeq, rwdSeq, doseSeq = EvalActor(path, para, env, actor, patientNo)
    doseCPA, doseLEU = np.array(doseSeq[:,0]).reshape(-1), np.array(doseSeq[:,1]).reshape(-1)
    CPATotalDosage, LEUTotalDosage = doseCPA.sum(), doseLEU.sum()
    CPAHolidayMonth, LEUHolidayMonth = np.where(doseCPA == 0)[0].shape[0], np.where(doseLEU == 0)[0].shape[0]
    CPAFreePercentage, LEUFreePercentage = CPAHolidayMonth/len(doseSeq), LEUHolidayMonth/len(doseSeq)
    # evolution process
    stateDF = pd.DataFrame(stateSeq, columns=['AD', "AI", "PSA"])
    stateDF['DAY'] = np.arange(0, stateSeq.shape[0]) * 28
    plt.style.use(plotStyle)
    plt.figure(figsize=(15, 5))
    plt.subplot(1, 2, 1)
    plt.plot(stateDF['DAY'], stateDF['PSA'])
    plt.xlabel("Time (Days)", fontsize = 14)
    plt.ylabel("PSA level (ug/ml)", fontsize = 14)

    plt.subplot(1, 2, 2)
    plt.plot(stateDF['DAY'], stateDF['AD'], label="HD")
    plt.plot(stateDF['DAY'], stateDF['AI'], label="HI")
    plt.xlabel("Time (Days)", fontsize = 14)
    plt.ylabel("Cell counts", fontsize = 14)
    plt.legend(loc='upper left',  fontsize=16)
    plt.savefig("../ManualScripts/RLPics/Analysis_patient006/Evolution_" + metasPar +"_"+ patientNo + ".png", dpi=300)
    #plt.savefig("../ManualScripts/RLPics/Evolution_" + patientNo + ".png", dpi=300)
    # plt.show()
    plt.close()
    # dose plot
    COLOR_CPA = "#69b3a2"
    COLOR_LEU = '#FF4500'
    doseDF = pd.DataFrame(doseSeq, columns=['CPA', "LEU"])
    actDF = pd.DataFrame(actSeq)
    actDF["DAY"] = np.arange(0, len(actSeq)) * 28
    doseDF['DAY'] = np.arange(0, len(doseSeq)) * 28
    plt.style.use(plotStyle)
    fig = plt.figure(figsize=(7.5, 5))
    ax1 = fig.add_subplot(111)
    ax2 = ax1.twinx()
    ax1.plot(doseDF['DAY'], doseDF['CPA'], color=COLOR_CPA, label='CPA', lw = 1.5)
    ax1.set_xlabel("Days")
    ax1.set_ylabel("CPA (mg/ Day)", fontsize=14)
    # ax1.set_ylim(-5, 205)
    ax1.set_yticks(np.arange(0, 250, 50))
    ax1.tick_params(axis="y", labelcolor=COLOR_CPA)
    ax1.legend(loc=(0.03, 0.9), fontsize=14, facecolor = COLOR_CPA)
    plt.grid(False)
    ax2.plot(doseDF['DAY'], doseDF['LEU'], color=COLOR_LEU, label='LEU', lw = 1.5, ls ='--')
    ax2.set_ylabel("LEU (ml/ Month)",  fontsize=14)
    ax2.tick_params(axis="y", labelcolor=COLOR_LEU)
    ax2.set_yticks(np.arange(0, 22.5, 7.5))
    plt.ylim(-.5, 15)
    ax2.legend(loc=(0.03, 0.8),  fontsize=14, facecolor = COLOR_LEU)
    plt.savefig("../ManualScripts/RLPics/Analysis_patient006/Dosage_" + metasPar +"_"+ patientNo + ".png", dpi=300)
    # plt.savefig("../ManualScripts/RLPics/Dosage_" + patientNo + ".png", dpi=300)
    # plt.show()
    plt.close()
    return [CPATotalDosage, CPAHolidayMonth, CPAFreePercentage, LEUTotalDosage, LEUHolidayMonth, LEUFreePercentage, len(doseSeq)], doseDF, actDF

def run(args):
    patientNo = args.patientNo
    # print(patientNo)
    list_df = args.patients_pars[patientNo]
    A, K, states, pars, best_pars = [np.array(list_df.loc[i, ~np.isnan(list_df.loc[i, :])]) for i in range(5)]
    A = A.reshape(2, 2)
    init_state = states[:3]
    terminate_state = states[3:]

    default_acts = pd.read_csv("../Model_creation/test-sigmoid/model_actions/" + patientNo + "_actions_seqs.csv")
    default_acts = np.array(default_acts)
    weight = np.array([1.46, 0.206, 1.5])
    weight1 = np.array([1.3, 0.7])
    base = 1.15
    m1 = 0.8
    m2 = 12

    patient = (A, K, best_pars, init_state, terminate_state, weight, weight1, base, m1, m2)
    # Create environments.
    env = gym.make(args.env_id, patient = patient).unwrapped
    policypath = args.policyPath
    # policypath =
    actor = CateoricalPolicy(3, 10) #.to("cuda:0")
    AnaDosage, DoseSeq, ActSeq = MakePlot(policypath, args.para, env, actor, args.plotStyle, patientNo, args.metasPar)
    return AnaDosage, DoseSeq, ActSeq

env_dict = gym.envs.registration.registry.env_specs.copy()
for env in env_dict:
    if 'CancerControl-v0' in env:
        print("Remove {} from registry".format(env))
        del gym.envs.registration.registry.env_specs[env]
parsdir = "../Model_creation/retrain-sigmoid/model_pars"
parslist = os.listdir(parsdir)
patient_pars = {}
patient_test = []
patient_train = []
# reading the ode parameters and the initial/terminal states
for args in parslist:
    pars_df = pd.read_csv(parsdir + '/' + args)
    patient = args[5:(-4)]
    patient_train.append(patient)
    if patient not in patient_test:
        patient_pars[patient] = pars_df
DoseList = []
paraSetting = ["test"]#['0112', '01512', '0212', '02512', '0312', '03512', '0412', '0512', '05512', '0612', '0712', "07512", "0812", "0912"]
parser = argparse.ArgumentParser()
parser.add_argument('--plotStyle', type=str, default='seaborn')
parser.add_argument('--para', type=str, default='all')
parser.add_argument('--env_id', type=str, default='gym_cancer:CancerControl-v0')
parser.add_argument('--seed', type=int, default= 0) # random.randint(0,100000))
parser.add_argument('--patients_pars', type=dict, default = patient_pars)
parser.add_argument('--patients_train', type=list, default=patient_train)
parser.add_argument('--number', '-n', type=int, help='Patient No., int type, requested', default=6)
parser.add_argument('--patientNo', type=str, default="patient006")
parser.add_argument('--policyPath', type=str, default='./')
parser.add_argument('--metasPar', type = str, default="0412")
args = parser.parse_args()
for i in range(len(paraSetting)):
    paras = paraSetting[i]
    args.metasPar = paras
    args.policyPath = './Patient006Training/' + paras + "/actor/"
    d, _, _ = run(args)
    DoseList.append(d)
    print(args.metasPar)
# DoseDf = pd.DataFrame(np.array(DoseList).reshape(-1, len(paraSetting)),index =paraSetting,  columns=['CPATotalDosage', 'CPAHolidayMonth', 'CPAFreePercentage', 'LEUTotalDosage', 'LEUHolidayMonth', 'LEUFreePercentage'])
# DoseDf.to_csv('./Analysis_patient006_RL_DoseHistory.csv')

# if __name__ == '__main__':
#     parsdir = "../Model_creation/test-sigmoid/model_pars"
#     parslist = os.listdir(parsdir)
#     patient_pars = {}
#     patient_test = []
#     patient_train = []
#     # reading the ode parameters and the initial/terminal states
#     for args in parslist:
#         pars_df = pd.read_csv(parsdir + '/' + args)
#         patient = args[5:(-4)]
#         patient_train.append(patient)
#         if patient not in patient_test:
#             patient_pars[patient] = pars_df
#
#     env_dict = gym.envs.registration.registry.env_specs.copy()
#     for env in env_dict:
#         if 'CancerControl-v0' in env:
#             print("Remove {} from registry".format(env))
#             del gym.envs.registration.registry.env_specs[env]
#
#     DoseList = []
#     parser = argparse.ArgumentParser()
#     parser.add_argument('--plotStyle', type=str, default='seaborn')
#     parser.add_argument('--para', type=str, default='all')
#     parser.add_argument('--env_id', type=str, default='gym_cancer:CancerControl-v0')
#     parser.add_argument('--seed', type=int, default= 0) # random.randint(0,100000))
#     parser.add_argument('--patients_pars', type=dict, default = patient_pars)
#     parser.add_argument('--patients_train', type=list, default=patient_train)
#     parser.add_argument('--number', '-n', type=int, help='Patient No., int type, requested', default=1)
#     parser.add_argument('--patientNo', type=str, default="patient001")
#     parser.add_argument('--policyPath', type=str, default="./Actor/")
#     parser.add_argument('--metasPar', type = str, default=None)
#     args = parser.parse_args()
#     patientKeys = list(patient_pars.keys())
#     NewpatientKeys = []
#     for patientNo in patientKeys:
#         try:
#             args.patientNo = patientNo
#             d1, d2, d3 = run(args)
#             DoseList.append(d1)
#             NewpatientKeys.append(patientNo)
#             d2.to_csv("./PatientDose/" + patientNo + "_doseSeq.csv")
#             d3.to_csv("./PatientAction/" + patientNo + "_actSeq.csv")
#         except (FileNotFoundError, KeyError):
#             # patientKeys.remove(patientNo)
#             print(patientNo)
#     DoseDf = pd.DataFrame(np.array(DoseList).reshape(-1, 7), index = NewpatientKeys,columns=['CPATotalDosage', 'CPAHolidayMonth', 'CPAFreePercentage', 'LEUTotalDosage', 'LEUHolidayMonth', 'LEUFreePercentage', "TotalMonth"])
#     DoseDf.to_csv('./RL_DoseHistory.csv')
#
# #
# #
# #
# #
# #
# #
# #

max_stars_repo_name: Michaelrising/PPO-PyTorch

max_issues_repo_name: Michaelrising/PPO-PyTorch

max_forks_repo_name: Michaelrising/PPO-PyTorch

reason: import numpy,from scipy


text:
import tensorflow as tf
import librosa
import numpy as np
import soundfile as sf
def test_20ms_pred():
    model=tf.saved_model.load('vad_model')
    wav = librosa.load('./test.wav', 8000)[0]
    T = (len(wav) // 160) * 160
    wav = wav[:T]
    wav = wav.reshape([-1, 2, 80])
    sil_pred,wav_pred = model.inference(wav)
    wav_pred = wav_pred.numpy().flatten()
    wav_pred/=np.abs(wav_pred).max()
    output = tf.nn.sigmoid(sil_pred)
    vad_pred = output.numpy().flatten()
    T = len(vad_pred) // 10 * 10
    vad_pred = vad_pred[:T]
    vad_pred = vad_pred.reshape([-1, 10])
    vad_pred = np.mean(vad_pred, -1)
    vad_pred = vad_pred.tolist()
    for idx, i in enumerate(vad_pred):
        if i >= 0.5:
            print(idx * 0.1, 's', 'voice')
        else:
            print(idx*0.1,'s','sil')
    sf.write('20ms_model_out.wav',wav_pred,8000)

def test_100ms_pred():
    model=tf.saved_model.load('vad_model')
    wav = librosa.load('./test.wav', 8000)[0]
    T = (len(wav) // 800) * 800
    wav = wav[:T]
    wav = wav.reshape([-1, 10, 80])
    sil_pred,wav_pred = model.inference(wav)
    wav_pred = wav_pred.numpy().flatten()
    wav_pred/=np.abs(wav_pred).max()
    output = tf.nn.sigmoid(sil_pred)
    vad_pred = output.numpy().flatten()
    T = len(vad_pred) // 10 * 10
    vad_pred = vad_pred[:T]
    vad_pred = vad_pred.reshape([-1, 10])
    vad_pred = np.mean(vad_pred, -1)
    vad_pred = vad_pred.tolist()
    for idx, i in enumerate(vad_pred):
        if i >= 0.5:
            print(idx * 0.1, 's', 'voice')
        else:
            print(idx*0.1,'s','sil')
    sf.write('100ms_model_out.wav',wav_pred,8000)
def test_200ms_pred():
    model=tf.saved_model.load('vad_model')
    wav = librosa.load('./test.wav', 8000)[0]
    T = (len(wav) // 1600) * 1600
    wav = wav[:T]
    wav = wav.reshape([-1, 20, 80])
    sil_pred,wav_pred = model.inference(wav)
    wav_pred = wav_pred.numpy().flatten()
    wav_pred/=np.abs(wav_pred).max()
    output = tf.nn.sigmoid(sil_pred)
    vad_pred = output.numpy().flatten()
    T = len(vad_pred) // 10 * 10
    vad_pred = vad_pred[:T]
    vad_pred = vad_pred.reshape([-1, 10])
    vad_pred = np.mean(vad_pred, -1)
    vad_pred = vad_pred.tolist()
    for idx, i in enumerate(vad_pred):
        if i >= 0.5:
            print(idx * 0.1, 's', 'voice')
        else:
            print(idx*0.1,'s','sil')
    sf.write('200ms_model_out.wav',wav_pred,8000)
def test_all_pred():
    model=tf.saved_model.load('vad_model')
    wav = librosa.load('./test.wav', 8000)[0]
    T = (len(wav) // 80) * 80
    wav = wav[:T]
    wav = wav.reshape([1, -1, 80])
    sil_pred,wav_pred = model.inference(wav)
    wav_pred = wav_pred.numpy().flatten()
    wav_pred/=np.abs(wav_pred).max()
    output = tf.nn.sigmoid(sil_pred)
    vad_pred = output.numpy().flatten()
    T = len(vad_pred) // 10 * 10
    vad_pred = vad_pred[:T]
    vad_pred = vad_pred.reshape([-1, 10])
    vad_pred = np.mean(vad_pred, -1)
    vad_pred = vad_pred.tolist()
    for idx, i in enumerate(vad_pred):
        if i >= 0.5:
            print(idx * 0.1, 's', 'voice')
        else:
            print(idx*0.1,'s','sil')
    sf.write('all_model_out.wav',wav_pred,8000)
if __name__ == '__main__':
    test_100ms_pred()


text:
import random

import numpy as np
import torch

def createDB():
    max = 10
    min = -10
    size = 19000
    x = (max - min) * torch.rand((size, 2)) + min
    t = []
    w = []

    for i in x:
        t.append(i[0])
        w.append(i[1])

    t = torch.tensor(t)
    w = torch.tensor(w)

    f = torch.sin(t + (w / np.pi))
    p = torch.column_stack((x, f))
    print(x)
    print(f)
    print(p)
    torch.save(p, "mydataset.dat")
createDB()

max_stars_repo_name: PaulMardar/mi-casa-es-su-casa

max_issues_repo_name: PaulMardar/mi-casa-es-su-casa

max_forks_repo_name: PaulMardar/mi-casa-es-su-casa


text:
from PyMieSim.Source import GaussianBeam
import matplotlib.pyplot as plt
from matplotlib.offsetbox import AnchoredText
import numpy as np

""" Results shoudl be the same as ref[2] figure 2. """

beam = GaussianBeam(Wavelength   = 0.6328e-6,
                    NA           = 0.2,
                    Polarization = 0,
                    Offset       = [5e-6,3e-6,0])

angle0, val0 = beam.Anm_integrand(20,2, Sampling=500)

angle1, val1 = beam.Anm_integrand(20,15, Sampling=500)

fig = plt.figure(figsize=(8,5))
ax0 = fig.add_subplot(211)
ax1 = fig.add_subplot(212)
ax0.plot(np.rad2deg(angle0), val0.real,'k', label=r"$\mathcal{Re}$ {$A_{nm}$}")
ax0.plot(np.rad2deg(angle0), val0.imag,'r--', label=r"$\mathcal{Im}$ {$A_{nm}$}")

ax1.plot(np.rad2deg(angle1), val1.real,'k', label=r"$\mathcal{Re}$ {$A_{nm}$}")
ax1.plot(np.rad2deg(angle1), val1.imag,'r--', label=r"$\mathcal{Im}$ {$A_{nm}$}")

dataStr0 = \
f"""\
n = 20; m = 2
$x_0$ = {beam.offset[0]*1e6} $\mu$m
$y_0$ = {beam.offset[1]*1e6}  $\mu$m
$z_0$ = {beam.offset[2]*1e6} $\mu$m
$w_0$ = {beam.w0*1e6:.2f} $\mu$m\
"""

anchored_text0 = AnchoredText(dataStr0, loc=4)

dataStr1 = \
f"""\
n = 20; m = 15
$x_0$ = {beam.offset[0]*1e6} $\mu$m
$y_0$ = {beam.offset[1]*1e6}  $\mu$m
$z_0$ = {beam.offset[2]*1e6} $\mu$m
$w_0$ = {beam.w0*1e6:.2f} $\mu$m\
"""

anchored_text1 = AnchoredText(dataStr1, loc=4)

ax0.grid()
ax1.grid()
ax0.legend(loc=3)
ax1.legend(loc=3)
ax0.add_artist(anchored_text0)
ax1.add_artist(anchored_text1)
ax0.set_ylabel(r'Integrand of $A_{nm}$')
ax1.set_ylabel(r'Integrand of $A_{nm}$')
ax1.set_xlabel(r'Angle $\theta$ [degree]')
plt.show()

"""
6283.185307179585
6283.185307179585
6283.185307179585
6283.185307179585
6283.185307179585

"""

max_stars_repo_name: paaube/PyMieSim

max_issues_repo_name: paaube/PyMieSim

max_forks_repo_name: paaube/PyMieSim


text:
#!/usr/bin/env python3

# https://www.reddit.com/r/compsci/comments/ahilne/sorting_pictures_by_shade/eef6wq1/?st=jr5xjy84

from os import listdir, rename
from os.path import isfile

import cv2
import numpy as np

def calc_brightness(filename):
    img = cv2.imread(filename, cv2.IMREAD_GRAYSCALE)
    return np.mean(img)

files = [x for x in listdir() if isfile(x) and not x.endswith(".py")]
brightnesses = [calc_brightness(x) for x in files]
pairs = sorted(zip(brightnesses, files))

for i, (bval, name) in enumerate(pairs):
    print(f"{bval:5.1f} {name}")
    rename(name, f"{i:03} - {name}")

max_stars_repo_name: YodaEmbedding/experiments

max_issues_repo_name: YodaEmbedding/experiments

max_forks_repo_name: YodaEmbedding/experiments


text:
import nltk
import os
import torch
import torch.utils.data as data
from vocabulary import Vocabulary
from PIL import Image
from pycocotools.coco import COCO
import numpy as np
from tqdm import tqdm
import random
import json

def get_loader(transform,
               mode='train',
               batch_size=1,
               vocab_threshold=None,
               vocab_file='./vocab.pkl',
               start_word="<start>",
               end_word="<end>",
               unk_word="<unk>",
               vocab_from_file=True,
               num_workers=0,
               cocoapi_loc='data'):
    """Returns the data loader.
    Args:
      transform: Image transform.
      mode: One of 'train' or 'test'.
      batch_size: Batch size (if in testing mode, must have batch_size=1).
      vocab_threshold: Minimum word count threshold.
      vocab_file: File containing the vocabulary.
      start_word: Special word denoting sentence start.
      end_word: Special word denoting sentence end.
      unk_word: Special word denoting unknown words.
      vocab_from_file: If False, create vocab from scratch & override any existing vocab_file.
                       If True, load vocab from from existing vocab_file, if it exists.
      num_workers: Number of subprocesses to use for data loading
      cocoapi_loc: The location of the folder containing the COCO API: https://github.com/cocodataset/cocoapi
    """

    assert mode in ['train', 'test'], "mode must be one of 'train' or 'test'."
    if vocab_from_file==False: assert mode=='train', "To generate vocab from captions file, must be in training mode (mode='train')."

    # Based on mode (train, val, test), obtain img_folder and annotations_file.
    if mode == 'train':
        if vocab_from_file==True: assert os.path.exists(vocab_file), "vocab_file does not exist.  Change vocab_from_file to False to create vocab_file."

        img_folder = os.path.join(cocoapi_loc, 'images/train2014')
        # annotations_file = os.path.join(cocoapi_loc, 'annotations\captions_train2014.json')
        annotations_file = 'data/annotations/captions_train2014.json'
    if mode == 'test':
        assert batch_size==1, "Please change batch_size to 1 if testing your model."
        assert os.path.exists(vocab_file), "Must first generate vocab.pkl from training data."
        assert vocab_from_file==True, "Change vocab_from_file to True."
        img_folder = os.path.join(cocoapi_loc, 'images/test2014')
        # annotations_file = os.path.join(cocoapi_loc, '\annotations\image_info_test2014.json')
        annotations_file = 'data/annotations/image_info_test2014.json'

    print('annotations_file: ', annotations_file)
    print('vocab_from_file: ', vocab_from_file)

    # COCO caption dataset.
    dataset = CoCoDataset(transform=transform,
                          mode=mode,
                          batch_size=batch_size,
                          vocab_threshold=vocab_threshold,
                          vocab_file=vocab_file,
                          start_word=start_word,
                          end_word=end_word,
                          unk_word=unk_word,
                          annotations_file=annotations_file,
                          vocab_from_file=vocab_from_file,
                          img_folder=img_folder)

    if mode == 'train':
        # Randomly sample a caption length, and sample indices with that length.
        indices = dataset.get_train_indices()
        # Create and assign a batch sampler to retrieve a batch with the sampled indices.
        initial_sampler = data.sampler.SubsetRandomSampler(indices=indices)
        # data loader for COCO dataset.
        data_loader = data.DataLoader(dataset=dataset,
                                      num_workers=num_workers,
                                      batch_sampler=data.sampler.BatchSampler(sampler=initial_sampler,
                                                                              batch_size=dataset.batch_size,
                                                                              drop_last=False))
    else:
        data_loader = data.DataLoader(dataset=dataset,
                                      batch_size=dataset.batch_size,
                                      shuffle=True,
                                      num_workers=num_workers)

    return data_loader

class CoCoDataset(data.Dataset):

    def __init__(self, transform, mode, batch_size, vocab_threshold, vocab_file, start_word,
        end_word, unk_word, annotations_file, vocab_from_file, img_folder):
        self.transform = transform
        self.mode = mode
        self.batch_size = batch_size
        self.vocab = Vocabulary(vocab_threshold, vocab_file, start_word, end_word, unk_word, annotations_file, vocab_from_file)
        self.img_folder = img_folder
        if self.mode == 'train':
            self.coco = COCO(annotations_file)
            self.ids = list(self.coco.anns.keys())
            print('Obtaining caption lengths...')
            all_tokens = [nltk.tokenize.word_tokenize(str(self.coco.anns[self.ids[index]]['caption']).lower()) for index in tqdm(np.arange(len(self.ids)))]
            self.caption_lengths = [len(token) for token in all_tokens]
        else:
            test_info = json.loads(open(annotations_file).read())
            self.paths = [item['file_name'] for item in test_info['images']]

    def __getitem__(self, index):
        # obtain image and caption if in training mode
        if self.mode == 'train':
            ann_id = self.ids[index]
            caption = self.coco.anns[ann_id]['caption']
            img_id = self.coco.anns[ann_id]['image_id']
            path = self.coco.loadImgs(img_id)[0]['file_name']

            # Convert image to tensor and pre-process using transform
            image = Image.open(os.path.join(self.img_folder, path)).convert('RGB')
            image = self.transform(image)

            # Convert caption to tensor of word ids.
            tokens = nltk.tokenize.word_tokenize(str(caption).lower())
            caption = []
            caption.append(self.vocab(self.vocab.start_word))
            caption.extend([self.vocab(token) for token in tokens])
            caption.append(self.vocab(self.vocab.end_word))
            caption = torch.Tensor(caption).long()

            # return pre-processed image and caption tensors
            return image, caption

        # obtain image if in test mode
        else:
            path = self.paths[index]

            # Convert image to tensor and pre-process using transform
            PIL_image = Image.open(os.path.join(self.img_folder, path)).convert('RGB')
            orig_image = np.array(PIL_image)
            image = self.transform(PIL_image)

            # return original image and pre-processed image tensor
            return orig_image, image

    def get_train_indices(self):
        sel_length = np.random.choice(self.caption_lengths)
        all_indices = np.where([self.caption_lengths[i] == sel_length for i in np.arange(len(self.caption_lengths))])[0]
        indices = list(np.random.choice(all_indices, size=self.batch_size))
        return indices

    def __len__(self):
        if self.mode == 'train':
            return len(self.ids)
        else:
            return len(self.paths)


text:
#! /usr/bin/python
import sys
import numpy
fh=sys.stdin

# Two dictionaries to maintain stats across stalls
duration_stats = dict();
position_stats = dict();

for line in fh.readlines():
  records=line.split()
  stall_num      = float(records[0])
  stall_duration = float(records[1])
  stall_position = float(records[2])
  if (stall_num in duration_stats) :
    duration_stats[stall_num] += [stall_duration];
    position_stats[stall_num] += [stall_position];
  else :
    duration_stats[stall_num] = [stall_duration];
    position_stats[stall_num] = [stall_position];

for stall_num in duration_stats:
  print "Duration stats: stall #", stall_num,"mean stall duration: ",numpy.mean(duration_stats[stall_num]),"ms","sd stall duration: ",numpy.std(duration_stats[stall_num]),"ms", len(duration_stats[stall_num]), "samples"

for stall_num in position_stats:
  print "Position stats: stall #", stall_num,"mean stall position: ",numpy.mean(position_stats[stall_num]),"sd stall position: ",numpy.std(position_stats[stall_num]), len(position_stats[stall_num]), "samples"