<jupyter_start><jupyter_text>Create data slice for conditional GANNov 23, 2020<jupyter_code>import argparse
import os
# import random
import logging
import sys

import torch
import torch.nn as nn
import torch.nn.parallel
import torch.backends.cudnn as cudnn
import torch.optim as optim
import torch.utils.data
from torchsummary import summary
from torch.utils.data import DataLoader, TensorDataset

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.animation as animation
from IPython.display import HTML

import time
from datetime import datetime
import glob
import pickle
import yaml

main_dir='/global/cfs/cdirs/m3363/vayyar/cosmogan_data/raw_data/128_square/dataset_5_4univ_cgan/'
def f_get_data():
    num_imgs=200
    for count,sigma in enumerate([0.5,0.65,0.8,1.1]):
        ip_fname=main_dir+'norm_1_sig_%s_train_val.npy'%(sigma)
        x=np.load(ip_fname,mmap_mode='r')[:num_imgs].transpose(0,1,2,3)
        size=x.shape[0]
        y=count*np.ones(size)

        if count ==0: 
            img=x[:]
            lab=y[:]
        else: 
            img=np.vstack([img,x])
            lab=np.hstack([lab,y])

    t_img=torch.from_numpy(img)
    labels=torch.LongTensor(lab).view(size*4,1,1)
    print("%s, %s"%(labels.shape,t_img.shape))

# dataset=TensorDataset(t_img,labels)
# dataloader=DataLoader(dataset,batch_size=batch_size,shuffle=True,num_workers=1,drop_last=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1111111111111111, 'fraction_numerical': 0.026224982746721876, 'mean_word_length': 4.390334572490707, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '531568', 'license': 'BSD-3-Clause-LBNL', 'n_tokens_mistral': 546, 'n_tokens_neox': 527, 'n_words': 102, 'path': 'code/3_cond_GAN/0_create_data.ipynb', 'repo_name': 'vmos1/cosmogan_pytorch', 'size': '1449'}