<jupyter_start><jupyter_text>pytorch dataloadersThis notebook is used to develop the pytorch dataset and correspond dataloaders.pytorch dataloaders use a well-defined pytorch dataset to handle the process of generating training/testing/validation sets. The pytorch dataset is just a class that contains two methods, `__len__()` and `__getitem__`. The `len` method just returns the size of the dataset and the `getitem` method returns a single sample and its corresponding label.<jupyter_code># %matplotlib widget
import glob
import os

from matplotlib.backends.backend_pdf import PdfPages
import matplotlib.pyplot as plt
from matplotlib.gridspec import GridSpec

import numpy as np
import pandas as pd
from pyts.image import RecurrencePlot
import torch
from torch.utils.data import Dataset, DataLoader
from torchvision.transforms import ToTensor

import sys
sys.path.append('/Users/ndmiles/ClassWork/FallQuarter2021/aos_c204/aos_c204_final_project')

from utils import generate_data_chunks
class ICMEDataset(Dataset):
    def __init__(self, icme_labels, rootdir, datadir, transform=None):

        self.rootdir = rootdir
        self.datadir = datadir
        self.df = pd.read_csv(
            f'{rootdir}/data/{icme_labels}',
            header=0, 
            parse_dates=['start_time', 'stop_time']
        )
        self.transform=transform
        
    def __len__(self):
        return len(self.df)
    
    def __getitem__(self, idx):
        if torch.is_tensor(idx):
            idx = idx.tolist()
        f = self.df.fname_img.iloc[idx]
        f_ts = self.df.fname.iloc[idx]
        label = self.df.label.iloc[idx]
        img = np.load(f'{self.datadir}/{f}')

        return img, label, f_ts
icme_train_dataset = ICMEDataset(
    'sta_train_set.txt', 
    rootdir=(
        '/Users/ndmiles/ClassWork/FallQuarter2021/'
        'aos_c204/aos_c204_final_project/'
    ),
    datadir=(
        '/Users/ndmiles/ClassWork/FallQuarter2021/'
        'aos_c204/aos_c204_final_project/data/sta_chunks/'
    )
)
icme_test_dataset = ICMEDataset(
    'sta_test_set.txt', 
    rootdir=(
        '/Users/ndmiles/ClassWork/FallQuarter2021/'
        'aos_c204/aos_c204_final_project/'
    ),
    datadir=(
        '/Users/ndmiles/ClassWork/FallQuarter2021/'
        'aos_c204/aos_c204_final_project/data/sta_chunks/'
    )
)
icme_val_dataset = ICMEDataset(
    'sta_validation_set.txt', 
    rootdir=(
        '/Users/ndmiles/ClassWork/FallQuarter2021/'
        'aos_c204/aos_c204_final_project/'
    ),
    datadir=(
        '/Users/ndmiles/ClassWork/FallQuarter2021/'
        'aos_c204/aos_c204_final_project/data/sta_chunks/'
    )
)
train_loader = DataLoader(
    icme_train_dataset, 
    batch_size=1, 
    shuffle=True,
    num_workers=0
)
test_loader = DataLoader(
    icme_test_dataset, 
    batch_size=1, 
    shuffle=True,
    num_workers=0
)
val_loader = DataLoader(
    icme_val_dataset, 
    batch_size=1, 
    shuffle=True,
    num_workers=0
)
for img, label, fname_ts in train_loader:
    print(img.shape, fname_ts)
t_df = pd.read_csv('../data/sta_chunks/sta_ts_interval_2012-04-16_12_02_00_to_2012-04-19_12_02_00.txt', header=0, index_col=0, parse_dates=True)
t_df.head()
t_df.shape
t_df.plot(y='BTOTAL')
img1, label1 = next(iter(train_loader))
img2, label2 = next(iter(train_loader))
def plot_img(img, label, cols=None):
    img_dict = dict()
#     img, label = next(iter(loader))
    if cols is None:
        cols = [f'{i:0.0f}' for i in range(len(img[0]))]
    for i, col in enumerate(cols):
        img_dict[col] = [img[0][i].numpy()]
    fig = generate_data_chunks.visualize_chunk_img(img_dict, icme=label.numpy()[0])
    return fig
pdf = PdfPages('icmes_images_trainining.pdf', 'a')
cols = (
    'BTOTAL',
    'BX(RTN)',
    'BY(RTN)',
    'BZ(RTN)',
    'VP_RTN',
    'NP'
    'TEMPERATURE',
    'BETA'
)
for img, label in train_loader:
    if label == 1:
        fig = plot_img(img, label, cols=cols)
        pdf.savefig(bbox_inches='tight', dpi=150)
        plt.close(fig)
pdf.close()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11353495290034706, 'fraction_numerical': 0.029747149231531978, 'mean_word_length': 3.3201284796573876, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '529820', 'license': 'MIT', 'n_tokens_mistral': 1556, 'n_tokens_neox': 1490, 'n_words': 279, 'path': 'notebooks/prototyping_pytorch_dataloader.ipynb', 'repo_name': 'nmiles2718/aos_c204_final_project', 'size': '4034'}