<jupyter_start><jupyter_text>Packages<jupyter_code>%load_ext autoreload
%autoreload 2

import os
import os.path as osp
from pathlib import Path
from time import time
import math

import numpy as np
import pandas as pd

from IPython.display import Audio, clear_output, display
import sys
sys.path.append("../scripts/")
import data_loader as dl
import torch
from torch.utils.data import Dataset, DataLoader

from torch import nn
from torch.nn import functional as F

import pytorch_lightning as pl
from pytorch_lightning import Trainer
from pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping
from pytorch_lightning.loggers import TensorBoardLogger<jupyter_output><empty_output><jupyter_text>Arguments & User Defined Functions<jupyter_code>transcript_path = "../outputs/all_transcripts.csv"
transcripts = pd.read_csv(transcript_path)

# # Only for sample purposes:
# file_path = "142-orig.wav"
# file_transcripts = transcripts.loc[transcripts["file"] == file_path]

bert_scores_path = "../outputs/bert_scores.csv"
bert_scores = pd.read_csv(bert_scores_path)

transcripts = transcripts.merge(bert_scores, on=["file", "line"])
transcripts["line"] = transcripts["line"].astype(str)

data_path = "../outputs/npy"

summary_data_path = "../outputs/nn_summary_info.csv"
summary_data = pd.read_csv(summary_data_path)

seed = 42
batch_size = 128
num_workers = 8
sequence_len = 2048
n_features = 5

write_dir = "../outputs/splits/"
if not osp.exists(write_dir):
    os.makedirs(write_dir)
npys = [path for path in Path(data_path).rglob("*.npy")]

npys_name = [n.name for n in npys]
npys_name = pd.DataFrame(npys_name, columns=["name"])
sequences = npys_name.loc[~npys_name["name"].str.contains("shape")].copy()
shapes = npys_name.loc[npys_name["name"].str.contains("shape")].copy()

print("Sequence Arrays:", len(sequences))
sequences["splits"] = sequences["name"].apply(lambda x: x.split("_"))
sequences["file"] = sequences["splits"].apply(lambda x: x[0])
sequences["line"] = sequences["splits"].apply(lambda x: x[1].split(".")[0])

transcripts = transcripts.merge(
    sequences[["file", "line"]], on=["file", "line"]
).reset_index(drop=True)

print("Matching Sequences:", len(transcripts))
try:
  transcripts = transcripts.drop("Unnamed: 0", axis = 1)
except:
  pass
transcripts.sample(n=4)

transcripts.to_csv("../outputs/valid_transcripts.csv", index=False)

transcripts = pd.read_csv("../outputs/valid_transcripts.csv")
transcripts = transcripts.merge(summary_data, on=["file", "line"])
print(transcripts.shape)<jupyter_output><empty_output><jupyter_text>Dataset<jupyter_code># PyTorch Dataset
# Windows devices may have incompatability with the dataset defined in the notebook, please use the `../scripts/data_loader.py` import instead.

class AudioDataset(Dataset):
    def __init__(
        self,
        metadata,
        data_dir,
        y_name="gs_score",
        trunc_pad_len=2048,
        in_dim=35,
    ):
        super().__init__()
        self.metadata = metadata

        # Faster than using a .loc on column names directly
        self.columns_dict = dict([(c, i) for i, c in enumerate(self.metadata.columns)])
        self.data_dir = data_dir
        self.y_name = y_name
        self.trunc_pad_len = trunc_pad_len
        self.in_dim = in_dim

    def __len__(self):
        return len(self.metadata)

    def __getitem__(self, idx):
        # Get row with .iloc
        row = self.metadata.iloc[idx]

        means_stds = [
            c for c in self.columns_dict.keys() if ("mean" in c) or ("std" in c)
        ]

        # self.in_dim += len(means_stds)

        # Read in pre-computed numpy array
        file_name = row[self.columns_dict["file"]]
        line_name = row[self.columns_dict["line"]]
        npy_path = osp.join(self.data_dir, f"{file_name}_{line_name}.npy")
        data = np.load(npy_path)

        # Get y_true
        score = row[self.columns_dict[self.y_name]]

        # Get averages and standard deviations of the features before padding.
        summary_arr = row[[self.columns_dict[c] for c in means_stds]].values
        summary_arr_tiled = np.tile(summary_arr, (data.shape[0], 1))

        data = np.concatenate([data, summary_arr_tiled], axis=1)

        # Pad/Truncate
        data_aug = np.zeros((self.trunc_pad_len, self.in_dim))
        data_aug[: min(data.shape[0], self.trunc_pad_len), :] = data[
            : self.trunc_pad_len
        ]
        item = {
            "x": torch.tensor(data_aug, dtype=torch.float),
            "y": torch.tensor([score], dtype=torch.float),
        }

        return (item["x"], item["y"])<jupyter_output><empty_output><jupyter_text>Data Module<jupyter_code># PyTorch Lightning Module
class AudioDataModule(pl.LightningDataModule):
    def __init__(self,
                 metadata,
                 data_maxes = np.load("../outputs/data_maxes.npy"),
                 split_directory="../outputs/splits",
                 data_directory="../outputs/npy",
                 seq_len=2048,
                 num_features=5,
                 y_col="gs_score",
                 batch_size=128,
                 num_workers=4,
                 seed=42
                ):
        super().__init__()
        self.metadata = metadata
        self.data_maxes = data_maxes
        self.split_directory = split_directory
        self.data_directory = data_directory
        self.seq_len = seq_len
        self.num_features = num_features
        self.y_col = y_col
        self.batch_size = batch_size
        self.num_workers = num_workers
        self.seed = seed
        
    def prepare_data(self):
        rng = np.random.default_rng(self.seed)
        indices = rng.permutation(self.metadata.shape[0])
        train_size = math.floor(len(indices) * 0.80)
        val_size = math.floor(len(indices) * 0.10)
        train_idx = indices[:train_size]
        val_idx = indices[train_size : train_size + val_size]
        test_idx = indices[train_size + val_size :]
    
        self.train = self.metadata.iloc[train_idx].reset_index(drop=True)
        self.train.to_csv(osp.join(self.split_directory, "train.csv"), index=False)

        self.val = self.metadata.iloc[val_idx].reset_index(drop=True)
        self.val.to_csv(osp.join(self.split_directory, "val.csv"), index=False)

        self.test = self.metadata.iloc[test_idx].reset_index(drop=True)
        self.test.to_csv(osp.join(self.split_directory, "test.csv"), index=False)
        
    def setup(self):
        self.train_data = pd.read_csv(osp.join(self.split_directory, "train.csv"))
        self.val_data = pd.read_csv(osp.join(self.split_directory, "val.csv"))
        self.test_data = pd.read_csv(osp.join(self.split_directory, "test.csv"))
    
    def train_dataloader(self):
        self.train_dataset = dl.NewAudioDataset(metadata=self.train_data,
            data_maxes = self.data_maxes,
            data_directory=self.data_directory,
            num_features= self.num_features,
            seq_len = self.seq_len,
            y_col=self.y_col)
        train_loader = DataLoader(self.train_dataset, 
                                  batch_size = self.batch_size, 
                                  shuffle = False, 
                                  num_workers = self.num_workers)
        
        return train_loader

    def val_dataloader(self):
        self.val_dataset = dl.NewAudioDataset(metadata=self.val_data,
            data_maxes = self.data_maxes,
            data_directory=self.data_directory,
            num_features= self.num_features,
            seq_len = self.seq_len,
            y_col=self.y_col)
        val_loader = DataLoader(self.val_dataset, 
                                batch_size = self.batch_size, 
                                shuffle = False, 
                                num_workers = self.num_workers)

        return val_loader

    def test_dataloader(self):
        self.test_dataset = dl.NewAudioDataset(
            metadata=self.test_data,
            data_maxes = self.data_maxes,
            data_directory=self.data_directory,
            num_features= self.num_features,
            seq_len = self.seq_len,
            y_col=self.y_col)
        test_loader = DataLoader(self.test_dataset, 
                                 batch_size = self.batch_size, 
                                 shuffle = False, 
                                 num_workers = self.num_workers)

        return test_loader
dm = AudioDataModule(metadata=transcripts,
    data_maxes = np.load("../outputs/data_maxes.npy"),                    
    split_directory="../outputs/splits",
    data_directory="../outputs/npy",
    seq_len=2048,
    num_features=5,
    y_col="gs_score",
    batch_size=128,
    num_workers=4,
    seed=42
)

dm.prepare_data()
dm.setup()
one_batch = next(iter(dm.train_dataloader()))
print(one_batch[0].size())
print(one_batch[1].size())<jupyter_output>torch.Size([128, 2048, 5])
torch.Size([128])<jupyter_text>Model Module<jupyter_code>class LSTMRegressor(pl.LightningModule):
    '''
    Standard PyTorch Lightning module:
    https://pytorch-lightning.readthedocs.io/en/latest/lightning_module.html
    '''
    def __init__(self, 
                 n_features, 
                 hidden_size, 
                 seq_len, 
                 batch_size,
                 num_layers, 
                 dropout, 
                 learning_rate,
                 criterion):
        super(LSTMRegressor, self).__init__()
        self.n_features = n_features
        self.hidden_size = hidden_size
        self.seq_len = seq_len
        self.batch_size = batch_size
        self.num_layers = num_layers
        self.dropout = dropout
        self.criterion = criterion
        self.learning_rate = learning_rate

        self.lstm = nn.LSTM(input_size=n_features, 
                            hidden_size=hidden_size,
                            num_layers=num_layers, 
                            dropout=dropout, 
                            batch_first=True)
        self.linear = nn.Linear(hidden_size, 1)
        
    def forward(self, x):
        # lstm_out = (batch_size, seq_len, hidden_size)
        lstm_out, _ = self.lstm(x)
        y_pred = self.linear(lstm_out[:,-1])
        return y_pred
    
    def configure_optimizers(self):
        return torch.optim.Adam(self.parameters(), lr=self.learning_rate)

    def training_step(self, batch, batch_idx):
        x, y = batch
        y = y.unsqueeze(1)
        y_hat = self(x)
        loss = self.criterion(y_hat, y)
        self.log("train_loss", loss, prog_bar=True, logger=True)
        return loss

    def validation_step(self, batch, batch_idx):
        x, y = batch
        y = y.unsqueeze(1)
        y_hat = self(x)
        loss = self.criterion(y_hat, y)
        self.log('val_loss', loss, prog_bar=True, logger=True)
        return loss
    
    def test_step(self, batch, batch_idx):
        x, y = batch
        y = y.unsqueeze(1)
        y_hat = self(x)
        loss = self.criterion(y_hat, y)
        self.log('test_loss', loss, prog_bar=True, logger=True)
        return loss
"""
All parameters are aggregated in one place.
This is useful for reporting experiment params to experiment tracking software
"""

p = dict(
    seq_len=2048,
    batch_size=128,
    criterion=nn.MSELoss(),
    max_epochs=50,
    n_features=n_features,
    hidden_size=128,
    num_layers=2,
    dropout=0,
    learning_rate=0.001,
    bidirectional=False,
    track_level_flag=False,
    convnet_flag=False,
    kernel_size=16,
    stride=1,
    conv_output=8,
)
checkpoint_callback = ModelCheckpoint(
    dirpath = "checkpoints",
    filename = "best-checkpoint", 
    save_top_k=1, 
    verbose =True, 
    monitor = "val_loss",
    mode="min"
)


logger = TensorBoardLogger("lstm", name="audio_change")
early_stopping_callback = EarlyStopping(monitor="val_loss", patience = 4)

trainer = Trainer(
    max_epochs=p['max_epochs'],
    logger=logger,
    gpus=1,
    checkpoint_callback=checkpoint_callback,
    callbacks = [early_stopping_callback],
#     overfit_batches=1
)

model = LSTMRegressor(
    n_features = p['n_features'],
    hidden_size = p['hidden_size'],
    seq_len = p['seq_len'],
    batch_size = p['batch_size'],
    criterion = p['criterion'],
    num_layers = p['num_layers'],
    dropout = p['dropout'],
    learning_rate = p['learning_rate']
)

dm = AudioDataModule(metadata=transcripts,
    data_maxes = np.load("../outputs/data_maxes.npy"),                    
    split_directory="../outputs/splits",
    data_directory="../outputs/npy",
    seq_len=2048,
    num_features=5,
    y_col="gs_score",
    batch_size=128,
    num_workers=4,
    seed=42
)

dm.setup()
dm.prepare_data()

trainer.fit(model, dm)
trainer.test(model, datamodule=dm)<jupyter_output>C:\Users\yagne\.conda\envs\grandstanding\lib\site-packages\pytorch_lightning\trainer\connectors\callback_connector.py:147: LightningDeprecationWarning: Setting `Trainer(checkpoint_callback=<pytorch_lightning.callbacks.model_checkpoint.ModelCheckpoint object at 0x0000024C8695ADF0>)` is deprecated in v1.5 and will be removed in v1.7. Please consider using `Trainer(enable_checkpointing=<pytorch_lightning.callbacks.model_checkpoint.ModelCheckpoint object at 0x0000024C8695ADF0>)`.
  rank_zero_deprecation(
GPU available: True, used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
C:\Users\yagne\.conda\envs\grandstanding\lib\site-packages\pytorch_lightning\core\datamodule.py:469: LightningDeprecationWarning: DataModule.setup has already been called, so it will not be called again. In v1.6 this behavior will change to always call DataModule.setup.
  rank_zero_deprecation(
LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]

  | Name      | Type    | Params
-------------[...]
{'fraction_non_alphanumeric': 0.11403636363636363, 'fraction_numerical': 0.011418181818181819, 'mean_word_length': 2.720508658008658, 'pattern_counts': {'":': 2, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '248599', 'license': 'MIT', 'n_tokens_mistral': 4620, 'n_tokens_neox': 4423, 'n_words': 802, 'path': 'notebooks/[07] neural_network_training.ipynb', 'repo_name': 'yagnesh7/grandstanding2021', 'size': '13750'}