<jupyter_start><jupyter_text>Intro in PyTorch, DevOps, R&D in Deep Learning     В данном семинаре мы познакомимся с pytorch на примере задачи классификации.  CIFAR10 and PyTorch<jupyter_code>import torch
import torchaudio
import pandas as pd
from matplotlib import pyplot as plt
from torch.utils.data import Dataset, DataLoader, random_split
# %pylab inline
# from itertools import islice
# from scipy.signal import hann
# from scipy.io import wavfile
# from torch import nn
path_to_dataset = "../resources/LJSpeech-1.1/"
path_to_metadata = path_to_dataset + "metadata.csv"
batch_size = 32
n_mels = 64

freq_mask_param = 20
time_mask_param = 20

column_names = ['wav_filename', 'text_with_numbers', 'text_in_letters'] 
max_str_len = 100
LJdata = pd.read_csv(path_to_metadata, sep='|', names=column_names, header=None)
LJdata.drop(columns=['text_with_numbers'], inplace=True)
LJdata = LJdata[LJdata['text_in_letters'].apply(lambda x: len(str(x)) <= max_str_len)]
LJdata
mel_transform = torchaudio.transforms.MelSpectrogram(
    sample_rate=22050,
    n_fft=1024,
    win_length=1024,
    hop_length=256,
    f_min=0,
    f_max=8000,
    n_mels=80)
class MyDataset(Dataset):
#     def __init__(self):
#         super(MyDataset, self).__init__()

#  -> Tuple[torch.Tensor, torch.Tensor, int, int]
    def __getitem__(self, index: int):
        wav_filename, text_in_letters = LJdata.iloc[index]
        path_to_file = path_to_dataset + "wavs/" + wav_filename + ".wav"

        waveform, sample_rate = torchaudio.load(path_to_file)
        # label = text_in_letters

        # channel, feature, time
        spectrogram = torch.log(mel_transform(waveform) + 1e-9)

        return spectrogram, text_in_letters  # , spectrogram.shape[-1] // 2, len(text_in_letters)

    def __len__(self):
        return len(LJdata)

my_dataset = MyDataset()
print(len(my_dataset))
spec, lbl = my_dataset.__getitem__(0)
print(my_dataset.__len__())
print(lbl)
print(len(lbl))
print(spec.shape[-1] // 2)

plt.figure(figsize=(20, 5))
plt.imshow(spec.squeeze())
plt.show()
dataset_length = len(my_dataset)
train_dataset_length = int(dataset_length * 0.95)
test_dataset_length = dataset_length - train_dataset_length

train_dataset, test_dataset = random_split(
    dataset=my_dataset,
    lengths=[train_dataset_length, test_dataset_length],
    generator=torch.Generator().manual_seed(21)
)
def augment(waveform):
    freq_transform = torchaudio.transforms.FrequencyMasking(freq_mask_param=freq_mask_param)
    time_transform = torchaudio.transforms.TimeMasking(time_mask_param=time_mask_param)
    return time_transform(freq_transform(waveform))
def collate_sequences(batch, is_test = False):
    specs = []
    labels = []
    for (log_mel_spec, text_in_letters) in batch:
        if not is_test:
            log_mel_spec = augment(log_mel_spec)
        specs.append(log_mel_spec.squeeze(0).transpose(0, 1))
        labels.append(text_in_letters)

    specs = nn.utils.rnn.pad_sequence(sequences=specs, batch_first=True).transpose(1, 2)
    labels = nn.utils.rnn.pad_sequence(sequences=labels, batch_first=True)

    return specs, labels
train_dataloader = DataLoader(
            train_dataset,
            batch_size=batch_size,
            collate_fn=collate_sequences,
            num_workers=4
        )

test_dataloader = DataLoader(
            test_dataset,
            batch_size=batch_size,
            collate_fn=collate_sequences,
            num_workers=4
        )
s1 = "Hello my name is Max"
t1 = torch.tensor(s1)
# mel_spec = torchaudio.transforms.MelSpectrogram(sample_rate=sample_rate, n_mels=n_mels)
plt.figure(figsize=(20, 5))

for batch in train_dataloader:
    log_mel_spec, text_in_letters = batch
    if (i % 1000 == 0):
        print(i, text_in_letters)
        plt.imshow(log_mel_spec.squeeze())
        plt.show()

def imshow(img):
    img = img / 2 + 0.5  # unnormalize
    npimg = img.numpy()
    plt.imshow(np.transpose(npimg, (1, 2, 0)))
    plt.show()
images, _ = list(islice(train_dataloader, 1))[0]

plt.figure(figsize=(10, 10))
imshow(torchvision.utils.make_grid(images))<jupyter_output><empty_output><jupyter_text>Следующая компанента это сама модель. Для примера просто возьмем сеть из туториал.<jupyter_code>from torch import nn
import torch.nn.functional as F


class Reshape(nn.Module):
    
    def __init__(self):
        super().__init__()
    
    def forward(self, x):
        # X.shape == [B, C, H, W] -> [B, H']
        return torch.flatten(x, start_dim=1)


class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        
        self.net = nn.Sequential(
            nn.Conv2d(3, 6, 5),
            nn.ReLU(True),
            nn.MaxPool2d(2, 2),
            nn.Conv2d(6, 16, 5),
            nn.ReLU(True),
            nn.MaxPool2d(2, 2),
            Reshape(),
            nn.Linear(16 * 5 * 5, 120),
            nn.ReLU(True),
            nn.Linear(120, 84),
            nn.ReLU(True),
            nn.Linear(84, 10),
        )
        
    def forward(self, x):
        return self.net(x)


net = Net()
device = torch.device('cpu')
net = net.to(device)
net = net.train()<jupyter_output><empty_output><jupyter_text>Определяем функцию потерь и метод оптимизации<jupyter_code>import torch.optim as optim

criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9,
                      nesterov=True, weight_decay=1e-5)<jupyter_output><empty_output><jupyter_text>И последняя компанента пайплайна это написание train/val loop-ов. В данном случае тут все просто, но    в больших задачах эта часть занимает много времени и кода :( Стоит присмотреться в библиотекам, которые    это делают за нас, например, pytorch-lighting<jupyter_code>for epoch in range(2):  # loop over the dataset multiple times

    running_loss = 0.0
    for i, data in enumerate(trainloader, 0):
        # get the inputs; data is a list of [inputs, labels]
        inputs, labels = data
        inputs = inputs.to(device)
        labels = labels.to(device)

        # zero the parameter gradients
        optimizer.zero_grad()

        # forward + backward + optimize
        outputs = net(inputs)
        loss = criterion(outputs, labels)
        loss.backward()
        
        optimizer.step()

        # print statistics
        running_loss += loss.item()
        if i % 2000 == 1999:  # print every 2000 mini-batches
            print('[%d, %5d] loss: %.3f' %
                  (epoch + 1, i + 1, running_loss / 2000))
            running_loss = 0.0

print('Finished Training')<jupyter_output><empty_output><jupyter_text>Посмотрим качество классификации на одном тестовом батче<jupyter_code>images, _ = list(islice(test_dataloader, 1))[0]

with torch.no_grad():
    logits = net(images)
    predicted_labels = torch.argmax(logits, dim=1)
_, axes = plt.subplots(4, 8, figsize=(25, 14), dpi=80)
axes = axes.flatten()

for ax, image, label in zip(axes, images, predicted_labels):
    image = image / 2 + 0.5
    image = image.permute(1, 2, 0)
    
    ax.set_title(classes[label.item()])
    ax.imshow(image)
    
plt.show()<jupyter_output><empty_output><jupyter_text>torchaudio    Немного познакомимся с torchaudio -- аналог torchvision, но для звука. torchaudio предоставляется возможность считать    STFT (short-time Fourier transform), Spectrograms, MelSpectrograms, а также много чего еще.<jupyter_code>import torchaudio
from scipy.signal import hann
from scipy.io import wavfile<jupyter_output><empty_output><jupyter_text>Считаем wav файл и посмотрим как он выглядит<jupyter_code>filename = "../resources/audio.wav"
waveform, sample_rate = torchaudio.load(filename)
plt.figure(figsize=(20, 5))
plt.plot(waveform.squeeze())
plt.xlabel('Time')
plt.ylabel('Amplitude')
plt.show()<jupyter_output><empty_output><jupyter_text>Short-time Fourier transform    Посмотрим на STFT из коборки. Всмомним, что STFT из себя представлят нарезанием аудио на окна с каким то шагом,    применением оконной функции, а потом применением преобразования Фурье.<jupyter_code>window = hann(1024)
plt.plot(window)
plt.show()
freqs = torch.stft(
    input=waveform,
    n_fft=1024,
    hop_length=256,
    win_length=1024,
    window=torch.from_numpy(window),
    center=False
)<jupyter_output><empty_output><jupyter_text>Чтобы избавиться от комлексной плоскости мы берем норму (p==2) по последней размерности.<jupyter_code>spectrograms = torch.norm(freqs, 2, -1)<jupyter_output><empty_output><jupyter_text>Если присмотреться -- можно увидеть белые пятна<jupyter_code>plt.imshow(spectrograms.squeeze())<jupyter_output><empty_output><jupyter_text>Spectrograms<jupyter_code>spectrogramer = torchaudio.transforms.Spectrogram(
    n_fft=1024,
    win_length=1024,
    hop_length=256,
    power=2.0,
)

spectrogram = spectrogramer(waveform)

print("Shape of spectrogram: {}".format(spectrogram.size()))

plt.figure(figsize=(20, 5))
plt.imshow(np.log(spectrogram.squeeze()))
plt.show()<jupyter_output>Shape of spectrogram: torch.Size([1, 513, 833])<jupyter_text>MelSpectrograms<jupyter_code>mel_spectrogramer = torchaudio.transforms.MelSpectrogram(
    sample_rate=22050,
    n_fft=1024,
    win_length=1024,
    hop_length=256,
    f_min=0,
    f_max=8000,
    n_mels=64,
)

mel_spectrogram = mel_spectrogramer(waveform)

print("Shape of spectrogram: {}".format(mel_spectrogram.size()))

plt.figure(figsize=(20, 5))
plt.imshow(torch.log(mel_spectrogram.squeeze()))
plt.show()<jupyter_output>Shape of spectrogram: torch.Size([1, 64, 833])
{'fraction_non_alphanumeric': 0.11484061642389698, 'fraction_numerical': 0.025543592991344732, 'mean_word_length': 3.841594276954522, 'pattern_counts': {'":': 0, '<': 48, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '301065', 'license': 'MIT', 'n_tokens_mistral': 3688, 'n_tokens_neox': 3552, 'n_words': 671, 'path': 'notebooks/.ipynb_checkpoints/seminar-checkpoint.ipynb', 'repo_name': 'MaxTheHuman/MyQuartzNet', 'size': '9474'}