<jupyter_start><jupyter_text>Text generation with deep learning<jupyter_code>import sys
import os
import os.path
import random
import collections
import shutil
import time
import glob
import csv
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from math import pi
import scipy

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.utils.data import Dataset, DataLoader
from torch.autograd import Variable

import torchvision
from torchvision import transforms, datasets



%matplotlib inline


from bokeh.io import show,output_notebook
from bokeh.models import ColumnDataSource, HoverTool, LinearColorMapper, BasicTicker, PrintfTickFormatter,ColorBar
from bokeh.models import FuncTickFormatter
from bokeh.plotting import figure

output_notebook()<jupyter_output><empty_output><jupyter_text>**Задача**   Обучить character recurrent neural network (char RNN) на датасете статей arxiv, а также на своих данных и исследовать её свойства. Определим структуру сети, в качестве основного блока используем **GRU**<jupyter_code>class RNN(nn.Module):
    def __init__(self, input_size, hidden_size, output_size, n_layers=1, batch_first=True):
        super(RNN, self).__init__()
        self.input_size = input_size
        self.hidden_size = hidden_size
        self.output_size = output_size
        self.n_layers = n_layers
        
        self.encoder = nn.Embedding(input_size, hidden_size)
        self.gru = nn.GRU(hidden_size, hidden_size, n_layers)
        self.decoder = nn.Linear(hidden_size, output_size)
    
    def forward(self, input, hidden):
        input = self.encoder(input.view(1, -1))
        output, hidden = self.gru(input.view(1, 1, -1), hidden)
        output = self.decoder(output.view(1, -1))
        return output, hidden

    def init_hidden(self):
        return torch.zeros(self.n_layers, 1, self.hidden_size)
    
    def save_model(model, filename='rnn.ckpt'):
        checkpoint = {'input_size': model.input_size,
                      'hidden_size': model.hidden_size,
                      'output_size': model.output_size,
                      'n_layers': model.n_layers,
                      'state_dict': model.state_dict()}
        with open(filename, 'wb') as f:
            torch.save(checkpoint, f)

    def load_model(filename):

        with open(filename, 'rb') as f:
            checkpoint = torch.load(f)

        model = RNN(input_size = checkpoint['input_size'], output_size = checkpoint['output_size'], hidden_size=checkpoint['hidden_size'], n_layers=checkpoint['n_layers'])
        model.load_state_dict(checkpoint['state_dict'])

        return model<jupyter_output><empty_output><jupyter_text>Пример реализации класса для получения данных из датасета<jupyter_code>class TextDataset(Dataset):
    def __init__(self, text, chunk_len=10):
        self.chunks_number = int(len(text) / chunk_len)
        print(f'chunks: {self.chunks_number}')
        self.chunk_len = chunk_len
        self.text = text
        self.all_symbols = list({x for x in self.text})
    
    def __len__(self):
        return self.chunks_number
    
    def _get_tensor(self, text):
        return torch.LongTensor(list(map(self.all_symbols.index, text))).cuda()
    
    def _decode_vector(self, text):
        text = text.view(-1).cpu().detach().numpy()
        if text.shape[0] == 1:
            text = list(text)
        print(f'x - {text[0]}')
        return ''.join([self.all_symbols[x] for x in text])
    
    def _get_random_slice(self):
        start_index = random.randint(0, len(self.text) - self.chunk_len)
        end_index = start_index + self.chunk_len + 1
        return self.text[start_index:end_index]
    
    def __getitem__(self, idx):
        chunk = self._get_random_slice()
        return self._get_tensor(chunk[:-1]), self._get_tensor(chunk[1:])<jupyter_output><empty_output><jupyter_text>Здесь должен быть код для считывания данных и создания даталоадера<jupyter_code># download the file from google-drive
!pip install --quiet gdown
!gdown --id 1ZzxFJBdyz56b1AKVRFWxw9y9UPkeC1AS
!ls -l

# import as a pandas DataFrame
dataset_raw = pd.read_csv('arxiv.csv')
dataset_raw.drop(['time', 'favorites', 'rts', 'authors', 'category', 'published', 'tweeted'], axis=1, inplace=True)
dataset_raw
summaries_list = dataset_raw['summary'].tolist()
summaries_text = ' '.join(summaries_list) 
summaries_text = summaries_text[:5000]
summaries_text = summaries_text.split()
print(summaries_text)

# Parameters
params = {
    # 'batch_size': 0,
    # 'shuffle': True,
    # 'num_workers': 1
}

torch.multiprocessing.set_start_method("spawn", force=True)

# Generators
arxiv_dataset = TextDataset(text=summaries_text, chunk_len=100)
arxiv_generator = torch.utils.data.DataLoader(arxiv_dataset, **params)<jupyter_output>['In', 'summary', 'of', 'the', 'research', 'findings', 'presented', 'in', 'this', 'paper,', 'various', 'brain', 'regions', 'are', 'correlated', 'with', 'vocabulary', 'and', 'vocabulary', 'acquisition.', 'Semantic', 'associations', 'for', 'vocabulary', 'seem', 'to', 'be', 'located', 'near', 'brain', 'areas', 'that', 'vary', 'according', 'to', 'the', 'type', 'of', 'vocabulary,', 'e.g.', 'ventral', 'temporal', 'regions', 'important', 'for', 'words', 'for', 'things', 'that', 'can', 'be', 'seen.', 'Semantic', 'processing', 'is', 'believed', 'to', 'be', 'strongly', 'associated', 'with', 'the', 'ANG.', 'Phonological', 'ability', 'has', 'been', 'closely', 'related', 'to', 'the', 'anterior', 'surfaces', 'of', 'the', 'SMG.', 'Pathways', 'through', 'the', 'posterior', 'SMG', 'are', 'thought', 'to', 'link', 'the', 'anterior', 'SMG', 'and', 'the', 'ANG.', 'In', 'vocabulary', 'tasks,', 'mediotemporal', 'structures', 'may', 'be', 'related', 'to', 'long-term', 'memory', 'processing,', 'with', 'left', [...]<jupyter_text>Смотрим, сколько уникальных символов есть в датасете - это и будет размер выходного вектора у нейронной сети<jupyter_code>unique_symbols = len(arxiv_dataset.all_symbols)
n_layers = 2
hidden_size = 100
unique_symbols
net = RNN(unique_symbols, hidden_size, unique_symbols, n_layers)
device = torch.device("cuda")
net = net.to(device)
optimizer = torch.optim.Adam(net.parameters(), lr=1e-4)
criterion = nn.CrossEntropyLoss()<jupyter_output><empty_output><jupyter_text>Здесь нужно реализовать функцию для генерации текста с помощью сети<jupyter_code>def generate(net, device, dataset, prime_str='In', predict_len=10, temperature=0.8):
    #Code to generate string based on input sequence (prime_str)
    hidden = net.init_hidden().cuda()

    for p in range(predict_len):        
        prime_input = dataset._get_tensor(prime_str.split())
        inp = prime_input[-1:] #last word as input
        output, hidden = net(inp, hidden)
        
        # Sample from the network as a multinomial distribution
        output_dist = output.data.view(-1).div(temperature).exp()
        top_i = torch.multinomial(output_dist, 1)[0]

        # Add predicted word to string and use as next input
        predicted_word = dataset.all_symbols[top_i.item()]
        prime_str += ' ' + predicted_word

    return prime_str


generate(net, device, arxiv_dataset)<jupyter_output><empty_output><jupyter_text>Здесь должен быть цикл тренировки<jupyter_code>epochs = 10000
start = time.time()
all_losses = []
temperature = 0.8

for epoch in range(1, epochs+1):
  # #Your code for training here
  for local_batch, local_labels in arxiv_generator:
    for input, label in zip(local_batch, local_labels):
      hidden = net.init_hidden().cuda()
      net.zero_grad()
      loss = 0

      # get output
      output, hidden = net(input[-1:].cuda(), hidden)
      output_dist = output.data.view(-1).div(temperature).exp()
      word_index = torch.multinomial(output_dist, 1)
      predicted_word = arxiv_dataset.all_symbols[word_index.item()]
      
      # compute loss
      loss = criterion(output, label[-1:].view(-1).long())

      loss.backward()
      optimizer.step()
      all_losses.append(loss.data.item())

print("Saving...")
RNN.save_model(net, "TrainedModel.ckpt")
net(input[-1:], hidden)[0].shape
label[-1:]
plt.figure()
plt.plot(all_losses)<jupyter_output><empty_output><jupyter_text>Результаты тренировки<jupyter_code>generated_text = generate(net, device, arxiv_dataset, 'In this research', 200 )
print(generated_text)<jupyter_output>In this research proposals. We present a consumer-quality RGB failures in such as the failure of the failure of feature extractors, such codes. These sets are of box-like objects. We consider three main culprits in the speed-accuracy tradeoff curve, ranging from low-level cues like corners, edges, and agents is introduced and their important properties are exploited to classical approaches which fit a approaches art on convolutional neural networks. We localize of the error state of the large variety of many different points on convolutional neural networks. We develop a leader-follower multi-agent system, and agents is suitable for remaining controllable following the communication links and auditory joint t-controllability are introduced and their important properties are associated with some useful indices and sufficient condition for irregular low-density parity-check (LDPC) codes. participate in the trade-off between accuracy and working memory, respectively. Precentral structures[...]<jupyter_text>Здесь должны быть эксперименты с архитектурой и параметрами сети<jupyter_code>!pip install alive_progress
import alive_progress

def get_losses(num_layers=1, lr=1e-4, temperature=0.8, epochs=3000):
  net = RNN(unique_symbols, hidden_size, unique_symbols, num_layers)
  device = torch.device("cuda")
  net = net.to(device)
  optimizer = torch.optim.Adam(net.parameters(), lr=lr)
  criterion = nn.CrossEntropyLoss()

  # 
  all_losses = []
  with alive_progress.alive_bar(epochs) as bar:
    for epoch in range(1, epochs+1):
      # #Your code for training here
      for local_batch, local_labels in arxiv_generator:
        for input, label in zip(local_batch, local_labels):
          hidden = net.init_hidden().cuda()
          net.zero_grad()
          loss = 0

          # get output
          output, hidden = net(input[-1:].cuda(), hidden)
          output_dist = output.data.view(-1).div(temperature).exp()
          word_index = torch.multinomial(output_dist, 1)
          predicted_word = arxiv_dataset.all_symbols[word_index.item()]
      
          # compute loss
          loss = criterion(output, label[-1:].view(-1).long())

          loss.backward()
          optimizer.step()
          all_losses.append(loss.data.item())
      bar()
  return all_losses
n_layers = [1, 2, 3]
total_losses = []

for n in n_layers:
  total_losses.append(get_losses(num_layers=n))
fig, axs = plt.subplots(1, len(n_layers), figsize=(25,5))

for ax, n, al in zip(axs.flat, n_layers, total_losses):
    ax.set_title(f'n_layers:{n}')
    ax.plot(al)<jupyter_output><empty_output><jupyter_text>из-за большого разброса при подсчете loss трудно выявить наличие или отсутствие какого-то влияния от количества слоев(  Здесь будет работа с вашими датасетами (которые вы захотите использовать)<jupyter_code>from os import listdir
from os.path import isfile, join
import json
import pathlib

BASE_PATH = 'clinguistics_labs'

if not os.path.exists(BASE_PATH):
  ! git clone https://github.com/aailyinh/clinguistics_labs.git

DATASET_PATH = f'{BASE_PATH}/Arzamas/texts'

texts = []

files = [f for f in listdir(DATASET_PATH) if isfile(join(DATASET_PATH, f))]
files = sorted(files)
for fn in files:
		full_fn = f'{DATASET_PATH}/{fn}'
		#print(full_fn)
		with open(full_fn) as txt_file:
			data = txt_file.read()
			texts.append(data)
summaries_text = ' '.join(texts) 
summaries_text = summaries_text[:5000]
summaries_text = summaries_text.split()
print(summaries_text)

# Parameters
params = {
    # 'batch_size': 0,
    # 'shuffle': True,
    # 'num_workers': 1
}

torch.multiprocessing.set_start_method("spawn", force=True)

# Generators
russian_dataset = TextDataset(text=summaries_text, chunk_len=100)
russian_generator = torch.utils.data.DataLoader(russian_dataset, **params)
unique_symbols = len(russian_dataset.all_symbols)
n_layers = 2
hidden_size = 100

net_2 = RNN(unique_symbols, hidden_size, unique_symbols, n_layers)
device = torch.device("cuda")
net_2 = net_2.to(device)
optimizer = torch.optim.Adam(net_2.parameters(), lr=1e-4)
criterion = nn.CrossEntropyLoss()
epochs = 10000

start = time.time()
all_losses_2 = []
temperature = 0.8

for epoch in range(1, epochs+1):
  # #Your code for training here
  for local_batch, local_labels in russian_generator:
    for input, label in zip(local_batch, local_labels):
      hidden = net_2.init_hidden().cuda()
      net_2.zero_grad()
      loss = 0

      # get output
      output, hidden = net_2(input[-1:].cuda(), hidden)
      output_dist = output.data.view(-1).div(temperature).exp()
      word_index = torch.multinomial(output_dist, 1)
      predicted_word = russian_dataset.all_symbols[word_index.item()]
      
      # compute loss
      loss = criterion(output, label[-1:].view(-1).long())

      loss.backward()
      optimizer.step()
      all_losses_2.append(loss.data.item())

print("Saving...")
RNN.save_model(net_2, "TrainedModel.ckpt")
plt.figure()
plt.plot(all_losses_2)
!pip install alive_progress
import alive_progress


def get_losses(num_layers=1, lr=1e-4, temperature=0.8, epochs=3000):
  net = RNN(unique_symbols, hidden_size, unique_symbols, num_layers)
  device = torch.device("cuda")
  net = net.to(device)
  optimizer = torch.optim.Adam(net.parameters(), lr=lr)
  criterion = nn.CrossEntropyLoss()

  # 
  all_losses = []
  with alive_progress.alive_bar(epochs) as bar:
    for epoch in range(1, epochs+1):
      # #Your code for training here
      for local_batch, local_labels in russian_generator:
        for input, label in zip(local_batch, local_labels):
          hidden = net.init_hidden().cuda()
          net.zero_grad()
          loss = 0

          # get output
          output, hidden = net(input[-1:].cuda(), hidden)
          output_dist = output.data.view(-1).div(temperature).exp()
          word_index = torch.multinomial(output_dist, 1)
          predicted_word = russian_dataset.all_symbols[word_index.item()]
      
          # compute loss
          loss = criterion(output, label[-1:].view(-1).long())

          loss.backward()
          optimizer.step()
          all_losses.append(loss.data.item())
      bar()
  return all_losses
n_layers = [1, 2, 3]
total_losses = []

for n in n_layers:
  total_losses.append(get_losses(num_layers=n))
fig, axs = plt.subplots(1, len(n_layers), figsize=(25,5))

for ax, n, al in zip(axs.flat, n_layers, total_losses):
    ax.set_title(f'n_layers:{n}')
    ax.plot(al)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12523629489603025, 'fraction_numerical': 0.010802052389954091, 'mean_word_length': 4.041865214431586, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 39, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '255890', 'license': 'MIT', 'n_tokens_mistral': 5154, 'n_tokens_neox': 5053, 'n_words': 1174, 'path': 'Lab4.ipynb', 'repo_name': 'pazamelin/clinguistics_labs', 'size': '14812'}