<jupyter_start><jupyter_text>Settings<jupyter_code>need_to_encode = False # if you need to clean and encode texts
need_sample = False # if you need to sample some items of datasets<jupyter_output><empty_output><jupyter_text>Read model<jupyter_code>import encoder

if (need_to_encode):
    model = encoder.parse_model('models/ruwikiruscorpora_upos_skipgram_300_2_2018.vec')
    
    print(len(model))<jupyter_output><empty_output><jupyter_text>Load and clean texts<jupyter_code>import pickle

if (need_to_encode):
    train_texts = encoder.load_from_pickle('data/train_texts_cleaned_short.pickle')
    test_texts = encoder.load_from_pickle('data/test_texts_cleaned_short.pickle')
    
    print(f'{len(train_texts)} train texsts are loaded')
    print(f'{len(test_texts)} test texsts are loaded')

    cleaned_train_texts = list(map(lambda t: encoder.clean_text(t, model), train_texts))
    cleaned_test_texts = list(map(lambda t: encoder.clean_text(t, model), test_texts))
    
    print(f'{len(cleaned_train_texts)} train texts are cleaned')
    print(f'{len(cleaned_test_texts)} test texts are cleaned')<jupyter_output><empty_output><jupyter_text>Encode texts to vectors<jupyter_code>if (need_to_encode):
    train_encoded = list(map(lambda t: encoder.encode_text(t, model, (256, 300)), cleaned_train_texts))
    test_encoded = list(map(lambda t: encoder.encode_text(t, model, (256, 300)), cleaned_test_texts))

    print(f'{len(train_encoded)} train texts are encoded')
    print(f'{len(test_encoded)} test texts are encoded')
    
    print(train_encoded[1])

    encoder.save_to_pickle(train_encoded, 'data/train_texts_cleaned_short_encoded.pickle')
    encoder.save_to_pickle(test_encoded, 'data/test_texts_cleaned_short_encoded.pickle')<jupyter_output><empty_output><jupyter_text>Load encoded data<jupyter_code>train_labels = encoder.load_from_pickle('data/train_labels2.pickle')
train_encoded = encoder.load_from_pickle('data/train_texts_cleaned_short_encoded.pickle')
print(f'{len(train_labels)} train_labels are loaded')
print(f'{len(train_encoded)} train_encoded texts are loaded')

test_labels = encoder.load_from_pickle('data/val_labels2.pickle')
test_encoded = encoder.load_from_pickle('data/test_texts_cleaned_short_encoded.pickle')
print(f'{len(test_labels)} test labels are loaded')
print(f'{len(test_encoded)} test encoded texts are loaded')<jupyter_output>4492 train_labels are loaded
4492 train_encoded texts are loaded
1000 test labels are loaded
1000 test encoded texts are loaded<jupyter_text>Sample items<jupyter_code>import random
if (need_sample):
    train_number = 260
    test_number = 30

    train_sample = []
    train_labels_sample = []
    test_sample = []
    test_labels_sample = []

    for i in range(0, train_number):
        j = random.randint(0, 4491)
        train_sample.append(train_encoded[j])
        train_labels_sample.append(train_labels[j])

    for i in range(0, test_number):
        j = random.randint(0, 999)
        test_sample.append(test_encoded[j])
        test_labels_sample.append(test_labels[j])

    train_encoded = train_sample
    train_labels = train_labels_sample
    test_encoded = test_sample
    test_labels = test_labels_sample<jupyter_output><empty_output><jupyter_text>Convert data to PyTorch<jupyter_code>import torch
from torch.utils.data import TensorDataset, DataLoader

train_tensors_x = torch.FloatTensor(train_encoded).view(4492, 1, 256, 300)
train_tensors_y = torch.FloatTensor(train_labels)
test_tensors_x = torch.FloatTensor(test_encoded).view(1000, 1, 256, 300)
test_tensors_y = torch.FloatTensor(test_labels)

train_dataset = TensorDataset(train_tensors_x, train_tensors_y)
test_dataset = TensorDataset(test_tensors_x, test_tensors_y)

print(f'{len(train_dataset)} train encoded texts are converted to tensors')
print(f'{len(test_dataset)} test encoded texts are converted to tensors')<jupyter_output>4492 train encoded texts are converted to tensors
1000 test encoded texts are converted to tensors<jupyter_text>Split data to train and val<jupyter_code>from torch.utils.data import random_split

#I took train and val sets in a ratio of 7 to 3
train, val = random_split(train_dataset, [3992, 500])

train_loader = DataLoader(train, batch_size=4)
val_loader = DataLoader(val, batch_size=4)
test_loader = DataLoader(test_dataset, batch_size=4)<jupyter_output><empty_output><jupyter_text>Setup and train model<jupyter_code>from models.convnet import ConvNet
from model_utils import train
import torch.optim

#[conv-relu-conv-relu-pool]xN -> [affine]xM -> [softmax or SVM]
im_size = (256, 300, 1)
conv_params = [(256, 7, 2), (512, 5, 2)]
linear_params = [32, 2]
learning_rate = 5e-2

if torch.cuda.is_available():
    device = torch.device('cuda')
else:
    device = torch.device('cpu')
print('using device:', device)
    
model = ConvNet(im_size, conv_params, linear_params)
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9, nesterov=True)

train(model, train_loader, val_loader, optimizer, device, epochs=3)<jupyter_output>using device: cuda
Iteration 0, loss = 1.0493
Iteration 100, loss = 0.9622
Iteration 200, loss = 0.9518
Iteration 300, loss = 0.5827
Iteration 400, loss = 0.6969
Iteration 500, loss = 0.4569
Iteration 600, loss = 0.6760
Iteration 700, loss = 0.5846
Iteration 800, loss = 0.6443
Iteration 900, loss = 0.4479
Iteration 0, loss = 0.5385
Iteration 100, loss = 0.9487
Iteration 200, loss = 0.9006
Iteration 300, loss = 0.5692
Iteration 400, loss = 0.6944
Iteration 500, loss = 0.3447
Iteration 600, loss = 0.4529
Iteration 700, loss = 0.3216
Iteration 800, loss = 0.5805
Iteration 900, loss = 0.6234
Iteration 0, loss = 0.5460
Iteration 100, loss = 1.0900
Iteration 200, loss = 0.7038
Iteration 300, loss = 0.2966
Iteration 400, loss = 0.6052
Iteration 500, loss = 0.3135
Iteration 600, loss = 0.2557
Iteration 700, loss = 0.1816
Iteration 800, loss = 0.4332
Iteration 900, loss = 0.6031<jupyter_text>Test model<jupyter_code>from model_utils import eval_model

pred, groundtruth = eval_model(test_loader, model, device)<jupyter_output><empty_output><jupyter_text>Evaluation<jupyter_code>from sklearn.metrics import accuracy_score, f1_score, recall_score, precision_score

print(f'accuracy: {accuracy_score(pred, groundtruth):.3f}')
print(f'f1 score: {f1_score(pred, groundtruth):.3f}')
print(f'recall: {recall_score(pred, groundtruth):.3f}')
print(f'precision: {precision_score(pred, groundtruth):.3f}')<jupyter_output>accuracy: 0.821
f1 score: 0.802
recall: 0.896
precision: 0.726
{'fraction_non_alphanumeric': 0.10687960687960688, 'fraction_numerical': 0.05835380835380835, 'mean_word_length': 6.188741721854305, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '514878', 'license': 'MIT', 'n_tokens_mistral': 2595, 'n_tokens_neox': 2287, 'n_words': 504, 'path': 'Experiment.ipynb', 'repo_name': 'yordwynn/urban-waffle', 'size': '6512'}