<jupyter_start><jupyter_text>Lab 01 : Cross-entropy loss -- demo<jupyter_code># For Google Colaboratory
import sys, os
if 'google.colab' in sys.modules:
    # mount google drive
    from google.colab import drive
    drive.mount('/content/gdrive')
    # find automatically the path of the folder containing "file_name" :
    file_name = 'cross_entropy_demo.ipynb'
    import subprocess
    path_to_file = subprocess.check_output('find . -type f -name ' + str(file_name), shell=True).decode("utf-8")
    path_to_file = path_to_file.replace(file_name,"").replace('\n',"")
    # if previous search failed or too long, comment the previous line and simply write down manually the path below :
    #path_to_file = '/content/gdrive/My Drive/CS5242_2021_codes/codes/labs_lecture04/lab01_cross_entropy'
    print(path_to_file)
    # change current path to the folder containing "file_name"
    os.chdir(path_to_file)
    !pwd
import torch
import torch.nn as nn
import utils<jupyter_output><empty_output><jupyter_text>Make a Cross Entropy Criterion and call it mycrit<jupyter_code>mycrit=nn.CrossEntropyLoss()
print(mycrit)<jupyter_output>CrossEntropyLoss()<jupyter_text>Make a batch of labels<jupyter_code>labels=torch.LongTensor([2,3])

print(labels)<jupyter_output>tensor([2, 3])<jupyter_text>Make a batch of scores<jupyter_code>scores=torch.Tensor([ [-1.2, 0.5 , 5, -0.5], [1.4, -1.7 , -1.3, 5.0] ])

print(scores)
utils.display_scores(scores)<jupyter_output><empty_output><jupyter_text>compute the average loss on this batch<jupyter_code>average_loss = mycrit(scores,labels)

print('loss = ', average_loss.item() )<jupyter_output>loss =  0.023508310317993164<jupyter_text>Try with a different batch of scores<jupyter_code>scores=torch.Tensor([ [-1.2, 0.5 , 3.1, -0.5], [1.4, -1.7 , -1.3, 2.0] ])

print(scores)

utils.display_scores(scores)

average_loss = mycrit(scores,labels)

print('loss = ', average_loss.item() )<jupyter_output>tensor([[-1.2000,  0.5000,  3.1000, -0.5000],
        [ 1.4000, -1.7000, -1.3000,  2.0000]])<jupyter_text>Try with a different batch of scores<jupyter_code>scores=torch.Tensor([[0.8, 2.3, -1.0, -1.2] , [-1.2, 1.3, 5.0 , -2.0 ] ]  )

print(scores)

utils.display_scores(scores)

average_loss = mycrit(scores,labels)

print('loss = ', average_loss.item() )<jupyter_output>tensor([[ 0.8000,  2.3000, -1.0000, -1.2000],
        [-1.2000,  1.3000,  5.0000, -2.0000]])
{'fraction_non_alphanumeric': 0.1479933110367893, 'fraction_numerical': 0.06897993311036789, 'mean_word_length': 4.8508557457212715, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '100261', 'license': 'MIT', 'n_tokens_mistral': 1021, 'n_tokens_neox': 926, 'n_words': 182, 'path': 'codes/labs_lecture04/lab01_cross_entropy/cross_entropy_demo.ipynb', 'repo_name': 'tanfiona/CS5242_2021', 'size': '2392'}