<jupyter_start><jupyter_text>Loading packages<jupyter_code>from datasets import load_dataset
from collections import Counter
from sklearn.feature_extraction import DictVectorizer
# Get data
dataset = load_dataset("go_emotions", "raw") # https://huggingface.co/datasets/go_emotions
train = dataset["train"]
# Inspect dataset
train
# Calc term frequency for each text in texts

# Define function for Term Frequency (TF)
def term_freq(token_list):
    """
    Takes in a list of tokens (str) and return a dictionary of term frequency of each token
    """
    doc_length = len(token_list)
    term_count_dict = dict(Counter(token_list))
    tf_scores = [items[1]/doc_length for items in term_count_dict.items()]
    term_freq = dict(zip(term_count_dict.keys(), tf_scores))
    return(term_freq)

# For i in texts, split the text, and take the new split text and calculate tf for each term
texts_tf = [term_freq(i.split()) for i in train["text"]]
print(texts_tf[0:5])
# For texts_tf, use dictvectorizer from sklearn to also get all the 0's
v = DictVectorizer(sparse=False)
X = v.fit_transform(texts_tf)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09304347826086956, 'fraction_numerical': 0.0034782608695652175, 'mean_word_length': 4.813131313131313, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '413558', 'license': 'MIT', 'n_tokens_mistral': 385, 'n_tokens_neox': 372, 'n_words': 113, 'path': 'syllabus/classes/class5/exercise.ipynb', 'repo_name': 'emiltj/NLP-E21', 'size': '1150'}