<jupyter_start><jupyter_text>Train DISTILBERT MODELSEvaluate them on the hand-tagged sample.<jupyter_code>import torch
import transformers
from datasets import load_dataset, Dataset, DatasetDict,  concatenate_datasets
from transformers import Trainer, TrainingArguments, pipeline, EarlyStoppingCallback
from sklearn.metrics import classification_report, accuracy_score, precision_recall_fscore_support, log_loss
import pandas as pd
import numpy as np
from scipy.special import softmax
from sklearn.model_selection import train_test_split

MODEL_NAME = 'distilbert-base-uncased'
BATCH_SIZE = 128
SENTIMENT_NUM_LABELS = 2
EMOTION_NUM_LABELS = 7
EMOTIONS = ['neutral','happy','sad','anger','disgust','surprise','fear']
SENTIMENTS = ['bearish','bullish']
SEQ_LENGTH = 30
CALLBACK =  [EarlyStoppingCallback(early_stopping_patience = 2)]

def tokenize(batch):
    return tokenizer(batch["text"], padding=True, truncation=True)

if torch.cuda.is_available():       
    device = torch.device("cuda")
    print(f'There are {torch.cuda.device_count()} GPU(s) available.')
    print('Device name:', torch.cuda.get_device_name(0))

else:
    print('No GPU available, using the CPU instead.')
    device = torch.device("cpu")

def compute_metrics(pred):
    labels = pred.label_ids
    preds = pred.predictions.argmax(-1)
    precision, recall, f1, _ = precision_recall_fscore_support(labels, preds, average='macro')
    acc = accuracy_score(labels, preds)
    return {
        'accuracy': acc,
        'f1': f1,
        'precision': precision,
        'recall': recall
    }
#Emotion Test sets
test_df = pd.read_parquet('../emtract/data/hand_tagged_sample.parquet.snappy', columns=['text','label'])
test_set = test_df.copy() # for the 5 class classification
test_set = test_set[(test_set.label !=0) & (test_set.label != 4)].reset_index(drop=True)
test_set['label'] = np.where(test_set.label ==2, 0, test_set.label)
test_set['label'] = np.where(test_set.label==5, 7, test_set.label)
test_set['label'] = np.where(test_set.label>=6, test_set.label-2, test_set.label)

test_df = Dataset.from_pandas(test_df)
test_set = Dataset.from_pandas(test_set)<jupyter_output><empty_output><jupyter_text>Huggingface Twitter Emotion ModelDoes not have disgust and neutral. Group love and joy together.<jupyter_code># Prepare data
def change_labels(examples):
    label = examples['label']
    if label == 0:
        return {'label': 1}
    if label == 1 or label == 2:
        return {'label': 0}
    if label >= 3:
        return {'label': label-1}
    return {'label': output}


emotions = load_dataset("emotion")

dataset = DatasetDict({
    'train': concatenate_datasets([emotions['train'], emotions['test']]),
    'test': test_set,
    'valid': emotions['test']})

tokenizer = transformers.DistilBertTokenizerFast.from_pretrained(MODEL_NAME, model_max_length=SEQ_LENGTH)

emotions_encoded = dataset.map(tokenize, batched=True, batch_size=None)
emotions_encoded = emotions_encoded.map(change_labels)
emotions_encoded.set_format("torch", columns=["input_ids", "attention_mask", "label"])
num_labels = 5
logging_steps = len(emotions_encoded["train"]) // BATCH_SIZE
training_args = TrainingArguments(output_dir="results",
                                  num_train_epochs=8,
                                  learning_rate=2e-5,
                                  per_device_train_batch_size=BATCH_SIZE,
                                  per_device_eval_batch_size=BATCH_SIZE*4,
                                  load_best_model_at_end=True,
                                  metric_for_best_model="loss",
                                  weight_decay=0.01,
                                  evaluation_strategy="epoch",
                                  save_strategy="epoch",
                                  disable_tqdm=False)
model = transformers.DistilBertForSequenceClassification.from_pretrained(MODEL_NAME, num_labels=num_labels).to(device)
trainer = Trainer(model=model, args=training_args,
                  compute_metrics=compute_metrics,
                  train_dataset=emotions_encoded["train"],
                  eval_dataset=emotions_encoded["valid"],
                 callbacks= CALLBACK)
trainer.train();
#model.save_pretrained('./models/emotion-hf')
#tokenizer.save_pretrained('./models/emotion-hf')

preds_output = trainer.predict(emotions_encoded["test"])
print(preds_output.metrics)

m = softmax(preds_output[0], axis=1)
target_names = ['happy','sad','anger','fear','surprise']
print(classification_report(preds_output[1], np.argmax(m, axis=1), target_names=target_names))<jupyter_output><empty_output><jupyter_text>StockTwits Hand-TaggedUse 8k observations for training/validation and hold out 2k for testing.<jupyter_code>tokenizer = transformers.DistilBertTokenizerFast.from_pretrained(MODEL_NAME, model_max_length=SEQ_LENGTH)
model = transformers.DistilBertForSequenceClassification.from_pretrained(MODEL_NAME, num_labels=EMOTION_NUM_LABELS).to(device)
train_testvalid = test_df.train_test_split(test_size=0.4)
test_valid = train_testvalid['test'].train_test_split(test_size=0.5)
dataset = DatasetDict({
    'train': train_testvalid['train'],
    'test': test_valid['test'],
    'valid': test_valid['train']})
emotions_encoded = dataset.map(tokenize, batched=True, batch_size=None)
emotions_encoded.set_format("torch", columns=["input_ids", "attention_mask", "label"])
logging_steps = len(emotions_encoded["train"]) // BATCH_SIZE
training_args = TrainingArguments(output_dir="results",
                                  num_train_epochs=8,
                                  learning_rate=2e-5,
                                  per_device_train_batch_size=BATCH_SIZE,
                                  per_device_eval_batch_size=BATCH_SIZE*4,
                                  load_best_model_at_end=True,
                                  metric_for_best_model="loss",
                                  weight_decay=0.01,
                                  evaluation_strategy="epoch",
                                  save_strategy="epoch",
                                  disable_tqdm=False)

trainer = Trainer(model=model, args=training_args,
                  compute_metrics=compute_metrics,
                  train_dataset=emotions_encoded["train"],
                  eval_dataset=emotions_encoded["valid"],
                   callbacks= CALLBACK)
trainer.train();
preds_output = trainer.predict(emotions_encoded["test"])
print(preds_output.metrics)

m = softmax(preds_output[0], axis=1)

print(classification_report(preds_output[1], np.argmax(m, axis=1), target_names=EMOTIONS))<jupyter_output><empty_output><jupyter_text>Twitter Emotion<jupyter_code>dataset = Dataset.from_pandas(pd.read_parquet('../emtract/data/twitter_emotion_cleaned.parquet.snappy', columns=['text','label']))

# 90% train, 10% test + validation
train_testvalid = dataset.train_test_split(test_size=0.2)

# gather everyone if you want to have a single DatasetDict
dataset = DatasetDict({
    'train': train_testvalid['train'],
    'test': test_df,
    'valid': train_testvalid['test']})
tokenizer = transformers.DistilBertTokenizerFast.from_pretrained(MODEL_NAME, model_max_length=SEQ_LENGTH)
model = transformers.DistilBertForSequenceClassification.from_pretrained(MODEL_NAME, num_labels=EMOTION_NUM_LABELS).to(device)
emotions_encoded = dataset.map(tokenize, batched=True, batch_size=None)
emotions_encoded.set_format("torch", columns=["input_ids", "attention_mask", "label"])
logging_steps = len(emotions_encoded["train"]) // BATCH_SIZE
training_args = TrainingArguments(output_dir="results",
                                  num_train_epochs=8,
                                  learning_rate=2e-5,
                                  per_device_train_batch_size=BATCH_SIZE,
                                  per_device_eval_batch_size=BATCH_SIZE*4,
                                  load_best_model_at_end=True,
                                  metric_for_best_model="loss",
                                  weight_decay=0.01,
                                  evaluation_strategy="epoch",
                                  save_strategy="epoch",
                                  disable_tqdm=False)

trainer = Trainer(model=model, args=training_args,
                  compute_metrics=compute_metrics,
                  train_dataset=emotions_encoded["train"],
                  eval_dataset=emotions_encoded["valid"],
                  callbacks = CALLBACK)
trainer.train();
preds_output = trainer.predict(emotions_encoded["test"])
print(preds_output.metrics)

m = softmax(preds_output[0], axis=1)

print(classification_report(preds_output[1], np.argmax(m, axis=1), target_names=EMOTIONS))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10996326905417815, 'fraction_numerical': 0.008608815426997245, 'mean_word_length': 3.2857845548450566, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '264618', 'license': 'MIT', 'n_tokens_mistral': 2744, 'n_tokens_neox': 2627, 'n_words': 408, 'path': 'notebooks/Alternative Models.ipynb', 'repo_name': 'dvamossy/EmTract', 'size': '8712'}