<jupyter_start><jupyter_text>Simpletransformers<jupyter_code>import pandas as pd
import numpy as np
import torch

train_fname = "../data/lgbt-en.train.tsv"
test_fname = "../data/lgbt-en.test.tsv"

def read_file(fname: str) -> pd.DataFrame:
    """Reads a filename and formats it properly for simpletransformers"""
    df = pd.read_table(fname, sep="\t", header=None, names="text,labels,role".split(","))
    offensive_ids = df.labels != "Acceptable speech"

    df.labels[offensive_ids] = 1
    df.labels[~offensive_ids] = 0
    
    df["labels"] = df.labels.astype(np.int8)
    df = df.drop(columns=["role"])
    return df


train = read_file(train_fname)
test = read_file(test_fname)
%%time
from simpletransformers.classification import ClassificationModel

model_args = {
    "num_train_epochs": 5,
    "learning_rate": 1e-5,
    "overwrite_output_dir": True,
    "train_batch_size": 40
}

model = ClassificationModel(
    "roberta", "roberta-base", use_cuda=True,
    args=model_args
    
)

model.overwrite_output_dir = True
model.train_model(train, )

print(model.eval_model(test))
from sklearn.metrics import accuracy_score, f1_score
y_true = test["labels"]
y_pred = model.predict(list(test["text"].values))[0]

accuracy = accuracy_score(y_true, y_pred)
print("Accuracy: ", accuracy)
f1 = f1_score(y_true, y_pred, average="macro")
print("F1 score: ", f1)<jupyter_output>Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.decoder.weight', 'lm_head.dense.weight', 'lm_head.bias', 'lm_head.layer_norm.bias', 'lm_head.layer_norm.weight', 'lm_head.dense.bias']
- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.dense.weight', 'classifier.out_proj.bias', 'classifier.out_proj.weight', 'classifier.[...]<jupyter_text>Let's save the model<jupyter_code>model.<jupyter_output><empty_output><jupyter_text>Slovenian<jupyter_code>%%time
train_fname = "../data/lgbt-sl.train.tsv"
test_fname = "../data/lgbt-sl.test.tsv"

train = read_file(train_fname)
test = read_file(test_fname)


model_args = {
    "num_train_epochs": 5,
    "learning_rate": 1e-5,
    "overwrite_output_dir": True,
    "train_batch_size": 40
}

model = ClassificationModel(
    "roberta", "roberta-base", use_cuda=True,
    args=model_args
    
)

model.overwrite_output_dir = True
model.train_model(train, )

print(model.eval_model(test))
from sklearn.metrics import accuracy_score, f1_score
y_true = test["labels"]
y_pred = model.predict(list(test["text"].values))[0]

accuracy = accuracy_score(y_true, y_pred)
print("Accuracy: ", accuracy)
f1 = f1_score(y_true, y_pred, average="macro")
print("F1 score: ", f1)<jupyter_output>Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.layer_norm.bias', 'lm_head.layer_norm.weight', 'lm_head.decoder.weight', 'lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.bias']
- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.weight', 'classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.[...]<jupyter_text>Croatian<jupyter_code>%%time
train_fname = "../data/lgbt-hr.train.tsv"
test_fname = "../data/lgbt-hr.test.tsv"

train = read_file(train_fname)
test = read_file(test_fname)


model_args = {
    "num_train_epochs": 5,
    "learning_rate": 1e-5,
    "overwrite_output_dir": True,
    "train_batch_size": 40
}

model = ClassificationModel(
    "roberta", "roberta-base", use_cuda=True,
    args=model_args
    
)

model.overwrite_output_dir = True
model.train_model(train, )

print(model.eval_model(test))
from sklearn.metrics import accuracy_score, f1_score
y_true = test["labels"]
y_pred = model.predict(list(test["text"].values))[0]

accuracy = accuracy_score(y_true, y_pred)
print("Accuracy: ", accuracy)
f1 = f1_score(y_true, y_pred, average="macro")
print("F1 score: ", f1)<jupyter_output>Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['lm_head.layer_norm.bias', 'lm_head.layer_norm.weight', 'lm_head.decoder.weight', 'lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.bias']
- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.out_proj.weight', 'classifier.out_proj.bias', 'classifier.dense.weight', 'classifier.[...]<jupyter_text>"sangrimlee/bert-base-multilingual-cased-nsmc"<jupyter_code>import pandas as pd
import numpy as np
import torch

train_fname = "../data/lgbt-en.train.tsv"
test_fname = "../data/lgbt-en.test.tsv"

def read_file(fname: str) -> pd.DataFrame:
    """Reads a filename and formats it properly for simpletransformers"""
    df = pd.read_table(fname, sep="\t", header=None, names="text,labels,role".split(","))
    offensive_ids = df.labels != "Acceptable speech"

    df.labels[offensive_ids] = 1
    df.labels[~offensive_ids] = 0
    
    df["labels"] = df.labels.astype(np.int8)
    df = df.drop(columns=["role"])
    return df


train = read_file(train_fname)
test = read_file(test_fname)
%%time
from simpletransformers.classification import ClassificationModel

model_args = {
    "num_train_epochs": 5,
    "learning_rate": 1e-5,
    "overwrite_output_dir": True,
    "train_batch_size": 40
}

model = ClassificationModel(
    "bert", "sangrimlee/bert-base-multilingual-cased-nsmc", use_cuda=True,
    args=model_args
    
)

model.overwrite_output_dir = True
model.train_model(train, )

print(model.eval_model(test))
from sklearn.metrics import accuracy_score, f1_score
y_true = test["labels"]
y_pred = model.predict(list(test["text"].values))[0]

accuracy = accuracy_score(y_true, y_pred)
print("Accuracy: ", accuracy)
f1 = f1_score(y_true, y_pred, average="macro")
print("F1 score: ", f1)<jupyter_output><empty_output><jupyter_text>Slovenian<jupyter_code>%%time
train_fname = "../data/lgbt-sl.train.tsv"
test_fname = "../data/lgbt-sl.test.tsv"

train = read_file(train_fname)
test = read_file(test_fname)


model_args = {
    "num_train_epochs": 5,
    "learning_rate": 1e-5,
    "overwrite_output_dir": True,
    "train_batch_size": 40
}

model = ClassificationModel(
    "bert", "sangrimlee/bert-base-multilingual-cased-nsmc", use_cuda=True,
    args=model_args
    
)

model.overwrite_output_dir = True
model.train_model(train, )

print(model.eval_model(test))
from sklearn.metrics import accuracy_score, f1_score
y_true = test["labels"]
y_pred = model.predict(list(test["text"].values))[0]

accuracy = accuracy_score(y_true, y_pred)
print("Accuracy: ", accuracy)
f1 = f1_score(y_true, y_pred, average="macro")
print("F1 score: ", f1)<jupyter_output><empty_output><jupyter_text>Croatian<jupyter_code>%%time
train_fname = "../data/lgbt-hr.train.tsv"
test_fname = "../data/lgbt-hr.test.tsv"

train = read_file(train_fname)
test = read_file(test_fname)


model_args = {
    "num_train_epochs": 5,
    "learning_rate": 1e-5,
    "overwrite_output_dir": True,
    "train_batch_size": 40
}

model = ClassificationModel(
    "bert", "sangrimlee/bert-base-multilingual-cased-nsmc", use_cuda=True,
    args=model_args
    
)

model.overwrite_output_dir = True
model.train_model(train, )

print(model.eval_model(test))
from sklearn.metrics import accuracy_score, f1_score
y_true = test["labels"]
y_pred = model.predict(list(test["text"].values))[0]

accuracy = accuracy_score(y_true, y_pred)
print("Accuracy: ", accuracy)
f1 = f1_score(y_true, y_pred, average="macro")
print("F1 score: ", f1)<jupyter_output><empty_output><jupyter_text>"roberta", "unitary/multilingual-toxic-xlm-roberta", use_cuda=True,<jupyter_code>import pandas as pd
import numpy as np
import torch

train_fname = "../data/lgbt-en.train.tsv"
test_fname = "../data/lgbt-en.test.tsv"

def read_file(fname: str) -> pd.DataFrame:
    """Reads a filename and formats it properly for simpletransformers"""
    df = pd.read_table(fname, sep="\t", header=None, names="text,labels,role".split(","))
    offensive_ids = df.labels != "Acceptable speech"

    df.labels[offensive_ids] = 1
    df.labels[~offensive_ids] = 0
    
    df["labels"] = df.labels.astype(np.int8)
    df = df.drop(columns=["role"])
    return df


train = read_file(train_fname)
test = read_file(test_fname)
%%time
from simpletransformers.classification import ClassificationModel

model_args = {
    "num_train_epochs": 5,
    "learning_rate": 1e-5,
    "overwrite_output_dir": True,
    "train_batch_size": 40
}

model = ClassificationModel(
    "roberta", "unitary/multilingual-toxic-xlm-roberta", use_cuda=True,
    args=model_args
    
)

model.overwrite_output_dir = True
model.train_model(train, )

print(model.eval_model(test))
from sklearn.metrics import accuracy_score, f1_score
y_true = test["labels"]
y_pred = model.predict(list(test["text"].values))[0]

accuracy = accuracy_score(y_true, y_pred)
print("Accuracy: ", accuracy)
f1 = f1_score(y_true, y_pred, average="macro")
print("F1 score: ", f1)<jupyter_output>You are using a model of type xlm-roberta to instantiate a model of type roberta. This is not supported for all configurations of models and can yield errors.
Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at unitary/multilingual-toxic-xlm-roberta and are newly initialized: ['roberta.pooler.dense.bias', 'roberta.pooler.dense.weight']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
The tokenizer class you load from this checkpoint is not the same type as the class this function is called from. It may result in unexpected tokenization. 
The tokenizer class you load from this checkpoint is 'XLMRobertaTokenizer'. 
The class this function is called from is 'RobertaTokenizer'.<jupyter_text>Slovenian<jupyter_code>%%time
train_fname = "../data/lgbt-sl.train.tsv"
test_fname = "../data/lgbt-sl.test.tsv"

train = read_file(train_fname)
test = read_file(test_fname)


model_args = {
    "num_train_epochs": 5,
    "learning_rate": 1e-5,
    "overwrite_output_dir": True,
    "train_batch_size": 40
}

model = ClassificationModel(
    "roberta", "unitary/multilingual-toxic-xlm-roberta", use_cuda=True,
    args=model_args
    
)

model.overwrite_output_dir = True
model.train_model(train, )

print(model.eval_model(test))
from sklearn.metrics import accuracy_score, f1_score
y_true = test["labels"]
y_pred = model.predict(list(test["text"].values))[0]

accuracy = accuracy_score(y_true, y_pred)
print("Accuracy: ", accuracy)
f1 = f1_score(y_true, y_pred, average="macro")
print("F1 score: ", f1)<jupyter_output>You are using a model of type xlm-roberta to instantiate a model of type roberta. This is not supported for all configurations of models and can yield errors.
Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at unitary/multilingual-toxic-xlm-roberta and are newly initialized: ['roberta.pooler.dense.bias', 'roberta.pooler.dense.weight']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
The tokenizer class you load from this checkpoint is not the same type as the class this function is called from. It may result in unexpected tokenization. 
The tokenizer class you load from this checkpoint is 'XLMRobertaTokenizer'. 
The class this function is called from is 'RobertaTokenizer'.<jupyter_text>Croatian<jupyter_code>%%time
train_fname = "../data/lgbt-hr.train.tsv"
test_fname = "../data/lgbt-hr.test.tsv"

train = read_file(train_fname)
test = read_file(test_fname)


model_args = {
    "num_train_epochs": 5,
    "learning_rate": 1e-5,
    "overwrite_output_dir": True,
    "train_batch_size": 40
}

model = ClassificationModel(
    "roberta", "unitary/multilingual-toxic-xlm-roberta", use_cuda=True,
    args=model_args
    
)

model.overwrite_output_dir = True
model.train_model(train, )

print(model.eval_model(test))
from sklearn.metrics import accuracy_score, f1_score
y_true = test["labels"]
y_pred = model.predict(list(test["text"].values))[0]

accuracy = accuracy_score(y_true, y_pred)
print("Accuracy: ", accuracy)
f1 = f1_score(y_true, y_pred, average="macro")
print("F1 score: ", f1)<jupyter_output><empty_output><jupyter_text>Distilbert  Slovenian<jupyter_code>%%time
train_fname = "../data/lgbt-sl.train.tsv"
test_fname = "../data/lgbt-sl.test.tsv"

train = read_file(train_fname)
test = read_file(test_fname)


model_args = {
    "num_train_epochs": 5,
    "learning_rate": 1e-5,
    "overwrite_output_dir": True,
    "train_batch_size": 40
}

model = ClassificationModel(
     "distilbert", "distilbert-base-uncased-finetuned-sst-2-english", use_cuda=True,
    args=model_args
    
)

model.overwrite_output_dir = True
model.train_model(train, )

print(model.eval_model(test))
from sklearn.metrics import accuracy_score, f1_score
y_true = test["labels"]
y_pred = model.predict(list(test["text"].values))[0]

accuracy = accuracy_score(y_true, y_pred)
print("Accuracy: ", accuracy)
f1 = f1_score(y_true, y_pred, average="macro")
print("F1 score: ", f1)<jupyter_output><empty_output><jupyter_text>English<jupyter_code>%%time
train_fname = "../data/lgbt-en.train.tsv"
test_fname = "../data/lgbt-en.test.tsv"

train = read_file(train_fname)
test = read_file(test_fname)


model_args = {
    "num_train_epochs": 5,
    "learning_rate": 1e-5,
    "overwrite_output_dir": True,
    "train_batch_size": 40
}

model = ClassificationModel(
     "distilbert", "distilbert-base-uncased-finetuned-sst-2-english", use_cuda=True,
    args=model_args
    
)

model.overwrite_output_dir = True
model.train_model(train, )

print(model.eval_model(test))
from sklearn.metrics import accuracy_score, f1_score
y_true = test["labels"]
y_pred = model.predict(list(test["text"].values))[0]

accuracy = accuracy_score(y_true, y_pred)
print("Accuracy: ", accuracy)
f1 = f1_score(y_true, y_pred, average="macro")
print("F1 score: ", f1)<jupyter_output><empty_output><jupyter_text>Croatian<jupyter_code>%%time
train_fname = "../data/lgbt-hr.train.tsv"
test_fname = "../data/lgbt-hr.test.tsv"

train = read_file(train_fname)
test = read_file(test_fname)


model_args = {
    "num_train_epochs": 5,
    "learning_rate": 1e-5,
    "overwrite_output_dir": True,
    "train_batch_size": 40
}

model = ClassificationModel(
     "distilbert", "distilbert-base-uncased-finetuned-sst-2-english", use_cuda=True,
    args=model_args
    
)

model.overwrite_output_dir = True
model.train_model(train, )

print(model.eval_model(test))
from sklearn.metrics import accuracy_score, f1_score
y_true = test["labels"]
y_pred = model.predict(list(test["text"].values))[0]

accuracy = accuracy_score(y_true, y_pred)
print("Accuracy: ", accuracy)
f1 = f1_score(y_true, y_pred, average="macro")
print("F1 score: ", f1)<jupyter_output><empty_output><jupyter_text>Slovenian checkpoint<jupyter_code>%%time
train_fname = "../data/lgbt-sl.train.tsv"
test_fname = "../data/lgbt-sl.test.tsv"

train = read_file(train_fname)
test = read_file(test_fname)


model_args = {
    "num_train_epochs": 5,
    "learning_rate": 1e-5,
    "overwrite_output_dir": True,
    "train_batch_size": 40
}

model = ClassificationModel(
     "bert", "IMSyPP/hate_speech_slo", use_cuda=True,
    args=model_args
    
)

model.overwrite_output_dir = True
model.train_model(train, )

print(model.eval_model(test))
from sklearn.metrics import accuracy_score, f1_score
y_true = test["labels"]
y_pred = model.predict(list(test["text"].values))[0]

accuracy = accuracy_score(y_true, y_pred)
print("Accuracy: ", accuracy)
f1 = f1_score(y_true, y_pred, average="macro")
print("F1 score: ", f1)<jupyter_output><empty_output><jupyter_text>Hugging face<jupyter_code>import torch
from transformers import AdamW, AutoTokenizer, AutoModelForSequenceClassification

# Same as before
checkpoint = "roberta-base"
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
model = AutoModelForSequenceClassification.from_pretrained(checkpoint)
sequences = list(train.text.values)
batch = tokenizer(sequences, padding=True, truncation=True, return_tensors="pt")

# This is new
batch["labels"] = torch.tensor(list(train.labels.values))

optimizer = AdamW(model.parameters())
loss = model(**batch).loss
loss.backward()
optimizer.step()
from transformers import AutoTokenizer, AutoModelForSequenceClassification, AdamW
  
train_fname = "../data/lgbt-sl.train.tsv"
test_fname = "../data/lgbt-sl.test.tsv"

train = read_file(train_fname)
test = read_file(test_fname)


tokenizer = AutoTokenizer.from_pretrained("IMSyPP/hate_speech_slo")

model = AutoModelForSequenceClassification.from_pretrained("IMSyPP/hate_speech_slo")

sequences = list(train.text.values)
batch = tokenizer(sequences, padding=True, truncation=True, return_tensors="pt")

# This is new
batch["labels"] = torch.tensor(list(train.labels.values))

optimizer = AdamW(model.parameters())
loss = model(**batch).loss
loss.backward()
optimizer.step()
train.shape
len(sequences)<jupyter_output><empty_output><jupyter_text>Part 2: attempting to use correct models for the input language  Slovenian:<jupyter_code>%%time

from simpletransformers.classification import ClassificationModel


train_fname = "../data/lgbt-sl.train.tsv"
test_fname = "../data/lgbt-sl.test.tsv"

train = read_file(train_fname)
test = read_file(test_fname)


model_args = {
    "num_train_epochs": 5,
    "learning_rate": 1e-5,
    "overwrite_output_dir": True,
    "train_batch_size": 40
}

model = ClassificationModel(
    "camembert", "EMBEDDIA/sloberta", use_cuda=True,
    args=model_args
    
)

model.overwrite_output_dir = True
model.train_model(train, )

print(model.eval_model(test))
from sklearn.metrics import accuracy_score, f1_score
y_true = test["labels"]
y_pred = model.predict(list(test["text"].values))[0]

accuracy = accuracy_score(y_true, y_pred)
print(f"Accuracy: {accuracy:0.3}")
f1 = f1_score(y_true, y_pred, average="macro")
print(f"F1: {f1:0.3}")
%%time

from simpletransformers.classification import ClassificationModel


train_fname = "../data/lgbt-sl.train.tsv"
test_fname = "../data/lgbt-sl.test.tsv"

train = read_file(train_fname)
test = read_file(test_fname)


model_args = {
    "num_train_epochs": 5,
    "learning_rate": 1e-5,
    "overwrite_output_dir": True,
    "train_batch_size": 40
}

model = ClassificationModel(
    "bert", "EMBEDDIA/crosloengual-bert", use_cuda=True,
    args=model_args
    
)

model.overwrite_output_dir = True
model.train_model(train, )

print(model.eval_model(test))
from sklearn.metrics import accuracy_score, f1_score
y_true = test["labels"]
y_pred = model.predict(list(test["text"].values))[0]

accuracy = accuracy_score(y_true, y_pred)
print(f"Accuracy: {accuracy:0.3}")
f1 = f1_score(y_true, y_pred, average="macro")
print(f"F1: {f1:0.3}")<jupyter_output>Some weights of the model checkpoint at EMBEDDIA/crosloengual-bert were not used when initializing BertForSequenceClassification: ['cls.predictions.transform.dense.bias', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.bias', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.weight']
- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
Some weights of BertForSequenceClassification were not initialized from the model chec[...]<jupyter_text>Croatian<jupyter_code>%%time

from simpletransformers.classification import ClassificationModel


train_fname = "../data/lgbt-hr.train.tsv"
test_fname = "../data/lgbt-hr.test.tsv"

train = read_file(train_fname)
test = read_file(test_fname)


model_args = {
    "num_train_epochs": 5,
    "learning_rate": 1e-5,
    "overwrite_output_dir": True,
    "train_batch_size": 40
}

model = ClassificationModel(
    "bert", "EMBEDDIA/crosloengual-bert", use_cuda=True,
    args=model_args
    
)

model.overwrite_output_dir = True
model.train_model(train, )

#print(model.eval_model(test))
from sklearn.metrics import accuracy_score, f1_score
y_true = test["labels"]
y_pred = model.predict(list(test["text"].values))[0]

accuracy = accuracy_score(y_true, y_pred)
print(f"Accuracy: {accuracy:0.3}")
f1 = f1_score(y_true, y_pred, average="macro")
print(f"F1: {f1:0.3}")
%%time

from simpletransformers.classification import ClassificationModel
lang = "hr"
model_type = "electra"
model_name = "classla/bcms-bertic"

train_fname = f"../data/lgbt-{lang}.train.tsv"
test_fname = f"../data/lgbt-{lang}.test.tsv"

train = read_file(train_fname)
test = read_file(test_fname)


model_args = {
    "num_train_epochs": 5,
    "learning_rate": 1e-5,
    "overwrite_output_dir": True,
    "train_batch_size": 40
}

model = ClassificationModel(
    model_type, model_name, use_cuda=True,
    args=model_args
    
)

model.overwrite_output_dir = True
model.train_model(train, )

#print(model.eval_model(test))
from sklearn.metrics import accuracy_score, f1_score
y_true = test["labels"]
y_pred = model.predict(list(test["text"].values))[0]

accuracy = accuracy_score(y_true, y_pred)

f1 = f1_score(y_true, y_pred, average="macro")

print(f"""
# Model: {model_name}

|language|accuracy|f1 score|
|---|---|---|
|{lang}|{accuracy:0.3}|{f1:0.3}|

""")<jupyter_output>Some weights of the model checkpoint at classla/bcms-bertic were not used when initializing ElectraForSequenceClassification: ['discriminator_predictions.dense_prediction.weight', 'discriminator_predictions.dense.weight', 'discriminator_predictions.dense.bias', 'discriminator_predictions.dense_prediction.bias']
- This IS expected if you are initializing ElectraForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing ElectraForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
Some weights of ElectraForSequenceClassification were not initialized from the model checkpoint at classla/bcms-bertic and are newly initialized: ['pooler.dense.bias', 'classifier.weight',[...]
{'fraction_non_alphanumeric': 0.12151992433277131, 'fraction_numerical': 0.008882674672040137, 'mean_word_length': 5.743760399334443, 'pattern_counts': {'":': 68, '<': 62, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '322800', 'license': 'MIT', 'n_tokens_mistral': 8627, 'n_tokens_neox': 8345, 'n_words': 1786, 'path': 'simpletransformer.ipynb', 'repo_name': '5roop/task1', 'size': '24317'}