<jupyter_start><jupyter_text>Source:

huggingface: https://huggingface.co/HooshvareLab/bert-fa-base-uncased-clf-persiannews

Tutorial:https://towardsdatascience.com/fine-tuning-hugging-face-model-with-custom-dataset-82b8092f5333<jupyter_code>from google.colab import drive
drive.mount('/content/drive')
import os
os.chdir('/content/drive/MyDrive/sharif/FineTuning/ipython(guide)')
!pip install transformers
!pip3 install sentencepiece
!git clone https://huggingface.co/HooshvareLab/bert-fa-base-uncased-clf-persiannews
GIT_LFS_SKIP_SMUDGE=1
import tensorflow as tf
import numpy
import pandas
import re
from sklearn.preprocessing import MultiLabelBinarizer
from sklearn.model_selection import train_test_split
from transformers import AutoTokenizer, AutoModelForSequenceClassification,AutoConfig,TFAutoModel
# tokenizer = AutoTokenizer.from_pretrained("m3hrdadfi/albert-fa-base-v2")
# model = TFAutoModel.from_pretrained("m3hrdadfi/albert-fa-base-v2")<jupyter_output><empty_output><jupyter_text>Tokenizer example:<jupyter_code>text = "ما در هوشواره معتقدیم با انتقال صحیح دانش و آگاهی، همه افراد می‌توانند از ابزارهای هوشمند استفاده کنند. شعار ما هوش مصنوعی برای همه است."
tokenizer.tokenize(text)
limit_number = 750
data = pandas.read_csv('../Data/limited_to_'+str(limit_number)+'.csv',index_col=0)
data = data.dropna().reset_index(drop=True)
X = data["body"].values.tolist()
y = pandas.read_csv('../Data/limited_to_'+str(limit_number)+'.csv')
labels = []
tag=[]
for item in y['tag']:
  labels += [i for i in re.sub('\"|\[|\]|\'| |=','',item.lower()).split(",") if i!='' and i!=' ']
  tag.append([i for i in re.sub('\"|\[|\]|\'| |=','',item.lower()).split(",") if i!='' and i!=' '])
labels = list(set(labels))
mlb = MultiLabelBinarizer()
Y=mlb.fit_transform(tag)
X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.2)
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.25)
print('train: ', len(X_train) , '\ntest: ', len(X_test) , '\nval: ', len(X_val) ,"\ny_tain:",len(y_train) )
###we would load the tokenizer
tokenizer = AutoTokenizer.from_pretrained("HooshvareLab/bert-fa-base-uncased-clf-persiannews")
##Tokenize training and validation sentences:
train_encodings = tokenizer(X_train,
                            truncation=True,
                            padding=True)
val_encodings = tokenizer(X_val,
                            truncation=True,
                            padding=True)
test_encodings=tokenizer(X_test,
                            truncation=True,
                            padding=True)
##Here we map sentences with labels, there is no need to pass label into fit function separately
train_dataset = tf.data.Dataset.from_tensor_slices((
    dict(train_encodings),
    y_train
))
val_dataset = tf.data.Dataset.from_tensor_slices((
    dict(val_encodings),
    y_val
))
###We need to get a pre-trained Hugging Face model
model = TFAutoModel.from_pretrained("HooshvareLab/bert-fa-base-uncased")
optimizer = tf.keras.optimizers.Adam(learning_rate=5e-5)
model.compile(optimizer=optimizer, loss="categorical_crossentropy", metrics=['accuracy'])

##Fine-tuning
model.fit(train_dataset.shuffle(100).batch(16),
          epochs=3,
          batch_size=16,
          validation_data=val_dataset.shuffle(100).batch(16))
model.save_pretrained("/tmp/sentiment_custom_model")
loaded_model = TFDistilBertForSequenceClassification.from_pretrained("/tmp/sentiment_custom_model")
predict_input = tokenizer.encode(X_test,
                                 truncation=True,
                                 padding=True,
                                 return_tensors="tf")tf_output = loaded_model.predict(predict_input)[0]
tf_prediction = tf.nn.softmax(tf_output, axis=1).numpy()[0]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13235678786293487, 'fraction_numerical': 0.011247711221553753, 'mean_word_length': 3.9921671018276763, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '123510', 'license': 'MIT', 'n_tokens_mistral': 1438, 'n_tokens_neox': 1387, 'n_words': 219, 'path': 'FineTuning/ipython(guide)/ParsBERT.ipynb', 'repo_name': 'ZeinabTaghavi/text_multi_label_classification', 'size': '3823'}