<jupyter_start><jupyter_text>Prepare data  Load dataset<jupyter_code>import pickle

PICKLE_PATH = "allocine_dataset/data/allocine_dataset.pickle"

with open(PICKLE_PATH, 'rb') as reader:
    data = pickle.load(reader)

# Reviews need to be tokenized
train_reviews = np.array(data["train_set"]['review'])
val_reviews = np.array(data["val_set"]['review'])
test_reviews = np.array(data["test_set"]['review'])

train_labels = data["train_set"]['polarity']
val_labels = data["val_set"]['polarity']
test_labels = data["test_set"]['polarity']
class_names = data['class_names']<jupyter_output><empty_output><jupyter_text>Preprocess dataset  Tokenize<jupyter_code>from transformers import CamembertTokenizer

model_name = "camembert-base"
tokenizer = CamembertTokenizer.from_pretrained(model_name)
some_review = train_reviews[0]
some_review
tokenizer.tokenize(some_review)[:15]
tokenizer.encode(some_review)[:15]
tokenizer.decode(tokenizer.encode(some_review))
reviews_len = [len(tokenizer.encode(review, max_length=512))
                          for review in train_reviews]
print("Average length: {:.1f}".format(np.mean(reviews_len)))
print("Max length: {}".format(max(reviews_len)))

plt.figure(figsize=(10,5))
ax = sns.distplot(reviews_len, bins=150, kde=False, hist_kws=dict(alpha=0.8))
ax.set(xlabel='Number of tokens')

# Finalize the plot
sns.despine(bottom=True)
plt.tight_layout(h_pad=2)

# Saving plot
fig = ax.get_figure()
fig.savefig('img/bert/number_of_tokens.png', dpi=200)
tokenizer.vocab_size
MAX_SEQ_LEN = 400 # in terms of generated tokens (not words)

short_reviews = sum(np.array(reviews_len) <= MAX_SEQ_LEN)
long_reviews = sum(np.array(reviews_len) > MAX_SEQ_LEN)

print("{} reviews with LEN > {} ({:.2f} % of total data)".format(
    long_reviews,
    MAX_SEQ_LEN,
    100 * long_reviews / len(reviews_len)
))
import numpy as np

def encode_reviews(tokenizer, reviews, max_length):
    token_ids = np.zeros(shape=(len(reviews), max_length),
                         dtype=np.int32)
    for i, review in enumerate(reviews):
        encoded = tokenizer.encode(review, max_length=max_length)
        token_ids[i, 0:len(encoded)] = encoded
    attention_mask = (token_ids != 0).astype(np.int32)
    return {"input_ids": token_ids, "attention_mask": attention_mask}
encoded_train = encode_reviews(tokenizer, train_reviews, MAX_SEQ_LEN)
encoded_valid = encode_reviews(tokenizer, val_reviews, MAX_SEQ_LEN)
encoded_test = encode_reviews(tokenizer, test_reviews, MAX_SEQ_LEN)<jupyter_output><empty_output><jupyter_text>Labels<jupyter_code>y_train = np.array(train_labels)
y_val = np.array(val_labels)
y_test = np.array(test_labels)<jupyter_output><empty_output><jupyter_text>Preprocessing pipeline<jupyter_code>from sklearn.base import BaseEstimator, TransformerMixin

class CamembertPreprocessor(BaseEstimator, TransformerMixin):
    def __init__(self, tokenizer, max_seq_length):
        self.tokenizer = tokenizer
        self.max_seq_length = max_seq_length

    def fit(self, X=None):
        pass
    
    def transform(self, X, y):
        # 1. Tokenize
        X_encoded = encode_reviews(self.tokenizer, X, self.max_seq_length)
        # 2. Labels
        y_array = np.array(y)
        return X_encoded, y_array     
    
    def fit_transform(self, X, y):        
        return self.transform(X, y)<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>from transformers import TFCamembertForSequenceClassification

model = TFCamembertForSequenceClassification.from_pretrained("jplu/tf-camembert-base")

opt = tf.keras.optimizers.Adam(learning_rate=5e-6, epsilon=1e-08)
loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)    

model.compile(optimizer=opt, loss=loss_fn, metrics=['accuracy'])
initial_weights = model.get_weights()
model.summary()<jupyter_output>Model: "tf_camembert_for_sequence_classification"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
roberta (TFRobertaMainLayer) multiple                  110621952 
_________________________________________________________________
classifier (TFRobertaClassif multiple                  592130    
=================================================================
Total params: 111,214,082
Trainable params: 111,214,082
Non-trainable params: 0
_________________________________________________________________<jupyter_text>Training<jupyter_code># TODO (in fact, done in the Accuracy vs Training Data part)
history = model.fit(
    encoded_train, y_train, epochs=1, batch_size=4, 
    validation_data=(encoded_valid, y_val), verbose=1
)<jupyter_output><empty_output><jupyter_text>Accuracy vs Training Data<jupyter_code>from sklearn.base import BaseEstimator

class EarlyStoppingModel(BaseEstimator):
    def __init__(self, transformers_model, max_epoches, batch_size, validation_data):
        self.model = transformers_model
        self.max_epoches = max_epoches
        self.batch_size = batch_size
        self.validation_data = validation_data
        
    def fit(self, X, y):
        # Defines early stopper
        early_stopper = tf.keras.callbacks.EarlyStopping(
            monitor='val_loss', mode='auto', patience=2, # only 1 !
            verbose=1, restore_best_weights=True
        )        

        # Train model on data subset
        self.model.fit(
            X, y,
            validation_data=self.validation_data,
            epochs=self.max_epoches, 
            batch_size=self.batch_size,
            callbacks=[early_stopper], 
            verbose=1
        )        
        return self
    
    def predict(self, X):        
        scores = self.model.predict(X)
        y_pred = np.argmax(scores, axis=1)
        return y_pred
from sklearn import metrics 

def accuracy_vs_training_data(camembert_model, initial_weights, 
                              preprocessor, sizes,
                              train_reviews, train_labels,
                              val_reviews, val_labels,
                              test_reviews, test_labels):
    test_accuracies = []
    for size in sizes:        
        # Preprocess data
        X_train, y_train = preprocessor.fit_transform(
            train_reviews[:size], train_labels[:size]
        )
        X_val, y_val = preprocessor.transform(val_reviews, val_labels)
        X_test, y_test = preprocessor.transform(test_reviews, test_labels)
        
        # Reset weights to initial value
        camembert_model.set_weights(initial_weights)
        best_model = EarlyStoppingModel(
            camembert_model, max_epoches=20, batch_size=4,
            validation_data=(X_val, y_val)
        )
        
        # Train model
        best_model.fit(X_train, y_train)
        
        # Evaluate on test set
        y_pred = best_model.predict(X_test)
        test_acc = metrics.accuracy_score(y_test, y_pred)
        test_accuracies.append(test_acc)
        print("Test acc: " + str(test_acc))
        
    return test_accuracies    
sizes = [int(p) for p in np.geomspace(500, 160000, 5)]
preprocessor = CamembertPreprocessor(tokenizer, MAX_SEQ_LEN)

test_accuracies = accuracy_vs_training_data(
    model, initial_weights, 
    preprocessor, sizes,
    train_reviews, train_labels,
    val_reviews, val_labels,
    test_reviews, test_labels
)
# Saving last model (full dataset)
model.save_weights('data/bert/camembert_weights.hdf5')
import pickle

OUTPUT_PATH = 'data/bert/camembert_accuracies.pickle'

output_dict = {
    "sizes": sizes,
    "test_accuracies": test_accuracies
}

with open(OUTPUT_PATH, 'wb') as writer:
    pickle.dump(output_dict, writer)<jupyter_output><empty_output><jupyter_text>Error Analysis<jupyter_code>from sklearn import metrics

model.load_weights('data/camembert_weights.hdf5')
scores = model.predict(encoded_valid)
y_pred = np.argmax(scores, axis=1)
    
print("Val Accuracy: {:.2f}".format(100 * metrics.accuracy_score(y_val, y_pred)))
print("Val F1-Score: {:.2f}".format(100 * metrics.f1_score(y_val, y_pred)))<jupyter_output>Val Accuracy: 97.39
Val F1-Score: 97.36<jupyter_text>Confusion Matrix<jupyter_code>from utils import print_confusion_matrix
from sklearn.metrics import confusion_matrix

conf_mx = confusion_matrix(y_val, y_pred)

fig = print_confusion_matrix(
    conf_mx, 
    class_names.values(), 
    figsize=(7,5)
)

# Finalize the plot
sns.despine(bottom=True)
plt.tight_layout(h_pad=2)

# Saving plot
fig.savefig('img/bert/val_confusion_mx.png', dpi=200)
## False positive / negative
false_pos = val_reviews[(y_val == 0) & (y_pred == 1)]
false_neg = val_reviews[(y_val == 1) & (y_pred == 0)]
pd.DataFrame(false_pos[:5])
pd.DataFrame(false_neg[:5])<jupyter_output><empty_output><jupyter_text>Testing best model<jupyter_code>model.load_weights('data/bert/camembert_weights.hdf5')

scores = model.predict(encoded_test)
y_pred = np.argmax(scores, axis=1)
from sklearn import metrics
    
print("Test Accuracy: {:.2f}".format(100 * metrics.accuracy_score(y_test, y_pred)))
print("Test F1-Score: {:.2f}".format(100 * metrics.f1_score(y_test, y_pred)))
print()

report = metrics.classification_report(
    y_test, y_pred, 
    target_names=class_names.values()
)
print(report)
from sklearn.metrics import confusion_matrix
conf_mx = confusion_matrix(y_test, y_pred)

fig = print_confusion_matrix(
    conf_mx, 
    class_names.values(), 
    figsize=(7,5)
)

# Finalize the plot
sns.despine(bottom=True)
plt.tight_layout(h_pad=2)

# Saving plot
fig.savefig('img/bert/test_confusion_mx.png', dpi=200)<jupyter_output><empty_output><jupyter_text>Inference time<jupyter_code>model.load_weights('data/bert/camembert_weights.hdf5')
import time

inference_times = []

for i in range(1000):
    x = {
    'input_ids': np.array([encoded_test['input_ids'][i], ]),
    'attention_mask':  np.array([encoded_test['attention_mask'][i], ]),
    }
    start_time = time.time()
    y_pred = model.predict(x)
    stop_time = time.time()
    
    inference_times.append(stop_time - start_time)    
OUTPUT_PATH = 'data/bert/camembert_times.pickle'

with open(OUTPUT_PATH, 'wb') as writer:
    pickle.dump(inference_times, writer)<jupyter_output><empty_output><jupyter_text>Generalizability<jupyter_code>import os
from utils_acl import get_data

ACL_FOLDER = 'data/cls-acl10-unprocessed/fr'
BOOKS_FOLDER = os.path.join(ACL_FOLDER, 'books')
DVD_FOLDER = os.path.join(ACL_FOLDER, 'dvd')
MUSIC_FOLDER = os.path.join(ACL_FOLDER, 'music')

_, _, test_reviews_b, test_labels_b = get_data(BOOKS_FOLDER)
_, _, test_reviews_d, test_labels_d  = get_data(DVD_FOLDER)
_, _, test_reviews_m, test_labels_m  = get_data(MUSIC_FOLDER)
from sklearn import metrics 

def evaluate(model, X, y):
    scores = model.predict(X)
    y_pred = np.argmax(scores, axis=1)
    print("Accuracy: {:.2f}".format(100 * metrics.accuracy_score(y, y_pred)))
    print("F1-Score: {:.2f}".format(100 * metrics.f1_score(y, y_pred)))
preprocessor = CamembertPreprocessor(tokenizer, MAX_SEQ_LEN)
model.load_weights('data/bert/camembert_weights.hdf5')<jupyter_output><empty_output><jupyter_text>Books<jupyter_code>X_books, y_books = preprocessor.transform(test_reviews_b, test_labels_b)
evaluate(model, X_books, y_books)<jupyter_output>Accuracy: 94.10
F1-Score: 94.33<jupyter_text>DVD<jupyter_code>X_dvd, y_dvd = preprocessor.transform(test_reviews_d, test_labels_d)
evaluate(model, X_dvd, y_dvd)<jupyter_output>Accuracy: 93.25
F1-Score: 93.48<jupyter_text>Music<jupyter_code>X_music, y_music = preprocessor.transform(test_reviews_m, test_labels_m)
evaluate(model, X_music, y_music)<jupyter_output>Accuracy: 94.55
F1-Score: 94.71
{'fraction_non_alphanumeric': 0.12054606336395639, 'fraction_numerical': 0.0166566497810595, 'mean_word_length': 4.5361216730038025, 'pattern_counts': {'":': 4, '<': 57, '<?xml version=': 0, '>': 58, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '400541', 'license': 'MIT', 'n_tokens_mistral': 4220, 'n_tokens_neox': 4011, 'n_words': 659, 'path': '03_bert.ipynb', 'repo_name': 'Kwalys/french-sentiment-analysis-with-bert', 'size': '11647'}