<jupyter_start><jupyter_text>Data Import<jupyter_code>df = pd.read_csv(import_path)
df.head(5)<jupyter_output><empty_output><jupyter_text>Dependent variable frequencies<jupyter_code>plt.rcdefaults()
fig, ax = plt.subplots()
fig.set_size_inches(4, 2)

# Example data
example = ["relevant","not relevant"]
y_pos = np.arange(2)
performance = [df.loc[df['target'] == 1].count()[0],df.loc[df['target'] == 0].count()[0]]            
error = np.random.rand(2)
colors = ['lightgreen', 'royalblue'] # set the colors

ax.barh(y_pos, performance,  align='center', color = colors) # xerr=error
ax.set_yticks(y_pos)
ax.set_yticklabels(example)
ax.invert_yaxis()  # labels read top-to-bottom
ax.set_xlabel('Count')
ax.set_title('Number of records given by relevant and not relevant')

plt.show()
fig.savefig("./Data/counts.jpg",bbox_inches='tight')

# Print number of funny and number of serious texts
print("Number of relevant texts is " + str(df.loc[df['target'] == 1].count()[0]))
print("Number of not not relevant texts is " + str(df.loc[df['target'] == 0].count()[0]))<jupyter_output><empty_output><jupyter_text>Preprocessing<jupyter_code>if IsEmbedding == False:
    
    sentences_lst = df["maintext"].tolist()
    
    # Create Embeddings
    embeddings = embeddings_sentence_bert(sentences_lst, True, base_model_name)
    embeddings_tunned = embeddings_sentence_bert(sentences_lst, False, fine_tunned_model_path)
    
    # Convert to dataframe 
    data = pd.DataFrame([[i] for i in tqdm(embeddings)]).rename(columns={0:'maintext_embed'})
    data_tunned = pd.DataFrame([[i] for i in tqdm(embeddings_tunned)]).rename(columns={0:'maintext_embed'})
    
    # Add embeddings to initial data
    data_concat = pd.concat([df.reset_index(), data.reset_index()], axis=1)
    data_concat_tunned = pd.concat([df.reset_index(), data_tunned.reset_index()], axis=1)
    
    # save data with embeddings 
    data_concat.to_pickle(data_embeddings_dir + "embeddings.pickle")
    data_concat_tunned.to_pickle(data_embeddings_dir + "embeddings_tunned.pickle")<jupyter_output><empty_output><jupyter_text>Baseline BERT contextual representation<jupyter_code>result, matrix, report, precision, recall, auc_score = train_evaluate_model(data_embeddings_dir + "embeddings.pickle")
result
plot_confusion_matrix(matrix , title = 'Confusion Matrix Baseline model')
print(report)
plot_P_R_curve(precision, recall, auc_score)<jupyter_output><empty_output><jupyter_text>Fine-tunned BERT contextual representation (see Bert fine-tunning.ipynb)<jupyter_code>result, matrix, report, precision, recall, auc_score = train_evaluate_model(data_embeddings_dir + "embeddings_tunned.pickle")
result
plot_confusion_matrix(matrix, title = 'Confusion Matrix Fine-tunned on triplet Loss')
print(report)
plot_P_R_curve(precision, recall, auc_score)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13206214689265536, 'fraction_numerical': 0.006002824858757062, 'mean_word_length': 6.740437158469946, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '331873', 'license': 'MIT', 'n_tokens_mistral': 1008, 'n_tokens_neox': 975, 'n_words': 176, 'path': 'Main.ipynb', 'repo_name': 'petervajdecka02947/false_positive_reduction', 'size': '2832'}