<jupyter_start><jupyter_text>This approach checks if some data preprocessing helps on the results.<jupyter_code>import pandas as pd
import re
from sklearn.naive_bayes import MultinomialNB
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics import classification_report, f1_score
import preprocessing as pp
from sklearn.pipeline import Pipeline
from sklearn.model_selection import train_test_split
new_data = pd.read_csv('data/new_dataset.csv')
msr_data = pd.read_csv('data/msr_dataset.csv', encoding='ANSI')
new_data.head()
msr_data.head()
def preprocess(token):
    token = str(token) 
    token_lowered = token.lower()
    
    token_lowered = re.sub(r'(\brow\b)|(\btable\b)|(\binsert\b)|(\bid\b)', 'dbms', token_lowered)
    
    url_pattern = '(https?:\/\/(?:www\.|(?!www))[a-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]\.[^\s]{2,}|www\.[a-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]\.[^\s]{2,}|https?:\/\/(?:www\.|(?!www))[a-zA-Z0-9]+\.[^\s]{2,}|www\.[a-zA-Z0-9]+\.[^\s]{2,})'
    token_url = re.sub(url_pattern, 'urll<PASSWORD>', token_lowered)
    
    date_pattern = '([12]\d{3}/(0[1-9]|1[0-2])/(0[1-9]|[12]\d|3[01]))|([12]\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01]))|([12]\d{3}/(0[1-9]|1[0-2])/(0[1-9]|[12]\d|3[01]))'
    token_dates = re.sub(date_pattern, 'datetime', token_url)
    
    ip_pattern = '(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])'
    token_ip = re.sub(ip_pattern, ' ipaddrezz ', token_dates)
    
    token_num = re.sub(r'\b[0-9]+\b', 'numeric', token_ip)
    
    special_character_pattern = '[^A-Za-z0-9]+'
    token_char = re.sub(special_character_pattern, ' ', token_num)
    
    return token_char
msr_data['clean_token'] = msr_data['token'].map(lambda s:preprocess(s)) 
new_data['clean_token'] = new_data['token'].map(lambda t:preprocess(t)) 
msr_data[['token', 'clean_token']][1500:1700]
y_msr = msr_data['class']
y_new = new_data['class']

X_train, X_test, y_train, y_test = train_test_split(msr_data['clean_token'], 
                                                    y_msr, train_size=0.8, 
                                                    random_state=33, shuffle=True)
text_clf2 = Pipeline([
    ('vectorizer', CountVectorizer(ngram_range=(1,10))),
    ('model', MultinomialNB())])

text_clf2.fit(X_train, y_train)
preds = text_clf2.predict(X_test)
print(classification_report(y_test, preds))
new_preds = text_clf2.predict(new_data['clean_token'])
print(classification_report(y_new, new_preds))
print('f1', f1_score(y_new, new_preds))<jupyter_output>precision    recall  f1-score   support

           0       0.65      0.44      0.53       724
           1       0.58      0.76      0.66       737

    accuracy                           0.60      1461
   macro avg       0.61      0.60      0.59      1461
weighted avg       0.61      0.60      0.59      1461

f1 0.6600117439812095
{'fraction_non_alphanumeric': 0.1950558753809685, 'fraction_numerical': 0.07246867592279038, 'mean_word_length': 3.232091690544413, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '29360', 'license': 'MIT', 'n_tokens_mistral': 1294, 'n_tokens_neox': 1217, 'n_words': 161, 'path': 'cleaning_data.ipynb', 'repo_name': 'aianshay/flaky-tests', 'size': '2946'}