<jupyter_start><jupyter_text>Read the data<jupyter_code>mapping = {"ham": 0,
           "spam": 1}
# SMS Collection
filename = "SMSSpamCollection.txt"
df_coll = pd.read_csv(data_dir / filename, sep="\t", header=None, names=["label", "text"])
df_coll["label"] = df_coll["label"].map(mapping)
df_coll["source"] = "coll"
df_coll["label"].value_counts(normalize=True)
# SMS from DIT - only spam
tree = ET.parse(data_dir / 'spam.xml')
root = tree.getroot()
data = []
for sms in root.iterfind("sms"):
    record = {child.tag:child.text for child in sms.getchildren()}
    data.append(record)
df_dit = pd.DataFrame(data).rename(columns={"class": "label"})
df_dit["label"] = df_dit["label"].map(lambda x: mapping.get(x, x))
df_dit["label"].value_counts()
# only spam
df = pd.concat([df_coll, df_dit], axis=0)
# Clean up duplicates
s = df.text.map(lambda x: x.strip(" \t\n\r").lower())
duplicates = df.loc[s.duplicated(), "text"]
df.loc[df.text.isin(duplicates), ["source", "label", "text"]].sort_values(by="text").head(10)
df_clean = df.loc[~s.duplicated()]
df_clean.label.value_counts(normalize=True)<jupyter_output><empty_output><jupyter_text>Train - test split<jupyter_code>X = df_clean["text"]
y = df_clean["label"]
test_size = 0.2
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size, random_state=42,
                                                    stratify=y)
print(f"Num. of train: {len(X_train)}, Num. of test: {len(X_test)}")<jupyter_output>Num. of train: 4697, Num. of test: 1175<jupyter_text>Let's implement NB<jupyter_code>def calc_likelihood(data, vectorizer, y):
    tfidf =  vectorizer.fit_transform(data).toarray()
    prob_spam = tfidf[y > 0].sum(axis=0)
    prob_ham = tfidf[y < 1].sum(axis=0)
    prob_spam /= np.sum(prob_spam)
    prob_ham /= np.sum(prob_ham)
    return np.vstack((prob_ham, prob_spam)).T
def predict(X_test, vectorizer, priors, likelihood):
    tf = vectorizer.transform(X_test).toarray() 
    prob = tf[:, :, np.newaxis] * likelihood[np.newaxis, :, :]
    prob[prob==0] = 1
    score = np.exp(np.log(prob).sum(axis=1)) * priors
    score[score.max(axis=1)==0] = eps
    return score / score.sum(axis=1)[:, np.newaxis]
tf_params = {"lowercase": True,
             "analyzer": "char_wb",
             "stop_words": stop,
             "ngram_range": (3, 3),
             "min_df": 1,
             "max_df": 1.0,
             "preprocessor": None,#Preprocessor(),
             "max_features": 3500,
             "norm": "l2"*0,
             "use_idf": 1
             }
eps = 1e-15
priors = y_train.value_counts(normalize=True).values
vectorizer = TfidfVectorizer(**tf_params)
train = vectorizer.fit_transform(X_train)
test = vectorizer.transform(X_test)
likelihood = calc_likelihood(X_train, vectorizer, y_train)
likelihood[likelihood==0] = eps
likelihood[likelihood==1] = 1 - eps
#assert np.amax(np.fabs(np.sum(likelihood, axis=1) - train.toarray().sum(axis=0) / np.sum(train))) <= 1e-5
pred_probs = predict(X_test, vectorizer, priors, likelihood)
proba = pred_probs[:, 1]
pred = np.zeros_like(proba)
pred[proba>=0.5] = 1
metrics = calc_metrics(y_test, pred, proba, labels=["ham", "spam"])<jupyter_output>AUC: 0.960
Recall: 0.890
Precision: 0.906
F1: 0.898
accuracy: 0.953
      pred_ham  pred_spam
ham        878         25
spam        30        242<jupyter_text>Fit Scikit learn NB<jupyter_code>clf = MultinomialNB(alpha=2.5, class_prior=[0.5, 0.5])
clf.fit(train.toarray(), y_train)
pred = clf.predict(test.toarray())
proba = clf.predict_proba(test.toarray())[:, 1]
metrics = calc_metrics(y_test, pred, proba, labels=["ham", "spam"], mode="binary")<jupyter_output><empty_output><jupyter_text>Fit XGBoost<jupyter_code>params = {}
params['scale_pos_weight'] = sum(y_train==0) / sum(y_train==1)
params['learning_rate'] = 0.1
params['n_estimators'] = 1000
params['max_depth'] = 5
params['min_child_weight'] = 100
params['gamma'] = 0
params['subsample'] = 0.8
params['colsample_bytree'] = 0.8
params['objective'] = 'binary:logistic'
params['seed'] = 27
params['n_jobs'] = -1
params["eval_metric"] = ["error","auc"]
params["early_stopping_rounds"] = 50
dtrain = xgb.DMatrix(train, y_train)
dtest = xgb.DMatrix(test, y_test)
eval_set = [(dtrain, "train"), (dtest, "eval")]
model = xgb.train(dtrain=dtrain, num_boost_round=params.get("n_estimators"), 
                  early_stopping_rounds=params.get("early_stopping_rounds"), 
                  params=params, evals=eval_set, verbose_eval=50)
pred_proba_xgb = model.predict(dtest)
pred_xgb = np.zeros_like(pred_proba_xgb)
pred_xgb[pred_proba_xgb>=0.5] = 1
xgb_metrics = calc_metrics(y_test, pred_xgb, pred_proba_xgb)<jupyter_output>AUC: 0.974
Recall: 0.926
Precision: 0.741
F1: 0.824
accuracy: 0.908
      pred_ham  pred_spam
ham        815         88
spam        20        252
{'fraction_non_alphanumeric': 0.16684089483587705, 'fraction_numerical': 0.03115199665481915, 'mean_word_length': 4.333333333333333, 'pattern_counts': {'":': 13, '<': 20, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '304800', 'license': 'MIT', 'n_tokens_mistral': 1966, 'n_tokens_neox': 1849, 'n_words': 326, 'path': 'notebooks/0.1-klulu-baseline.ipynb', 'repo_name': 'KarimLulu/sms-spam', 'size': '4783'}