<jupyter_start><jupyter_text>词频初步统计<jupyter_code>from collections import Counter
def viewFrequency(df, column, cond):
    words = Counter()
    _words = df[df.loc[:, column]==cond].message.apply(lambda message: [word for word in message.split()])
    for word in _words:
        words.update(word)
    print(f'{column:s}-{cond}: ', words.most_common(50))


viewFrequency(data_df, 'target', 0)
viewFrequency(data_df, 'target', 1)<jupyter_output>target-0:  [('i', 2940), ('you', 1943), ('to', 1554), ('the', 1122), ('a', 1056), ('u', 1018), ('$number', 922), ('and', 857), ('in', 818), ('me', 772), ('my', 750), ('is', 732), ('it', 711), ('that', 551), ('of', 525), ('for', 501), ('s', 478), ('have', 440), ('can', 439), ('so', 435), ('but', 434), ('your', 417), ('not', 415), ('are', 414), ('m', 405), ('on', 393), ('do', 384), ('at', 378), ('t', 375), ('if', 354), ('we', 354), ('will', 341), ('be', 335), ('gt', 318), ('lt', 316), ('get', 305), ('how', 304), ('now', 300), ('no', 298), ('just', 293), ('ok', 287), ('when', 287), ('up', 285), ('what', 273), ('with', 272), ('ll', 265), ('this', 252), ('go', 249), ('all', 245), ('ur', 241)]
target-1:  [('$number', 1661), ('to', 688), ('$phonenumber', 431), ('a', 377), ('call', 355), ('å', 299), ('you', 297), ('your', 264), ('free', 224), ('the', 206), ('for', 203), ('now', 202), ('or', 188), ('u', 174), ('txt', 163), ('is', 158), ('on', 144), ('ur', 144), ('have', 135), ('from', 131), ('m[...]<jupyter_text>如上所呈现的，Ham同Spam的top50词频存在蛮大的重叠，这些重叠很大程度上是由这些'stopword'引起的。TODO: 同图展现词语出现率比对因此选择去掉stopword仍然是有必要的此处选择nltk.corp.stopwords词库来协助，并得到结果如下<jupyter_code>data_df = get_sms_dataset(noStopwords=True, overwrite=True)
viewFrequency(data_df, 'target', 0)
viewFrequency(data_df, 'target', 1)
X, y = data_df['message'], data_df['target']
from sklearn.model_selection import train_test_split
X_train,X_test,y_train,y_test=train_test_split(X,y,random_state=0,test_size=0.2)
from sklearn.linear_model import LogisticRegression
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.metrics import accuracy_score
cv=CountVectorizer()
lr=LogisticRegression(max_iter=10000)
X_train=cv.fit_transform(X_train)
from sklearn.naive_bayes import MultinomialNB
nb=MultinomialNB()
nb.fit(X_train,y_train)
pred_2=nb.predict(cv.transform(X_test))
score_2=accuracy_score(y_test,pred_2)
score_2
from sklearn import metrics
metrics.confusion_matrix(y_test, pred_2)
cv.transform(X_test)
y_pred_prob = nb.predict_proba(cv.transform(X_test))[:, 1]
metrics.roc_auc_score(y_test, y_pred_prob)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.24102964118564743, 'fraction_numerical': 0.09399375975039001, 'mean_word_length': 7.784246575342466, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '594292', 'license': 'MIT', 'n_tokens_mistral': 1254, 'n_tokens_neox': 1038, 'n_words': 228, 'path': 'display/feature_engineer.ipynb', 'repo_name': 'Peviroy/MailChecker', 'size': '2564'}