<jupyter_start><jupyter_text>Spam<jupyter_code>word_count_vector=cv.fit_transform(dataset_spam)
tfidf_transformer=TfidfTransformer(smooth_idf=True,use_idf=True) 
tfidf_transformer.fit(word_count_vector)
tfidf_spam = vectorizer.fit_transform(dataset_spam)
# tf-idf scores 
count_vector=cv.transform(dataset_spam)
tf_idf_vector=tfidf_transformer.transform(count_vector)
tfidf_spam
tf_idf_vector
word_count_vector
Number_of_spam_doc = 13719
tf_idf_cf_spam = np.empty(368099)
feature_names = cv.get_feature_names_out()
tf_idf_T = tf_idf_vector.T
word_count_T = word_count_vector.T
tf_idf_cf_spam = np.empty(shape=(368099,2), dtype=object)

for i in range(0,368099):
    if(i % 1000 == 0):
        print(i) 
    score_sum = tf_idf_T[i].sum()
    cf = word_count_T[i].sum() / Number_of_spam_doc
    tf_idf_cf_spam[i] = [feature_names[i],score_sum * cf]
tf_idf_cf_spam
for i in range(0,1):
    score_sum = tf_idf_T[i].sum()
    print(score_sum)
    cf = (word_count_T[i].count_nonzero()) / Number_of_spam_doc
    print(cf)
    print(score_sum*cf)
tf_idf_cf_spam

tf_idf_cf_spam[tf_idf_cf_spam[:,1].argsort()]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11405835543766578, 'fraction_numerical': 0.029177718832891247, 'mean_word_length': 7.028368794326241, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '424971', 'license': 'MIT', 'n_tokens_mistral': 529, 'n_tokens_neox': 509, 'n_words': 56, 'path': '489ProjectCheckpoint.ipynb', 'repo_name': 'sprisha/Data-Analytics-Final-Project', 'size': '1131'}