<jupyter_start><jupyter_text>Tfidf<jupyter_code>from sklearn.feature_extraction.text import TfidfVectorizer

vect1 = TfidfVectorizer(min_df=5).fit(X_train)
len(vect1.get_feature_names())
x_train_vectorized1 = vect1.transform(X_train)

model1 = LogisticRegression()
model1.fit(x_train_vectorized1, y_train)

predictions1 = model1.predict(vect1.transform(X_test))

print('AUC: ', roc_auc_score(y_test, predictions1))
feature_names1 = np.array(vect1.get_feature_names())

sorted_coef_index1 = x_train_vectorized1.max(0).toarray()[0].argsort()

print('Smallest Coefs:\n{}\n'.format(feature_names1[sorted_coef_index1[:10]]))
print('Largest Coefs:\n{}\n'.format(feature_names1[sorted_coef_index1[:-11:-1]]))
print(model1.predict(vect1.transform(['not an issue, phone is working',
                                      'an issue, phone is not working'])))
print(model.predict(vect.transform(['not an issue, phone is working',
                                      'an issue, phone is not working'])))<jupyter_output>[0 0]<jupyter_text>n-grams<jupyter_code>vect2 = CountVectorizer(min_df=5,  ngram_range=(1, 2)).fit(X_train)

x_train_vectorized2 = vect2.transform(X_train)

len(vect2.get_feature_names())
model2 = LogisticRegression()
model2.fit(x_train_vectorized2, y_train)

predictions2 = model2.predict(vect2.transform(X_test))

print('AUC: ', roc_auc_score(y_test, predictions2))
feature_names2 = np.array(vect2.get_feature_names())

sorted_coef_index2 = x_train_vectorized2.max(0).toarray()[0].argsort()

print('Smallest Coefs:\n{}\n'.format(feature_names2[sorted_coef_index2[:10]]))
print('Largest Coefs:\n{}\n'.format(feature_names2[sorted_coef_index2[:-11:-1]]))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14766686355581807, 'fraction_numerical': 0.03544004725339634, 'mean_word_length': 6.63063063063063, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '560574', 'license': 'MIT', 'n_tokens_mistral': 670, 'n_tokens_neox': 649, 'n_words': 74, 'path': 'sentiment_analysis.ipynb', 'repo_name': 'camara94/python-text-mining', 'size': '1693'}