<jupyter_start><jupyter_text>Generate plots of features and feature weights<jupyter_code>import matplotlib.pyplot as plt
import numpy as np

dataset_submit = pd.read_pickle('/Users/clarencecheng/Dropbox/~Insight/skimr/datasets/dataset_submit_len_lda_wpos_wpos_readmets')
print(dataset_submit)
hllens     = dataset_submit.loc[dataset_submit['highlightornot'] == 1]['length']
ftlens     = dataset_submit.loc[dataset_submit['highlightornot'] == 0]['length']
h_ldadists = dataset_submit.loc[dataset_submit['highlightornot'] == 1]['LDAdist']
f_ldadists = dataset_submit.loc[dataset_submit['highlightornot'] == 0]['LDAdist']
h_sposes   = dataset_submit.loc[dataset_submit['highlightornot'] == 1]['sentPos']
f_sposes   = dataset_submit.loc[dataset_submit['highlightornot'] == 0]['sentPos']
h_wposes   = dataset_submit.loc[dataset_submit['highlightornot'] == 1]['wordPos']
f_wposes   = dataset_submit.loc[dataset_submit['highlightornot'] == 0]['wordPos']
h_ARI      = dataset_submit.loc[dataset_submit['highlightornot'] == 1]['ARI']
f_ARI      = dataset_submit.loc[dataset_submit['highlightornot'] == 0]['ARI']
h_FRE      = dataset_submit.loc[dataset_submit['highlightornot'] == 1]['FRE']
f_FRE      = dataset_submit.loc[dataset_submit['highlightornot'] == 0]['FRE']
h_FKG      = dataset_submit.loc[dataset_submit['highlightornot'] == 1]['FKG']
f_FKG      = dataset_submit.loc[dataset_submit['highlightornot'] == 0]['FKG']
h_SMG      = dataset_submit.loc[dataset_submit['highlightornot'] == 1]['SMG']
f_SMG      = dataset_submit.loc[dataset_submit['highlightornot'] == 0]['SMG']
h_CLI      = dataset_submit.loc[dataset_submit['highlightornot'] == 1]['CLI']
f_CLI      = dataset_submit.loc[dataset_submit['highlightornot'] == 0]['CLI']
h_GFI      = dataset_submit.loc[dataset_submit['highlightornot'] == 1]['GFI']
f_GFI      = dataset_submit.loc[dataset_submit['highlightornot'] == 0]['GFI']
h_LIX      = dataset_submit.loc[dataset_submit['highlightornot'] == 1]['LIX']
f_LIX      = dataset_submit.loc[dataset_submit['highlightornot'] == 0]['LIX']
h_RIX      = dataset_submit.loc[dataset_submit['highlightornot'] == 1]['RIX']
f_RIX      = dataset_submit.loc[dataset_submit['highlightornot'] == 0]['RIX']
print(hllens)
plt.hist(hllens, bins=50, normed=1, alpha=0.5, range=(-10,140), label='highlighted')
plt.hist(ftlens, bins=50, normed=1, alpha=0.5, range=(-10,140), label='non-highlighted')
plt.title("Sentence length")
plt.legend(loc="upper right")
plt.xlabel("Number of words")
plt.ylabel("Frequency")
plt.show()

plt.hist(h_ldadists, bins=50, normed=1, range=(0,1), alpha=0.5, label='highlighted')
plt.hist(f_ldadists, bins=50, normed=1, range=(0,1), alpha=0.5, label='non-highlighted')
plt.title("Topic similarity")
plt.legend(loc="upper right")
plt.xlabel("Topic similarity score")
plt.ylabel("Frequency")
plt.show()

plt.hist(h_sposes, bins=25, normed=1, range=(0,1), alpha=0.5, label='highlighted')
plt.hist(f_sposes, bins=25, normed=1, range=(0,1), alpha=0.5, label='non-highlighted')
plt.title("Sentence position")
plt.legend(loc="upper right")
plt.xlabel("Fraction sentences into text")
plt.ylabel("Frequency")
plt.show()

plt.hist(h_wposes, bins=25, normed=1, range=(0,1), alpha=0.5, label='highlighted')
plt.hist(f_wposes, bins=25, normed=1, range=(0,1), alpha=0.5, label='non-highlighted')
plt.title("Sentence position")
plt.legend(loc="upper right")
plt.xlabel("Fraction words into text")
plt.ylabel("Frequency")
plt.show()

plt.hist(h_ARI, bins=25, normed=1, range=(0,50), alpha=0.5, label='highlighted')
plt.hist(f_ARI, bins=25, normed=1, range=(0,50), alpha=0.5, label='non-highlighted')
plt.title("Readability (ARI)")
plt.legend(loc="upper right")
plt.xlabel("ARI score")
plt.ylabel("Frequency")
plt.show()

plt.hist(h_FRE, bins=25, normed=1, range=(0,50), alpha=0.5, label='highlighted')
plt.hist(f_FRE, bins=25, normed=1, range=(0,50), alpha=0.5, label='non-highlighted')
plt.title("Readability (FRE)")
plt.legend(loc="upper right")
plt.xlabel("FRE score")
plt.ylabel("Frequency")
plt.show()

plt.hist(h_FKG, bins=25, normed=1, range=(-200,200), alpha=0.5, label='highlighted')
plt.hist(f_FKG, bins=25, normed=1, range=(-200,200), alpha=0.5, label='non-highlighted')
plt.title("Readability (FKG)")
plt.legend(loc="upper right")
plt.xlabel("FKG score")
plt.ylabel("Frequency")
plt.show()

plt.hist(h_GFI, bins=25, normed=1, range=(0,50), alpha=0.5, label='highlighted')
plt.hist(f_GFI, bins=25, normed=1, range=(0,50), alpha=0.5, label='non-highlighted')
plt.title("Readability (GFI)")
plt.legend(loc="upper right")
plt.xlabel("GFI score")
plt.ylabel("Frequency")
plt.show()

plt.hist(h_SMG, bins=25, normed=1, range=(0,30), alpha=0.5, label='highlighted')
plt.hist(f_SMG, bins=25, normed=1, range=(0,30), alpha=0.5, label='non-highlighted')
plt.title("Readability (SMG)")
plt.legend(loc="upper right")
plt.xlabel("SMG score")
plt.ylabel("Frequency")
plt.show()

plt.hist(h_CLI, bins=25, normed=1, range=(-50,50), alpha=0.5, label='highlighted')
plt.hist(f_CLI, bins=25, normed=1, range=(-50,50), alpha=0.5, label='non-highlighted')
plt.title("Readability (CLI)")
plt.legend(loc="upper right")
plt.xlabel("CLI score")
plt.ylabel("Frequency")
plt.show()

plt.hist(h_LIX, bins=25, normed=1, range=(0,100), alpha=0.5, label='highlighted')
plt.hist(f_LIX, bins=25, normed=1, range=(0,100), alpha=0.5, label='non-highlighted')
plt.title("Readability (LIX)")
plt.legend(loc="upper right")
plt.xlabel("LIX score")
plt.ylabel("Frequency")
plt.show()

plt.hist(h_RIX, bins=25, normed=1, range=(0,50), alpha=0.5, label='highlighted')
plt.hist(f_RIX, bins=25, normed=1, range=(0,50), alpha=0.5, label='non-highlighted')
plt.title("Readability (RIX)")
plt.legend(loc="upper right")
plt.xlabel("RIX score")
plt.ylabel("Frequency")
plt.show()



feature_weights = [-0.75450588,  0.00378554 , -0.14955534 ,  0.17832368 , -0.06008622 ,  0.69844673 ,  1.1656712  ,  0.01445301 ,  0.10138341 ,  0.12576411 ,  0.16656827 , -0.75734814 ]
print(sorted(feature_weights))
feature_names = ['length', 'LDAdist', 'wordPos', 'sentPos', 'ARI', 'FRE', 'FKG', 'GFI', 'SMG', 'CLI', 'LIX', 'RIX']
ind = np.arange(12)
fig, ax = plt.subplots()
rects1 = plt.bar(ind, sorted(feature_weights, reverse=True))

# add some text for labels, title and axes ticks
plt.ylabel('Weight')
plt.title('Feature weights')
# plt.xlabel('Feature')
# ax.set_xticks(ind + width / 2)
# plt.xticklabels(('FKG', 'FRE', 'sentPos', 'LIX', 'CLI', 'SMOG', 'GFI', 'LDAdist', 'ARI', 'wordPos', 'length', 'RIX'))
labels = [item.get_text() for item in ax.get_xticklabels()]
labels = ['FKG', 'FRE', 'sentPos', 'LIX', 'CLI', 'SMOG', 'GFI', 'LDAdist', 'ARI', 'wordPos', 'length', 'RIX']

ax.set_xticklabels(labels)

plt.show()

plt.hist(hllens, bins=50, normed=1, alpha=0.5, range=(-10,140), label='highlighted')
# plt.hist(ftlens, bins=50, normed=1, alpha=0.5, range=(-10,140), label='non-highlighted')
plt.title("Sentence length")
# plt.legend(loc="upper right")
plt.xlabel("Number of words")
plt.ylabel("Frequency")
plt.show()

plt.hist(h_ldadists, bins=50, normed=1, range=(0,1), alpha=0.5, label='highlighted')
# plt.hist(f_ldadists, bins=50, normed=1, range=(0,1), alpha=0.5, label='non-highlighted')
plt.title("Topic similarity")
# plt.legend(loc="upper right")
plt.xlabel("Topic similarity score")
plt.ylabel("Frequency")
plt.show()

plt.hist(h_sposes, bins=25, normed=1, range=(0,1), alpha=0.5, label='highlighted')
# plt.hist(f_sposes, bins=25, normed=1, range=(0,1), alpha=0.5, label='non-highlighted')
plt.title("Sentence position")
# plt.legend(loc="upper right")
plt.xlabel("Fraction sentences into text")
plt.ylabel("Frequency")
plt.show()


# for item in p:
#     item.set_height(item.get_height()/sum(x))

# plot feature weights for logistic regression + SMOTE
feature_weights_smote = [-0.53389911, 0.0017571 ,-0.06211337, 0.0919522 ,-0.07619093, 0.59428718, 0.9357417 ,-0.0548295 , 0.1296457 , 0.0961243 , 0.17552932,-0.59777636]
print(sorted(feature_weights_smote))
feature_names_smote = ['length', 'LDAdist', 'wordPos', 'sentPos', 'ARI', 'FRE', 'FKG', 'GFI', 'SMG', 'CLI', 'LIX', 'RIX']
ind = np.arange(12)
fig, ax = plt.subplots()
rects1 = plt.bar(ind, sorted(feature_weights_smote, reverse=True))

# add some text for labels, title and axes ticks
plt.ylabel('Weight')
plt.title('Feature weights, SMOTE')
# plt.xlabel('Feature')
# ax.set_xticks(ind + width / 2)
# plt.xticklabels(('FKG', 'FRE', 'sentPos', 'LIX', 'CLI', 'SMOG', 'GFI', 'LDAdist', 'ARI', 'wordPos', 'length', 'RIX'))
labels = [item.get_text() for item in ax.get_xticklabels()]
labels = ['FKG', 'FRE', 'LIX', 'SMG', 'CLI', 'sentPos', 'LDAdist', 'GFI', 'wordPos', 'ARI', 'length', 'RIX']

ax.set_xticklabels(labels)

plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.20407693193596682, 'fraction_numerical': 0.056316941149372336, 'mean_word_length': 7.115887850467289, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '616781', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 3869, 'n_tokens_neox': 3422, 'n_words': 582, 'path': 'jupyter/6_More_Plotting.ipynb', 'repo_name': 'cyucheng/skimmr', 'size': '8683'}