<jupyter_start><jupyter_text>end<jupyter_code>#factor analysis
ccols =['np_first_occ','root_first_occ','np_count.table', 'np_count.and_ig_ig', 'np_count.andnot_and_ig']
# ccols =['np_count.table','np_count.and_ig_ig', 'np_count.andnot_and_ig']

df3 = df2[ccols].sample(50000)
df3 = df3.dropna()
fact_2c = FactorAnalysis(n_components=2)
df3_factor = fact_2c.fit_transform(df3)
#Plot NPs in the factor space
plt.scatter(df3_factor[:,0], df3_factor[:,1], alpha=.8, edgecolors='none')
print(fact_2c.components_)
plt.show()
dfc = pd.DataFrame(fact_2c.components_.T, index=ccols, columns=['v1', 'v2']).unstack().reset_index()
dfc = dfc.rename(columns={'level_0': 'components', 'level_1': 'feature', 0: 'projection'})
sns.barplot(data=dfc, x='feature', y='projection', hue='components')
#factor analysis
ccols = ['np_count.table','np_count.and_ig_ig', 'np_count.andnot_and_ig']
df3 = df2[ccols].sample(50000)
df3 = df3.dropna()
fact_2c = FactorAnalysis(n_components=2)
df3_factor = fact_2c.fit_transform(df3)
#Plot NPs in the factor space
plt.scatter(df3_factor[:,0], df3_factor[:,1], alpha=.8, edgecolors='none')
print(fact_2c.components_)
plt.show()
dfc = pd.DataFrame(fact_2c.components_.T, index=ccols, columns=['v1', 'v2']).unstack().reset_index()
dfc = dfc.rename(columns={'level_0': 'components', 'level_1': 'feature', 0: 'projection'})
sns.barplot(data=dfc, x='feature', y='projection', hue='components')
roots = df.root.unique()
roots.shape
# number of papers without candidates; number of papers
sum(df['np'].isnull()), len(df['doi'].unique())
# df.loc[df['np'].isnull(), 'doi'].to_csv(expanduser('~/RE_Project/amj_ngrams/latest_listagg/empty_papers.csv'))
x = df['np_count.table']
y = df['np_count.and_ig_ig']
plt.scatter(x, y, alpha=0.1)
plt.show()
df2['np'] = df2['np'].apply(lambda x: literal_eval(x))
mask_long = df2['np'].apply(lambda x: len(list(x)) > 1)
sum(mask_long)
df3 = df2.loc[mask_long]
x = df3['np_count.table']
y = df3['np_count.and_ig_ig']
plt.scatter(x, y, alpha=0.1, s=50)
x = df3['np_first_occ']
y = df3['np_count.and_ig_ig']
plt.scatter(x, y, alpha=0.1, s=10)
plt.show()
lens = df2['np'].apply(lambda x: len(x))
lens.value_counts()
df2[['np_count.table', 'np_count.and_ig_ig', 'np_count.andnot_and_ig', 'np_count.andnot_andnot_and']].corr()
df2['np'].value_counts().head(50)
df2['root'].value_counts().head(50)
df3['np'].value_counts().head(50)
df3['np'].value_counts().head(150).tail(50)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.20671305771592305, 'fraction_numerical': 0.03643061809250921, 'mean_word_length': 8.32824427480916, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '451725', 'license': 'MIT', 'n_tokens_mistral': 1063, 'n_tokens_neox': 1023, 'n_words': 151, 'path': 'notebooks/np_desc.ipynb', 'repo_name': 'alexander-belikov/article_analysis', 'size': '2443'}