<jupyter_start><jupyter_text>Analyzing Syllabi<jupyter_code>import matplotlib
import pandas as pd
import seaborn
%matplotlib inline

df = pd.read_csv("disciplines-per-course.csv")
df.head()
disciplinary_breakdown_by_course = df.groupby(["school","program","aggregates"]).size()
disciplinary_breakdown_by_course.head()
ax = disciplinary_breakdown_by_course.unstack(level=2).plot(kind="barh", 
                                                            stacked=True, 
                                                            figsize=(11,8), 
                                                            colormap="Set2",
                                                            )
fig = ax.get_figure()
fig.tight_layout()
fig.savefig("raw_discipline_breakdown.pdf")<jupyter_output><empty_output><jupyter_text>proportions per course<jupyter_code>raw_course_discipline_counts = df.groupby(["course","aggregates"]).size()
raw_course_discipline_counts.head()
discipline_proportion_per_course = raw_course_discipline_counts.div(raw_course_discipline_counts.sum(level="course"), level="course")*100
discipline_proportion_per_course.unstack(level=1).fillna(0).to_csv("discipline_proportion_per_course.csv")<jupyter_output><empty_output><jupyter_text>Proportions per program<jupyter_code>raw_programs_discipline_counts = df.groupby(["school","program","aggregates"]).size()
raw_programs_discipline_counts

total_program_counts = raw_programs_discipline_counts.groupby(level=['school','program']).sum()
total_program_counts
proportion_per_program = raw_programs_discipline_counts.unstack().div(total_program_counts, axis=0) * 100
ax = proportion_per_program.plot(kind="barh", 
                                 stacked=True, 
                                 figsize=(11,8), 
                                 colormap="Set3",
                                 )

ax.set_xlim(0,100)
ax.legend(bbox_to_anchor=(1.31, .85))

fig = ax.get_figure()
fig.tight_layout(pad=12)
fig.savefig("proportion_per_program.pdf")
proportion_per_program
proportion_per_program.fillna(0).to_csv("proportion_per_program.csv")<jupyter_output><empty_output><jupyter_text>Entropy as a measure of complexity<jupyter_code>from scipy.stats import entropy
core_complexity = proportion_per_program.fillna(0).apply(entropy, axis=1)
ax = core_complexity.sort(ascending=True, inplace=False).plot(kind="barh",
                                                        figsize=(11,8))
fig = ax.get_figure()
fig.tight_layout()
fig.savefig("complexity_per_program.pdf")
core_complexity
proportion_per_program['diversity'] = core_complexity
proportion_per_program_ordered = proportion_per_program.sort(columns="diversity", inplace=False)
proportion_per_program_ordered.drop("diversity", axis=1, inplace=True)
ax = proportion_per_program_ordered.plot(kind="barh", 
                                 stacked=True, 
                                 figsize=(11,8), 
                                 colormap="Set3",
                                 )

ax.set_xlim(0,100)
ax.legend(bbox_to_anchor=(1.31, .85))

fig = ax.get_figure()
ax.set_title("Disciplinary porportions per Program")
ax.set_xlabel('Percentage')
ax.set_ylabel('School-Program')

fig.tight_layout(pad=12)
#fig.savefig("proportion_per_program.png", dpi=300)
fig.savefig("proportion_per_program.pdf")<jupyter_output><empty_output><jupyter_text>Topic areas across the ischools<jupyter_code>ischool_discipline_proportions = raw_programs_discipline_counts.groupby(level="aggregates").sum().div(raw_programs_discipline_counts.sum(),level='aggregates') * 100
ischool_discipline_proportions.sort(inplace=False)
ax = ischool_discipline_proportions.plot(kind="pie",
                                         stacked=True,
                                         figsize=(20,20),
                                         colormap="Set3",
                                         label='',
                                         fontsize=32,
                                         autopct='%.1f')
fig = ax.get_figure()
fig.tight_layout(pad=12)

ax.set_title("Interdisciplinarity Across Syllabi", fontsize=36)



fig.savefig("proportion_all_ischools.png")<jupyter_output><empty_output><jupyter_text>Graphing overlap<jupyter_code>cleaned_citations = pd.read_csv("cleaned_cites.csv")
journal_articles = cleaned_citations[cleaned_citations['type'] == 'article']
top_journal_articles = journal_articles.groupby('title').size().sort(inplace=False, ascending=False)
ax = top_journal_articles.plot(kind="line",
                               figsize=(11,4),
                               use_index=False,
                               )

ax.set_xlim(0,100)
ax.set_ylim(0,7)
ax.set_title("Distribution of Popular Articles across Syllabi")
ax.set_xlabel('Article Popularity Ranking')
ax.set_ylabel('Number of Syllabi')
fig = ax.get_figure()
fig.tight_layout()
fig.savefig("popular-articles.pdf")
top_journal_articles.head(15)<jupyter_output><empty_output><jupyter_text>top journal titles<jupyter_code>top_journal_titles = cleaned_citations[cleaned_citations['type'] == 'article'].groupby('journal').size().sort(inplace=False, ascending=False)
top_journal_titles.head(30)
top_journal_titles.describe()
ax = top_journal_titles.plot(kind="line",
                               figsize=(11,4),
                               use_index=False,
                               )

#ax.set_xlim(0,100)
#ax.set_ylim(0,7)
ax.set_title("Popularity Distribution of Journal Titles")
ax.set_xlabel('Journal Popularity Ranking')
ax.set_ylabel('Number of Citations')
fig = ax.get_figure()
fig.tight_layout()
fig.savefig("popular-journals.pdf")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1155547690674217, 'fraction_numerical': 0.016103344540789242, 'mean_word_length': 3.1773835920177382, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '524554', 'license': 'CC0-1.0', 'n_tokens_mistral': 1864, 'n_tokens_neox': 1721, 'n_words': 189, 'path': 'notebooks/clustering courses.ipynb', 'repo_name': 'mcburton/ischools-core', 'size': '5651'}