<jupyter_start><jupyter_text>Analyse university contact networks<jupyter_code>import networkx as nx
from os.path import join
import numpy as np
import matplotlib.pyplot as plt
import matplotlib
matplotlib.rc_file_defaults()
import pandas as pd
import seaborn as sns
src = '../data/networks'
G_all = nx.readwrite.gpickle.read_gpickle(\
        join(src, 'university_2019-10-16_to_2019-10-23_fraction-1_all.bz2')) 
G_TU = nx.readwrite.gpickle.read_gpickle(\
        join(src, 'university_2019-10-16_to_2019-10-23_fraction-1_TU.bz2'))
G_NaWi = nx.readwrite.gpickle.read_gpickle(\
        join(src, 'university_2019-10-16_to_2019-10-23_fraction-1_NaWi.bz2'))<jupyter_output><empty_output><jupyter_text>Number of agents<jupyter_code>N_students_all = len([n for n in G_all.nodes(data=True) if n[1]['type'] == 'unistudent'])
N_lecturers_all = len([n for n in G_all.nodes(data=True) if n[1]['type'] == 'lecturer'])
print('Full network: {} students, {} lecturers'.format(N_students_all, N_lecturers_all))
N_students_TU = len([n for n in G_TU.nodes(data=True) if n[1]['type'] == 'unistudent'])
N_lecturers_TU = len([n for n in G_TU.nodes(data=True) if n[1]['type'] == 'lecturer'])
print('TU only network: {} students, {} lecturers'.format(N_students_TU, N_lecturers_TU))
N_students_NaWi = len([n for n in G_NaWi.nodes(data=True) if n[1]['type'] == 'unistudent'])
N_lecturers_NaWi = len([n for n in G_NaWi.nodes(data=True) if n[1]['type'] == 'lecturer'])
print('NaWi only network: {} students, {} lecturers'.format(N_students_NaWi, N_lecturers_NaWi))<jupyter_output>NaWi only network: 2675 students, 533 lecturers<jupyter_text>Average node degree<jupyter_code>weekdays = {1:'Monday', 2:'Tuesday', 3:'Wednesday', 4:'Thursday',
            5:'Friday', 6:'Saturday', 7:'Sunday'}
degree_df_all = pd.DataFrame()
weekday_connections = {}
all_edges = G_all.edges(keys=True, data='weekday')

for j in range(1, 8):
    wd_edges = [(u, v, k) for (u, v, k, wd) in all_edges if wd == j]
    G_wd = G_all.edge_subgraph(wd_edges).copy()

    students = [x for x,y in G_wd.nodes(data=True) if y['type'] == 'unistudent']
    lecturers = [x for x,y in G_wd.nodes(data=True) if y['type'] == 'lecturer']

    student_degree = np.asarray([G_wd.degree(s) for s in students]).mean()
    lecturer_degree = np.asarray([G_wd.degree(t) for t in lecturers]).mean()

    degree_df_all = degree_df_all.append({
        'weekday':weekdays[j],
        'student_degree':student_degree,
        'lecturer_degree':lecturer_degree,
    }, ignore_index=True)
            
degree_df_all.drop([5, 6])['student_degree'].mean()
degree_df_all.drop([5, 6])['lecturer_degree'].mean()
degree_df_TU = pd.DataFrame()
weekday_connections = {}
all_edges = G_TU.edges(keys=True, data='weekday')

for j in range(1, 8):
    wd_edges = [(u, v, k) for (u, v, k, wd) in all_edges if wd == j]
    G_wd = G_TU.edge_subgraph(wd_edges).copy()

    students = [x for x,y in G_wd.nodes(data=True) if y['type'] == 'unistudent']
    lecturers = [x for x,y in G_wd.nodes(data=True) if y['type'] == 'lecturer']

    student_degree = np.asarray([G_wd.degree(s) for s in students]).mean()
    lecturer_degree = np.asarray([G_wd.degree(t) for t in lecturers]).mean()

    degree_df_TU = degree_df_TU.append({
        'weekday':weekdays[j],
        'student_degree':student_degree,
        'lecturer_degree':lecturer_degree,
    }, ignore_index=True)
            
degree_df_TU.drop([5, 6])['student_degree'].mean()
degree_df_TU.drop([5, 6])['lecturer_degree'].mean()
degree_df_NaWi = pd.DataFrame()
weekday_connections = {}
all_edges = G_NaWi.edges(keys=True, data='weekday')

for j in range(1, 8):
    wd_edges = [(u, v, k) for (u, v, k, wd) in all_edges if wd == j]
    G_wd = G_NaWi.edge_subgraph(wd_edges).copy()

    students = [x for x,y in G_wd.nodes(data=True) if y['type'] == 'unistudent']
    lecturers = [x for x,y in G_wd.nodes(data=True) if y['type'] == 'lecturer']

    student_degree = np.asarray([G_wd.degree(s) for s in students]).mean()
    lecturer_degree = np.asarray([G_wd.degree(t) for t in lecturers]).mean()

    degree_df_NaWi = degree_df_NaWi.append({
        'weekday':weekdays[j],
        'student_degree':student_degree,
        'lecturer_degree':lecturer_degree,
    }, ignore_index=True)
            
degree_df_NaWi.drop([5, 6])['student_degree'].mean()
degree_df_NaWi.drop([5, 6])['lecturer_degree'].mean()<jupyter_output><empty_output><jupyter_text>Degree distributions<jupyter_code># get each node's degree on Monday
all_edges = G_all.edges(keys=True, data='weekday')
wd_edges = [(u, v, k) for (u, v, k, wd) in all_edges if wd == 1]
G_wd = G_all.edge_subgraph(wd_edges).copy()
students_all = np.asarray([G_wd.degree(x) for x,y in G_wd.nodes(data=True) if y['type'] == 'unistudent'])
lecturers_all = np.asarray([G_wd.degree(x) for x,y in G_wd.nodes(data=True) if y['type'] == 'lecturer'])

all_edges = G_TU.edges(keys=True, data='weekday')
wd_edges = [(u, v, k) for (u, v, k, wd) in all_edges if wd == 1]
G_wd = G_TU.edge_subgraph(wd_edges).copy()
students_TU = np.asarray([G_wd.degree(x) for x,y in G_wd.nodes(data=True) if y['type'] == 'unistudent'])
lecturers_TU = np.asarray([G_wd.degree(x) for x,y in G_wd.nodes(data=True) if y['type'] == 'lecturer'])

all_edges = G_NaWi.edges(keys=True, data='weekday')
wd_edges = [(u, v, k) for (u, v, k, wd) in all_edges if wd == 1]
G_wd = G_NaWi.edge_subgraph(wd_edges).copy()
students_NaWi = np.asarray([G_wd.degree(x) for x,y in G_wd.nodes(data=True) if y['type'] == 'unistudent'])
lecturers_NaWi = np.asarray([G_wd.degree(x) for x,y in G_wd.nodes(data=True) if y['type'] == 'lecturer'])
dst = '../plots'

fig, axes = plt.subplots(2, 3, figsize=(12, 5))
cutoff = 100
cmap = sns.color_palette("mako", as_cmap=True)
c1 = cmap(0.75)
c2 = cmap(0.25)

ax = axes[0][0]
degrees_all_masked = np.where(students_all > cutoff, cutoff, students_all)
ax.hist(degrees_all_masked, bins=range(0, cutoff + 1, 2), density=True, color=c1)
ax.set_ylabel('density')
ax.set_title('all students')

ax = axes[0][1]
degrees_TU_masked = np.where(students_TU > cutoff, cutoff, students_TU)
ax.hist(degrees_TU_masked, bins=range(0, cutoff + 1, 2), density=True, color=c1)
ax.set_title('only TU students')

ax = axes[0][2]
degrees_NaWi_masked = np.where(students_NaWi > cutoff, cutoff, students_NaWi)
ax.hist(degrees_NaWi_masked, bins=range(0, cutoff + 1, 2), density=True, color=c1)
ax.set_title('only NaWi students')

for ax in axes[0]:
    ax.set_xticks([])
    ax.set_ylim(0, 0.12)
    ax.text(75, 0.1, 'students', fontsize=12)

ax = axes[1][0]
degrees_all_masked = np.where(lecturers_all > cutoff, cutoff, lecturers_all)
ax.hist(degrees_all_masked, bins=range(0, cutoff + 1, 2), density=True, color=c2)
ax.set_ylabel('density')

ax = axes[1][1]
degrees_TU_masked = np.where(lecturers_TU > cutoff, cutoff, lecturers_TU)
ax.hist(degrees_TU_masked, bins=range(0, cutoff + 1, 2), density=True, color=c2)

ax = axes[1][2]
degrees_NaWi_masked = np.where(lecturers_NaWi > cutoff, cutoff, lecturers_NaWi)
ax.hist(degrees_NaWi_masked, bins=range(0, cutoff + 1, 2), density=True, color=c2)


for ax in axes[1]:
    ax.set_xticks(range(0, cutoff + 1, 20))
    ax.set_xticklabels([0, 20, 40, 60, 80, '>=100'])
    ax.set_xlabel('node degree')
    ax.set_ylim(0, 0.12)
    ax.text(75, 0.10, 'faculty', fontsize=12)
    
for ax in axes[0][1:]:
    ax.set_yticks([])
for ax in axes[1][1:]:
    ax.set_yticks([])

for ax in axes.flatten():
    ax.spines['right'].set_visible(False)
    ax.spines['top'].set_visible(False)


fig.tight_layout()


params = {"ytick.color" : "k",
          "xtick.color" : "k",
          "axes.labelcolor" : "k",
          "axes.edgecolor" : "k",
          "axes.facecolor": "w",
          "savefig.facecolor": "w"}
plt.rcParams.update(params)
plt.savefig(join(dst, 'degree_distributios.svg'))<jupyter_output><empty_output><jupyter_text>Percentage of agents and dates<jupyter_code>src = '../data/processed'
students = pd.read_csv('../data/processed/students_processed.csv')
students_raw = pd.read_csv('../data/raw/Grunddaten-Simulation/Studierende_pro_LV_mit_Idf.csv')
students_raw = students_raw.rename(columns={
    'ST_PERSON_NR':'student_id', # unique student identifier
    'STP_SP_NR':'lecture_id', # unique lecture identifier
    'LV_GRP_NR':'group_id', # (almost) unique group identifier 
    'STUDIENIDENTIFIKATOR':'study_id'}) # unique study identifier
N_students_raw = len(students_raw['student_id'].unique())
print('{} unique students were active in WiSe 2019/20'.format(N1))

print('{} ({:1.2f}%) unique students remain in the network'\
      .format(N_students_all, N_students_all/N_students_raw * 100))
dates = pd.read_csv('../data/processed/dates_all_students.csv')
N_dates_raw = len(dates[dates['lecture_id'].isin(students_raw['lecture_id'])])
students_all = [n[0] for n in G_all.nodes(data=True) if n[1]['type'] == 'unistudent']
students_processed = students[students['student_id'].isin(students_all)]
N_dates_processed = len(dates[dates['lecture_id'].isin(students_processed['lecture_id'])])
print('{}/{} ({:1.2f}%) dates remain in the network'\
    .format(N_dates_raw, N_dates_processed, N_dates_processed/N_dates_raw * 100))<jupyter_output>25379/24924 (98.21%) dates remain in the network<jupyter_text>Network modularity<jupyter_code>from networkx.algorithms.community.modularity_max import greedy_modularity_communities
from networkx.algorithms.community.quality import modularity
partition = greedy_modularity_communities(G_all)
modularity(G_all, partition)<jupyter_output><empty_output><jupyter_text>Export network as node- and edgelist<jupyter_code>nx.readwrite.edgelist.write_edgelist(G_all, "../data/networks/university_2019-10-16_to_2019-10-23_fraction-1_all_edgelist.csv")
nodelist = pd.DataFrame()
for n in G_all.nodes(data=True):
    node_data = {"ID":n[0]}
    node_data.update(n[1])
    nodelist = nodelist.append(node_data, ignore_index=True)
    
nodelist = nodelist.drop(columns=["unit"])
nodelist["ID"] = nodelist["ID"].astype(int)
nodelist.to_csv("../data/networks/university_2019-10-16_to_2019-10-23_fraction-1_all_nodelist.csv")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1645682561713096, 'fraction_numerical': 0.025676613462873005, 'mean_word_length': 5.384810126582279, 'pattern_counts': {'":': 3, '<': 27, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '178161', 'license': 'MIT', 'n_tokens_mistral': 4264, 'n_tokens_neox': 3982, 'n_words': 745, 'path': 'code/network/analysis_network.ipynb', 'repo_name': 'JanaLasser/uni_SEIRX', 'size': '10087'}