<jupyter_start><jupyter_text>**Task 1: Relationship between the Survey network and the correlation network.**<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import networkx as nx
import random
df = pd.read_excel("./EE325_Final Marks.xlsx")
df.tail(5)
# do NOT run this cell
sur_df = pd.read_csv('./Study_Groups.tsv', sep='\t')
sur_df.drop(columns=['Timestamp'], inplace=True)
sur_df.columns = ['email', 'friends']

name_dict = {}
email_dict = {}
for i in range(df.shape[0]):
    email_dict[df['eMail'][i]] = i
    name_dict[df['Student Name'][i]] = i

name_dict['I always study alone'] = -1

adj_list = {}
for i in range(sur_df.shape[0]):
    adj_list[email_dict[sur_df['email'][i]]] = sur_df['friends'][i].split(', ')

for k in adj_list.keys():
    for i in range(len(adj_list[k])):
        adj_list[k][i] = name_dict[adj_list[k][i]]

'''import pickle

with open('survey_adj_list.pickle', 'wb') as handle:
    pickle.dump(adj_list, handle, protocol=pickle.HIGHEST_PROTOCOL)

handle.close()'''
import matplotlib.pyplot as plt
import networkx as nx
import pandas as pd
import numpy as np
from random import random
from random import randint
from networkx.algorithms.community import naive_greedy_modularity_communities

def jaccard(x1, x2): # where x1 and x2 are two lists of nodes
    x1, x2 = set(x1), set(x2)
    jcc = len(x1.intersection(x2))/len(x1.union(x2))
    return jcc

def structural_analysis(m1_data,m2_data):
    print("Number of clusters in Month 1 =",len(m1_data))
    print("Number of clusters in Month 2 =",len(m2_data))

    for i in range(0,60):
        node = i
        cluster1=[]
        cluster2=[]
        for cluster in m1_data:
            if node in cluster:
                cluster1=cluster
                break
        for cluster in m2_data:
            if node in cluster:
                cluster2=cluster
                break
        print('Similarty measure for node {}:'.format(node))
        print(jaccard(cluster1,cluster2))
    return 


def corr_network(X, t = 0.8):
    corr = np.corrcoef(X)
    # corr = 0.5 + 0.5*corr
    adj = corr - np.eye(60)
    adj = (adj >= t)
    g = nx.from_numpy_matrix(adj)
    return g


def community_detection(G):
    node_groups = list(naive_greedy_modularity_communities(G))
    total_comm = len(node_groups)
    color_map = []
    colors = [(random(),random(),random()) for _i in range(total_comm)]
    for node in G:
        for i in range(total_comm):
            if node in node_groups[i]:
                color_map.append(colors[i]) 
    pos = nx.spring_layout(G, k=0.15, iterations=20)
    nx.draw(G,pos,node_size = 5, width = 0.4,node_color=color_map, with_labels=True)
    plt.show()
    return node_groups
import pickle

with open('survey_adj_list.pickle', 'rb') as handle:
    adj_list = pickle.load(handle)

handle.close()

keys = adj_list.keys()
for k in keys:
    if k in adj_list[k]:
        adj_list[k].remove(k)
    if -1 in adj_list[k]:
        adj_list[k].remove(-1)

invalid = []
for k in keys:
    if len(adj_list[k]) == 0:
        invalid.append(k)

adj_copy = adj_list.copy()
for k in keys:
    if k in invalid:
        del adj_copy[k]


import pickle

with open('survey_adj_list_clean1.pickle', 'wb') as handle:
    pickle.dump(adj_copy, handle, protocol=pickle.HIGHEST_PROTOCOL)

handle.close()<jupyter_output><empty_output><jupyter_text>Start from here<jupyter_code>import pickle

with open('survey_adj_list_clean1.pickle', 'rb') as handle:
    adj_list = pickle.load(handle)

handle.close()
sur_net = nx.from_dict_of_lists(adj_list)
nx.draw(sur_net, node_size = 5, width = 0.4)
plt.show()
marks = [np.mean(df.iloc[:,[4,5,6,7,8]], axis = 1)[n] for n in sur_net.nodes()]
from pyvis.network import Network
net = Network('500px', '800px',notebook=True)
net.repulsion(node_distance=400, spring_length=200)

net.add_nodes(list(sur_net.nodes()), value = marks)
for i,j in sur_net.edges():
    net.add_edge(i,j)
    
net.show('sur.html')

def degree_dist(graph, log= False, w = None):
    degrees = [graph.degree(n, weight = w) for n in graph.nodes()]
    degrees = np.asarray(degrees)
    if log: dist =  np.log(degrees)
    else: dist =  degrees
    return dist
x = nx.to_numpy_array(sur_net)
d = np.sum(x, axis=1).reshape(-1,1)
d*np.eye(60)
def datacon(g1, g2, e = 1e-6):
    A1 = nx.to_numpy_array(g1)
    A2 = nx.to_numpy_array(g2)
    d1 = np.sum(A1, axis=1).reshape(-1,1)
    d2 = np.sum(A2, axis=1).reshape(-1,1)
    D1 = np.eye(A1.shape[0])*d1
    D2 = np.eye(A2.shape[0])*d2
    S1 = np.linalg.inv(np.eye(A1.shape[0]) + D1*(e**2) + A1*(-e))
    S2 = np.linalg.inv(np.eye(A2.shape[0]) + D2*(e**2) + A2*(-e))
    d = np.sqrt(np.sum(np.square(np.sqrt(S1) - np.sqrt(S2))))
    return d
    
barabasi = nx.barabasi_albert_graph(n = 60, m= 2)
dist_sur = degree_dist(sur_net, log=False)
plt.hist(dist_sur, bins=8, alpha = 0.5)
plt.show()
degrees = degree_dist(sur_net)
marks = np.asarray([df.iloc[:,17][n] for n in sur_net.nodes()])
np.corrcoef(x = marks, y = degrees)

def Monte_Carlo_ICM(g, p, seed, runs):
    estimate = []
    node_list = list(g.nodes())
    adj_list = nx.to_dict_of_lists(g)
    
    for t in range(runs):
        info_list = dict(zip(node_list, np.zeros((len(node_list),)) == 1))
        nx.set_node_attributes(g, info_list, "info")
        for s in seed:
            g.nodes[s]["info"] = True
        seed_proxy = seed.copy()

        for n in seed_proxy:
            for nei in adj_list[n]:
                if not(g.nodes[nei]["info"]):
                    toss = random.random()
                    if p > toss:
                        g.nodes[nei]["info"] = True
                        seed_proxy.append(nei)
        estimate.append(len(seed_proxy))
    return np.array(estimate, dtype=np.int32)
icm_sur = Monte_Carlo_ICM(sur_net, 0.1, [51], 10000)
round(icm_sur.mean())<jupyter_output><empty_output><jupyter_text>Correlation Network<jupyter_code>X = df.iloc[:60,4:9].to_numpy()
corr = np.corrcoef(X)
corr = 0.5 + 0.5*corr
adj = corr - np.eye(60)
#adj = adj >= 0.8
#marks_net = nx.from_numpy_matrix(adj)
datacon(sur_net, marks_net, e = 1e-12)
distances = []
for t in np.linspace(0.75,0.85, 22):
    adj_ = adj >= t
    marks_net = nx.from_numpy_matrix(adj_)
    d = datacon(marks_net, sur_net, 0.01)
    distances.append(d)
    print(t,'\t', d)
fig = plt.figure()
fig.set_figwidth(6)
fig.set_figheight(8)
plt.plot(distances)
plt.show()
icm_marks = Monte_Carlo_ICM(marks_net, 0.1, [51], 10000)
round(icm_marks.mean())
from sklearn.preprocessing import MinMaxScaler
marks_dist = degree_dist(marks_net, w='weight')
dist_sur = degree_dist(sur_net)
scale = MinMaxScaler()
marks_dist = scale.fit_transform(marks_dist.reshape(-1,1))
marks_dist -= np.mean(marks_dist)
scale = MinMaxScaler()
dist_sur = scale.fit_transform(dist_sur.reshape(-1,1))
dist_sur -= np.mean(dist_sur)
plt.hist(dist_sur, bins=8, alpha = 0.6)
plt.hist(marks_dist, bins=8, alpha = 0.6)
plt.legend(["survey", "corr"])
plt.show()
from sklearn.preprocessing import MinMaxScaler
marks_dist = degree_dist(marks_net)
dist_sur = degree_dist(sur_net)
#scale = MinMaxScaler()
#marks_dist = scale.fit_transform(marks_dist.reshape(-1,1))
#marks_dist -= np.mean(marks_dist)
#scale = MinMaxScaler()
#dist_sur = scale.fit_transform(dist_sur.reshape(-1,1))
#dist_sur -= np.mean(dist_sur)
plt.hist(dist_sur, bins=8, alpha = 0.6)
plt.hist(marks_dist, bins=8, alpha = 0.6)
plt.legend(["survey", "corr"])
plt.show()
marks_dist = degree_dist(marks_net)
dist_sur = degree_dist(sur_net)
sur_eig = nx.eigenvector_centrality(sur_net)
marks_eig = nx.eigenvector_centrality(marks_net, weight = 'weight')
eig_vec = np.asarray(list(sur_eig.values()))
marks_vec = np.asarray(list(marks_eig.values()))
np.corrcoef(x = dist_sur, y = marks_vec)
np.corrcoef(x = marks_dist, y = dist_sur)
adj = corr - np.eye(60)
adj = adj >= 0.8
def jaccard(x1, x2): # where x1 and x2 are two lists of nodes
    x1, x2 = set(x1), set(x2)
    jcc = len(x1.intersection(x2))/len(x1.union(x2))
    return jcc
for t in np.linspace(0.1, 0.9, num=1000):
    adj = corr - np.eye(60)
    adj = adj >= t
    print(jaccard(adj, sur_adj))
G = nx.from_numpy_matrix(adj)
names = list(df["Student Name"])
name_dict = dict()
for i in range(60):
    name_dict[i] = names[i]
pos = nx.spring_layout(sur_net, k=0.1, iterations=10)
nx.draw(sur_net, pos, node_size = 5, width = 0.4)
plt.show()
pos = nx.spring_layout(G, k=0.15, iterations=20)
nx.draw(G,pos, node_size = 5, width = 0.4,labels=name_dict, with_labels = True, font_size = 5)
plt.show()
import random
def ICM(G,initial_active_nodes,prob,iters):
    total_active_nodes = 0          # store total number of active nodes in each iterations
    for t in range(iters):
        active_nodes = initial_active_nodes.copy()   
        nodes_visited = []          # once a active node is considered, it should not consider it again
        
        # loop will continue if size of active_nodes is changing 
        while 1:
            new_active_nodes=[]
            # finding inactive node v, neighbour of active node u, such that rand probability <= diffusion probablity and is not visited 
            for u in active_nodes:
                if u not in nodes_visited:      # once a active node is considered, it should not consider it again
                    nodes_visited.append(u)
                    for v in nx.neighbors(G,u):
                        if v not in nodes_visited :     # active node should not be considered 
                            rand_p= random.random()
                            if rand_p <= prob:
                                new_active_nodes.append(v)

            if len(new_active_nodes)==0:            # if no change in new active nodes, break
                break
            active_nodes = new_active_nodes.copy()
        total_active_nodes+=len(nodes_visited)       # total active nodes after each iter
    return total_active_nodes/iters


def main1():
    nodes = 1000
    G = nx.barabasi_albert_graph(nodes,2)
    active_nodes = []
    while len(active_nodes)<50:
        x = random.randint(0,nodes-1)
        if x not in active_nodes:
            active_nodes.append(x)
    prob = random.uniform(0.05,0.15)
    iters = 10000
    print("Avg number of Active Nodes for ICM (prob = {}) after {} iters = {}".format(prob,iters,ICM(G,active_nodes,prob,iters)))
    prob = random.uniform(0.05,0.15)
    iters = 10000
    print("Avg number of Active Nodes for ICM (prob = {}) after {} iters = {}".format(prob,iters,ICM(G,active_nodes,prob,iters)))
    prob = random.uniform(0.05,0.15)
    iters = 10000
    print("Avg number of Active Nodes for ICM (prob = {}) after {} iters = {}".format(prob,iters,ICM(G,active_nodes,prob,iters)))
    return<jupyter_output><empty_output><jupyter_text>Task 3 and 4<jupyter_code>df = pd.read_excel("./EE325_Final Marks.xlsx")
df.head()
def data():
    df = pd.read_excel("./EE325_Final Marks.xlsx")
    df = df.replace('A',0)
    X = df.iloc[:60,5:16].to_numpy()
    X[:,9]/=2
    X[:,10]/=2.5
    X_new = np.delete(X,[5,6,7,8],1)
    return X_new

def main1():
    X = data()
    month1 = X[:,[0,1,2,5]]
    month2 = X[:,[3,4,6]]

    g1=corr_network(month1,0.8)
    g2=corr_network(month2,0.8)
    node_group1=community_detection(g1)
    node_group2=community_detection(g2)
    structural_analysis(node_group1,node_group2)
    return

main1()
def main2():
    X=data()
    g=corr_network(X,0.8)
    node_group=community_detection(g)
    print("Overall Number of clusters = ",len(node_group))

    degrees=[val for (node,val) in g.degree()]
    df=pd.read_excel("./EE325_Final Marks.xlsx")
    unique_nodes = []
    for i in range(60):
        if degrees[i]<2:
            unique_nodes.append(df.iloc[i,3])
    print(unique_nodes)

main2()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14210347174948945, 'fraction_numerical': 0.029952348536419333, 'mean_word_length': 3.305128205128205, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '431655', 'license': 'MIT', 'n_tokens_mistral': 4675, 'n_tokens_neox': 4465, 'n_words': 961, 'path': 'SNA_Project.ipynb', 'repo_name': 'ledalla2000/Student-Relationships-Based-on-Academic-Performance-using-Social-Network-Analysis', 'size': '11752'}