<jupyter_start><jupyter_text>Make Bratislava Blocks and graphThis notebook takes the Pokec graph data and processed metadata generated in **make_Pokec_Reduced_profiles.ipynb** as an input and outputs the lcc graph of nodes in the Bratislava region. We also output a coarse grained network consisting of 9 blocks made by differen combinations of the age and region name variables.The graph data is origianlly directed, however, we take the edges to be undirected.Created on 04/09/20<jupyter_code>import pandas as pd
import numpy as np
import networkx as nx
import tqdm
import geopy.distance
import matplotlib.pyplot as plt
import seaborn as sns

import Pokec_processing as PokProc
path_to_reduced_profiles = 'Data/reduced_pokec_profiles.csv'
all_profiles = pd.read_csv(path_to_reduced_profiles)
all_profiles.head()<jupyter_output><empty_output><jupyter_text>Filter missing valuesMissing ages and locations are set to 0.0. Filter these out.<jupyter_code>all_profiles = all_profiles.loc[(all_profiles['age'] > 0.0) & (all_profiles['user_lat'] > 0.0)]<jupyter_output><empty_output><jupyter_text>Use the 0,25,50,75 and 100th percentiles of the distribution of ages.  Set age and region bins<jupyter_code>ages=list(all_profiles['age'])
age_ranges = [(np.percentile(ages, 0), np.percentile(ages, 25)-1), (np.percentile(ages, 25), np.percentile(ages, 50)-1)
    , (np.percentile(ages, 50), np.percentile(ages, 75)-1), (np.percentile(ages, 75), np.percentile(ages, 100))]
print("age ranges")
print(age_ranges)

region_names = ['bratislavsky kraj, bratislava - okolie','bratislavsky kraj, bratislava - petrzalka',
'bratislavsky kraj, bratislava - ruzinov']
bratislava_profiles = all_profiles.loc[all_profiles['region'].isin(region_names)]

#Shorter naming convention with caps
bratislava_profiles['region'] = [ k.split(' ')[-1].title() for k in list(bratislava_profiles['region']) ]

region_names = [ k.split(' ')[-1].title() for k in region_names ]

bratislava_profiles
def age_range_string(age_ranges,index) :
    return f"ages_{int(age_ranges[index][0])}-{int(age_ranges[index][1])}"

def get_age_cat(age_ranges,age) :
    for group,age_range in enumerate(age_ranges) :
        if age_range[0] <= age <= age_range[1] :
            return age_range_string(age_ranges,group)

        
block_names = [ ]
block_ages = [ ]
for i in region_names :
    for j in range(len(age_ranges)) :
        block_names.append(i +'_'+ age_range_string(age_ranges,j) )
        block_ages.append(age_ranges[j])

bratislava_profiles['age_group'] = [get_age_cat(age_ranges,age) for age in list(bratislava_profiles['age'])]
bratislava_profiles['block'] = [ i +'_'+ j for i,j in zip(list(bratislava_profiles['region']),list(bratislava_profiles['age_group']))]
bratislava_profiles
bratislava_user_ids = list(bratislava_profiles['user_id'])
print(len(bratislava_user_ids))<jupyter_output>46106<jupyter_text>Read edge data<jupyter_code>def read_edge_list(path_to_edge_list) :
    print("Reading edge data...")
    edge_data = pd.read_csv(path_to_edge_list,sep='\t',names=['id_1','id_2'],dtype=np.int32)
    return edge_data
path_to_edge_list = 'Data/raw_data/soc-pokec-relationships.txt'

edge_data = read_edge_list(path_to_edge_list)
edge_data.head()
len(edge_data)
bratislava_edge_data = edge_data.loc[ (edge_data['id_1'].isin(bratislava_user_ids)) & (edge_data['id_2'].isin(bratislava_user_ids))]
print(len(bratislava_edge_data))<jupyter_output>91482<jupyter_text>Build graph and extract LCC<jupyter_code>graph = PokProc.make_graph_from_edge_data(bratislava_profiles,bratislava_edge_data,add_disconnected_nodes=False)
len(graph)
Gcc = sorted(nx.connected_components(graph), key=len, reverse=True)
lcc_graph = graph.subgraph(Gcc[0])
print(len(lcc_graph))
bratislava_lcc_profiles = bratislava_profiles.loc[bratislava_profiles['user_id'].isin(lcc_graph.nodes())]
len(bratislava_lcc_profiles)
bratislava_lcc_profiles.to_csv("Data/bratislava_profiles.csv")
nx.write_graphml(lcc_graph,"Data/Bratislava_graph.graphml")<jupyter_output><empty_output><jupyter_text>Extract Block information<jupyter_code>groups = [ bratislava_lcc_profiles.loc[bratislava_lcc_profiles['block']==block] for block in block_names]
groups_node_ids = [list(k['user_id']) for k in groups]
block_sizes = [len(k) for k in groups]

node_to_group_map = { i:j for i,j in zip(list(bratislava_lcc_profiles['user_id']),list(bratislava_lcc_profiles['block']))} # will this keep the correct order to map? 

for i,j in zip(block_names,block_sizes) :
    print(i,j)
assert np.sum(block_sizes) == len(lcc_graph)
block_info = pd.DataFrame({'block_name':block_names,'block_size':block_sizes,'average_age':[np.mean(k) for k in block_ages]})
block_info
block_info.to_csv("Data/block_info.csv")<jupyter_output><empty_output><jupyter_text>Count edges between blocks + make connection prob matrix<jupyter_code># could make the interaction dataframe at this stage?

edge_counts = np.zeros((len(groups_node_ids),len(groups_node_ids)))
i=0
for group_1 in tqdm.tqdm_notebook(groups_node_ids) :
    for j,group_2 in enumerate(groups_node_ids) :
        num_edges_between = len([ k for k in nx.edge_boundary(lcc_graph,group_1,nbunch2=group_2) ])
        edge_counts[i][j]=num_edges_between
    i+=1
def check_symmetric(X : np.ndarray) :
    np.testing.assert_array_equal( X - X.T , np.zeros((X.shape[0],X.shape[1])) )
check_symmetric(edge_counts)
def prob_mat_from_edge_counts(block_sizes :  list , edge_counts : np.ndarray) :

    prob_mat = np.zeros(edge_counts.shape)
    
    for i in range(edge_counts.shape[0]) :
        for j in range(edge_counts.shape[1]) : 
            
            if i == j :
                prob_mat[i][j] = (2.0/(block_sizes[i]*(block_sizes[i]-1.0)))*edge_counts[i,j]
            else :
                prob_mat[i][j] = (1.0/(block_sizes[i]*block_sizes[j]))*edge_counts[i,j]
                
    return prob_mat
prob_mat = prob_mat_from_edge_counts(block_sizes,edge_counts)
prob_mat[:4,:4]
coupling_mat = np.dot( prob_mat , np.diag(block_sizes ) )
print(coupling_mat[:4,:4])
coupling_graph = nx.from_numpy_matrix(coupling_mat)
nx.write_graphml(coupling_graph,"Data/Bratislava_coupling.graphml")<jupyter_output><empty_output><jupyter_text>Plotting of the coupling matrix (TBC - use this rather than dedicated script?!)<jupyter_code>block_names_with_newline = [ k.split('_')[0] + '\n' + k.split('_')[2]  for k in block_names]
fig,ax = plt.subplots(figsize=(6,6))
#plt.imshow(coupling_mat,cmap='plasma')

hm = sns.heatmap(coupling_mat,cmap='Blues')

plt.yticks(np.arange(0,len(block_names),1))
ax.set_yticklabels(block_names_with_newline, rotation=0,fontsize=11)
plt.xticks(np.arange(0,len(block_names),1))
ax.set_xticklabels(block_names_with_newline, rotation=90,fontsize=11)

cbar = ax.collections[0].colorbar
# here set the labelsize by 20
cbar.ax.tick_params(labelsize=20)

plt.text(-3.75,0.5,'(a)',fontsize=25)

#cbar = plt.colorbar()
plt.savefig("Plots/block_interactions.pdf",bbox_inches='tight')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12932395579158892, 'fraction_numerical': 0.016219319649777523, 'mean_word_length': 6.665566556655666, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '407325', 'license': 'MIT', 'n_tokens_mistral': 2756, 'n_tokens_neox': 2671, 'n_words': 440, 'path': 'Figures_3-4_Pokec/make_bratislava_graph_and_blocks.ipynb', 'repo_name': 'MGarrod1/unobserved_spin_influence', 'size': '6967'}