<jupyter_start><jupyter_text>Describe the graph properties Observe the graph properties: density, connectivity, diameter, etc. Note the benchmarks from <NAME>'s MSN network.- Degree distribution: heavily skewed (power law), avg degree = 14.4- Path length: 6.6- Clustering coefficient: 0.11- Connectivity: giant component<NAME>, <NAME>. Worldwide Buzz: Planetary-Scale Views on an Instant-Messaging Network. Proc. International WWW Conference, 2008https://chih-ling-hsu.github.io/2020/05/15/network-exampleconnectivity<jupyter_code>import numpy as np
import pandas as pd
import geopandas as gpd
import networkx as nx
import matplotlib.pyplot as plt
import pickle
import copy
import scipy.sparse as sp
from scipy.sparse import csr_matrix

import torch
from torch_geometric.data import Data
from torch_geometric.loader import DataLoader
# read files.
with open("../data/02_intermediate/place_graph_A.pickle", 'rb') as f:
    A_place = pickle.load(f) # sparse matrix

with open("../data/02_intermediate/place_graph_weighted_A.pickle", 'rb') as f:
    A_weighted_place = pickle.load(f) # sparse matrix    

A_weighted_place.shape<jupyter_output><empty_output><jupyter_text>Adjacency matrix with threshold filters  Threshold = 0<jupyter_code>A_weighted_array = A_weighted_place.toarray()
print(np.sum(A_weighted_array))
pd.DataFrame(A_weighted_array.reshape(-1, 1)).describe()
epsilon = 0.0
# create a threshold to filter the weighting array.
import copy
A_threshold = copy.copy(A_weighted_array)

import copy
A_threshold = copy.copy(A_weighted_array)
smaller_than_threshold_mask = A_threshold < epsilon
A_threshold[smaller_than_threshold_mask] = 0.0
larger_than_threshold_mask = A_threshold > epsilon
A_threshold[larger_than_threshold_mask] = 1.0

print("Num of edges in the new adjacency matrix: ", np.sum(A_threshold))
print("Num of edges in the initial adjacency matrix: ", np.sum(A_place))
print("Total number of potential edges: ", A_place.shape[0]*A_place.shape[1])

# Q: what is the right density in a graph?

A_threshold
# Obtain the characters of the A_threshold graph. 
# density, diameter, etc. 
G = nx.from_numpy_matrix(A_threshold)
# density
print(nx.density(G))
# degree distribution.
degree_dict = dict(nx.degree(G))
degree_counts = degree_dict.values()

degree_hist, degree_bins, _ = plt.hist(degree_counts, bins=100)
plt.close()

fig, ax = plt.subplots(figsize = (5, 5))
ax.loglog(degree_bins[1:], degree_hist)
ax.set_xscale('log')
ax.set_yscale('log')
ax.set(title = 'Degree Distribution (Log-Log)')

print("Average degree is: ", np.mean(list(degree_counts)))

# print(hist)
# print(bins)
# components
print(nx.number_connected_components(G))
# clustering coeffcient.
clustering_dict = nx.clustering(G)
clustering_keys = clustering_dict.keys()
clustering_values = clustering_dict.values()
# average clustering coefficient
print("average clustering coefficient: ", np.mean(clustering_values))

# clustering_values
degree_counts

degree_cluster_df = pd.DataFrame([list(degree_counts), list(clustering_values)]).T
degree_cluster_df.columns = ['degree', 'clustering_coeff']
degree_cluster_df = degree_cluster_df.sort_values('degree').reset_index()
degree_cluster_df

fig, ax = plt.subplots(figsize = (5, 5))
ax.loglog(degree_cluster_df['degree'], degree_cluster_df['clustering_coeff'])
ax.set_xscale('log')
ax.set_yscale('log')
ax.set_xlabel('Degree')
ax.set_xlabel('Clustering Coefficients')
ax.set(title = 'Degree - Clustering Coefficients (Log-Log)')
# Component diameter (VERY VERY SLOW)
largest_G_nodes = max(nx.connected_components(G), key=len)
largest_sub_G = G.subgraph(largest_G_nodes)
print(nx.diameter(largest_sub_G)) # it takes time to compute.<jupyter_output><empty_output><jupyter_text>Threshold = 1000<jupyter_code>A_weighted_array = A_weighted_place.toarray()
print(np.sum(A_weighted_array))
# describe
pd.DataFrame(A_weighted_array.reshape(-1, 1)).describe()
# Set up the threshold
epsilon = 1000.0
# create a threshold to filter the weighting array.
import copy
A_threshold = copy.copy(A_weighted_array)

import copy
A_threshold = copy.copy(A_weighted_array)
smaller_than_threshold_mask = A_threshold < epsilon
A_threshold[smaller_than_threshold_mask] = 0.0
larger_than_threshold_mask = A_threshold > epsilon
A_threshold[larger_than_threshold_mask] = 1.0

print("Num of edges in the new adjacency matrix: ", np.sum(A_threshold))
print("Num of edges in the initial adjacency matrix: ", np.sum(A_place))
print("Total number of potential edges: ", A_place.shape[0]*A_place.shape[1])

# Q: what is the right density in a graph?

A_threshold
# Obtain the characters of the A_threshold graph. 
# density, diameter, etc. 
G = nx.from_numpy_matrix(A_threshold)
# density
print(nx.density(G))
# degree distribution.
degree_dict = dict(nx.degree(G))
degree_counts = degree_dict.values()

degree_hist, degree_bins, _ = plt.hist(degree_counts, bins=100)
plt.close()

fig, ax = plt.subplots(figsize = (5, 5))
ax.loglog(degree_bins[1:], degree_hist)
ax.set_xscale('log')
ax.set_yscale('log')
ax.set(title = 'Degree Distribution (Log-Log)')

print("Average degree is: ", np.mean(list(degree_counts)))

# print(hist)
# print(bins)

# components
print(nx.number_connected_components(G))
# components (giant component!)
S = [len(c.copy()) for c in nx.connected_components(G)]
S

connect_hist, connect_bins, _ = plt.hist(S, bins=100)
plt.close()

fig, ax = plt.subplots(figsize = (5, 5))
ax.loglog(connect_bins[1:], connect_hist)
ax.set_xscale('log')
ax.set_yscale('log')
ax.set(title = 'Size of Connected Components (Log-Log)')

# clustering coeffcient.
clustering_dict = nx.clustering(G)
clustering_keys = clustering_dict.keys()
clustering_values = clustering_dict.values()
# average clustering coefficient
print("Average clustering coefficient is: ", np.mean(list(clustering_values)))
# clustering_values
degree_counts

degree_cluster_df = pd.DataFrame([list(degree_counts), list(clustering_values)]).T
degree_cluster_df.columns = ['degree', 'clustering_coeff']
degree_cluster_df = degree_cluster_df.sort_values('degree').reset_index()
degree_cluster_df

fig, ax = plt.subplots(figsize = (5, 5))
ax.loglog(degree_cluster_df['degree'], degree_cluster_df['clustering_coeff'])
ax.set_xscale('log')
ax.set_yscale('log')
ax.set_xlabel('Degree')
ax.set_xlabel('Clustering Coefficients')
ax.set(title = 'Degree - Clustering Coefficients (Log-Log)')

# Component diameter (VERY VERY SLOW)
largest_G_nodes = max(nx.connected_components(G), key=len)
largest_sub_G = G.subgraph(largest_G_nodes)
print(nx.diameter(largest_sub_G)) # it takes time to compute.<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1204069237777103, 'fraction_numerical': 0.011235955056179775, 'mean_word_length': 6.964933494558646, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '612845', 'license': 'MIT', 'n_tokens_mistral': 2400, 'n_tokens_neox': 2244, 'n_words': 498, 'path': 'src/06_report/.ipynb_checkpoints/describe_graph_properties-checkpoint.ipynb', 'repo_name': 'cjsyzwsh/economic_growth_usa', 'size': '6602'}