<jupyter_start><jupyter_text>Compare the graph parameters for the activities- degree distributions, clustering coefficient distributions, average degree, and average clustering coeffQ: Should I focus on the directed vs. undirected graph comparison???<jupyter_code>import numpy as np
import pandas as pd
import geopandas as gpd
import networkx as nx
import matplotlib.pyplot as plt
import pickle
import copy
from scipy.sparse import csr_matrix
import time
from sklearn.preprocessing import normalize
import sys
sys.path.append("../")
import utils
import importlib
importlib.reload(utils)
# read files
with open("../../data/02_intermediate/boston_stays.pickle", 'rb') as f:
    df_boston = pickle.load(f)

with open("../../data/02_intermediate/miami_stays.pickle", 'rb') as f:
    df_miami = pickle.load(f)

with open("../../data/02_intermediate/chicago_stays.pickle", 'rb') as f:
    df_chicago = pickle.load(f)

with open("../../data/03_processed/A_home_activity_three_cities_unweighted_dic.pickle", 'rb') as f:
    A_home_activity_unweighted_dic = pickle.load(f)
    
with open("../../data/03_processed/A_home_activity_three_cities_weighted_dic.pickle", 'rb') as f:
    A_home_activity_weighted_dic = pickle.load(f)
    
# read the spatial network dictionary.
with open("../../data/03_processed/spatial_network_boston_miami_chicago_dic.pickle", 'rb') as f:
    spatial_network_dic = pickle.load(f)
    
# read shapefiles
with open("../../data/02_intermediate/boston_miami_chicago_ct_shp_dic.pickle", 'rb') as f:
    shp_dic = pickle.load(f)

# read evaluation files
with open("../../data/05_model_outputs/lasso_coefficients.pickle", 'rb') as f:
    lasso_coef = pickle.load(f)
# check the coefficients
lasso_coef['boston']['inc_median_household_2018']['lasso (no socio-demographics)'].index

# check the mobility network
A_home_activity_unweighted_dic['boston'][1.0]['ATM']<jupyter_output><empty_output><jupyter_text>Turn the directed to undirected adj matrice (redundant...)<jupyter_code>def turn_asymmetric_directed_to_symmetric_undirected_adj(activity_dir_adj_df):
    ''' turn an asymmetric directed adj matrix to a symmetric directed one '''
#     activity_dir_adj_df = A_home_activity_unweighted_dic['boston'][1.0]['ATM']
    union_list = sorted(set(activity_dir_adj_df.columns).union(set(activity_dir_adj_df.index)))
    activity_zeros_df = pd.DataFrame(np.zeros((len(union_list), len(union_list))), 
                                     columns = union_list,
                                     index = union_list)
    activity_zeros_df = activity_zeros_df.add(activity_dir_adj_df, fill_value = 0.0)
#     print(activity_zeros_df)

    activity_undir_adj_df = pd.DataFrame(np.maximum(activity_zeros_df.values.T, activity_zeros_df.values),
                                   columns = union_list,
                                   index = union_list)
    return activity_undir_adj_df


A_home_activity_unweighted_sym_adj_dic = {}

for city in ['boston','chicago','miami']:
    print(city)
    A_home_activity_unweighted_sym_adj_dic[city] = {}
    for threshold in [1.0, 10.0]:
        print(threshold)
        A_home_activity_unweighted_sym_adj_dic[city][threshold] = {}
        for activity_name in A_home_activity_unweighted_dic[city][threshold].keys():
            A_home_activity_unweighted_sym_adj_dic[city][threshold][activity_name]=turn_asymmetric_directed_to_symmetric_undirected_adj(A_home_activity_unweighted_dic[city][threshold][activity_name])<jupyter_output>boston
1.0
10.0
chicago
1.0
10.0
miami
1.0
10.0<jupyter_text>Compute the graph parameters for activity categories<jupyter_code>###
beginning_time = time.time()
mobility_network_parameters_dic = {}
threshold = 1.0

for city in ['boston','chicago','miami']:
    print(city)
    mobility_network_parameters_dic[city]={}
    for idx in range(len(A_home_activity_unweighted_sym_adj_dic[city][threshold].keys())):
        activity_name = list(A_home_activity_unweighted_sym_adj_dic[city][threshold].keys())[idx]
        mobility_network_parameters_dic[city][activity_name] = {}
        current_time = time.time()
        elapse_time = current_time - beginning_time
        print(idx, activity_name, elapse_time/60.0, "minutes", end = '\r')

        # init the mobility graph
        adj_m = A_home_activity_unweighted_sym_adj_dic[city][threshold][activity_name]
        G_mobility = nx.from_pandas_adjacency(adj_m)

        # computing the parameters
        # degrees, degree distributions, avearge degree, clustering coeff distributions. 
        degrees = nx.degree(G_mobility)
        degree_distributions = np.unique(list(dict(nx.degree(G_mobility)).values()), return_counts = True)
        average_degree = np.mean(list(dict(degrees).values()))
        clustering = nx.clustering(G_mobility)
        average_clustering = np.mean(list(clustering.values()))        
        
        # save
        mobility_network_parameters_dic[city][activity_name]['degrees']=degrees
        mobility_network_parameters_dic[city][activity_name]['degree_distributions']=degree_distributions
        mobility_network_parameters_dic[city][activity_name]['average_degree']=average_degree
        mobility_network_parameters_dic[city][activity_name]['clustering']=clustering
        mobility_network_parameters_dic[city][activity_name]['average_clustering']=average_clustering        

# save!
with open('../../data/05_model_outputs/network_property_degree_clustering.pickle', 'wb') as f:
    pickle.dump(mobility_network_parameters_dic, f)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11012658227848102, 'fraction_numerical': 0.009584086799276673, 'mean_word_length': 4.5199600798403194, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '622376', 'license': 'MIT', 'n_tokens_mistral': 1882, 'n_tokens_neox': 1806, 'n_words': 299, 'path': 'src/04_modelling/model_02_weak_ties_structural_three_cities_furtherfurtherfurthermore.ipynb', 'repo_name': 'cjsyzwsh/economic_growth_usa', 'size': '5530'}