<jupyter_start><jupyter_text>Analyze the LASSO results- Inputs: LASSO coefficients from the 3 * 3 regressions.- Analyzing the performance, key coefficients, etc.<jupyter_code>import numpy as np
import pandas as pd
import geopandas as gpd
import networkx as nx
import matplotlib.pyplot as plt
import pickle
import copy
import scipy.sparse as sp
from scipy.sparse import csr_matrix
from sklearn.linear_model import LinearRegression

# regression tools
import statsmodels.api as sm
from sklearn.preprocessing import normalize
with open('../../data/05_model_outputs/lasso_performance.pickle', 'rb') as f:
    performance_dic = pickle.load(f)
    
with open('../../data/05_model_outputs/lasso_coefficients.pickle', 'rb') as f:
    coeff_dic = pickle.load(f)<jupyter_output><empty_output><jupyter_text>Obtain the key activities<jupyter_code># Export all the important activities
import itertools

city_list = ['total', 'boston', 'chicago', 'miami']
output_list = ['inc_median_household_2018', 'property_value_median_2018', 
               'rent_median_2018']

### 
coeff_dic_for_df = {}

# pd.DataFrame(np.zeros((15, 3*3)), 
#                         columns = list(itertools.product(city_list, output_list)))

for city in city_list:
    for output in output_list:
        print(city, output)
        coeff_dic_for_df[(city, output)] = copy.copy(coeff_dic[city][output]['lasso (no socio-demographics)'])
        coeff_dic_for_df[(city, output)] = coeff_dic_for_df[(city, output)].reset_index()
        
pd.concat(coeff_dic_for_df, axis = 1)
# Export all the important activities
import itertools

city_list = ['total', 'boston', 'chicago', 'miami']
output_list = ['inc_median_household_2018', 'property_value_median_2018', 
               'rent_median_2018']

### 
coeff_dic_with_socio_for_df = {}

# pd.DataFrame(np.zeros((15, 3*3)), 
#                         columns = list(itertools.product(city_list, output_list)))

for city in city_list:
    for output in output_list:
        print(city, output)
        coeff_dic_with_socio_for_df[(city, output)] = copy.copy(coeff_dic[city][output]['lasso (with socio-demographics)'])
        coeff_dic_with_socio_for_df[(city, output)] = coeff_dic_with_socio_for_df[(city, output)].reset_index()
        
pd.concat(coeff_dic_with_socio_for_df, axis = 1)<jupyter_output><empty_output><jupyter_text>Top performances<jupyter_code>import itertools

top_performance_dic = {}
city_list = ['total', 'boston', 'chicago', 'miami']
output_list = ['inc_median_household_2018', 'property_value_median_2018', 'rent_median_2018']
perform_category_list = ['baseline', 'lasso (no socio-demographics)', 'lasso (with socio-demographics)']

for city in city_list:
    top_performance_dic[city] = {}
    for output in output_list:
        for perform_category in perform_category_list:
            if perform_category == 'baseline':
                top_performance_dic[city][(output, perform_category)] = performance_dic[city][output][perform_category]
                
            else:
                max_col = np.argmax(performance_dic[city][output][perform_category].loc['testing',:])
                top_performance_dic[city][(output, perform_category)] = performance_dic[city][output][perform_category].iloc[:, max_col]
            
pd.concat(top_performance_dic['total'], axis = 1)
pd.concat(top_performance_dic['boston'], axis = 1)
pd.concat(top_performance_dic['chicago'], axis = 1)
pd.concat(top_performance_dic['miami'], axis = 1)<jupyter_output><empty_output><jupyter_text>Visualization (Need to be revised)<jupyter_code>def plot_multiple_lines_ax(names, x, x_label, d1, d2, d3, title):
    # overall style
    # x_label set up is still not ideal...
    plt.style.use('ggplot') # set the style; or "ggplot", "dark_background", "classic";
    fig = plt.figure(figsize = (15, 8))
    ax = plt.axes()
    ax2 = ax.twinx()

    ax.plot(x, d1, label = names[0])
    ax.plot(x, d2, label = names[1])
    ax.set_title(title)
    ax.set(xlabel = 'X', ylabel = "Y")
    for item in ([ax.title, ax.xaxis.label, ax.yaxis.label] +
                 ax.get_xticklabels() + ax.get_yticklabels()):
        item.set_fontsize(15) # adjust all font size...
    ax.title.set_fontsize(20)
    
    ax.legend(loc = 2, fontsize = 20, title = None) # remove the legend title of "City_province"
    ax.set_xlabel("Alpha (regularization)")
    ax.set_ylabel("R2")
    
    ax2.bar(x, d3, alpha = 0.2)
    for c in ax2.containers:
        ax2.bar_label(c)

    ax2.set_ylabel("Coefficient Counts")
#     ax.set_xticks([2001, 2007, 2014]); 
    ax.set_xticklabels(x_label)    


performance_dic['boston']['inc_median_household_2018']['lasso (no socio-demographics)']

list(performance_dic['boston']['inc_median_household_2018']['lasso (no socio-demographics)'].columns[3:])
# income
x = np.arange(9)
x_label = list(performance_dic['boston']['inc_median_household_2018']['lasso (no socio-demographics)'].columns[2:-1])
Training = list(performance_dic['boston']['inc_median_household_2018']['lasso (no socio-demographics)'].iloc[0, 3:])
Testing = list(performance_dic['boston']['inc_median_household_2018']['lasso (no socio-demographics)'].iloc[1, 3:])
Counts = list(performance_dic['boston']['inc_median_household_2018']['lasso (no socio-demographics)'].iloc[2, 3:])
names = ['training R2', 'testing R2']
title = "Predicting HH income in 2018"
plot_multiple_lines_ax(names, x, x_label, Training, Testing, Counts, title)
plt.show()

performance_dic['property_value_median_2018']['lasso (no socio-demographics)']
# property value
x = np.arange(6)
x_label = list(performance_dic['property_value_median_2018']['lasso (no socio-demographics)'].columns[2:-1])
Training = list(performance_dic['property_value_median_2018']['lasso (no socio-demographics)'].iloc[0, 3:])
Testing = list(performance_dic['property_value_median_2018']['lasso (no socio-demographics)'].iloc[1, 3:])
Counts = list(performance_dic['property_value_median_2018']['lasso (no socio-demographics)'].iloc[2, 3:])
names = ['training R2', 'testing R2']
title = "Predicting median property values in 2018"
plot_multiple_lines_ax(names, x, x_label, Training, Testing, Counts, title)
plt.show()
performance_dic['inc_per_capita_2018']['lasso (with socio-demographics)']
# income
x = np.arange(7)
x_label = list(performance_dic['inc_per_capita_2018']['lasso (with socio-demographics)'].columns[1:-1])
Training = list(performance_dic['inc_per_capita_2018']['lasso (with socio-demographics)'].iloc[0, 2:])
Testing = list(performance_dic['inc_per_capita_2018']['lasso (with socio-demographics)'].iloc[1, 2:])
Counts = list(performance_dic['inc_per_capita_2018']['lasso (with socio-demographics)'].iloc[2, 2:])
names = ['training R2', 'testing R2']
title = "Predicting income per capita in 2018"
plot_multiple_lines_ax(names, x, x_label, Training, Testing, Counts, title)
plt.show()

performance_dic['property_value_median_2018']['lasso (with socio-demographics)']
# income
x = np.arange(7)
x_label = list(performance_dic['property_value_median_2018']['lasso (with socio-demographics)'].columns[1:-1])
Training = list(performance_dic['property_value_median_2018']['lasso (with socio-demographics)'].iloc[0, 2:])
Testing = list(performance_dic['property_value_median_2018']['lasso (with socio-demographics)'].iloc[1, 2:])
Counts = list(performance_dic['property_value_median_2018']['lasso (with socio-demographics)'].iloc[2, 2:])
names = ['training R2', 'testing R2']
title = "Predicting median property value in 2018"
plot_multiple_lines_ax(names, x, x_label, Training, Testing, Counts, title)
plt.show()
coeff_dic['inc_per_capita_2018']['lasso (no socio-demographics)'].sort_values('value')

coeff_dic['property_value_median_2018']['lasso (no socio-demographics)'].sort_values('value')

coeff_dic['inc_per_capita_2018']['lasso (with socio-demographics)'].sort_values('value')

coeff_dic['property_value_median_2018']['lasso (with socio-demographics)'].sort_values('value')

set1 = set(coeff_dic['inc_per_capita_2018']['lasso (no socio-demographics)'].index).union(set(coeff_dic['property_value_median_2018']['lasso (no socio-demographics)'].index))
set2 = set(coeff_dic['inc_per_capita_2018']['lasso (with socio-demographics)'].index).union(set(coeff_dic['property_value_median_2018']['lasso (with socio-demographics)'].index))
full_set = set1.union(set2)
full_set
set(coeff_dic['property_value_median_2018']['lasso (no socio-demographics)'])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14609541894137196, 'fraction_numerical': 0.03311179681936862, 'mean_word_length': 5.873572593800978, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '382242', 'license': 'MIT', 'n_tokens_mistral': 3254, 'n_tokens_neox': 2911, 'n_words': 532, 'path': 'src/06_report/lasso_results.ipynb', 'repo_name': 'cjsyzwsh/economic_growth_usa', 'size': '8426'}