<jupyter_start><jupyter_text>GoalAnalyse the sctruture of adjacency matrices and their metrics over time and by congress Main Takeaways- The congress 111 is the most sparse one and with strong separation between republicans and democrats (we see it by averaging the adjacency matrics and through metrics of density and centrality)- For most of congresses, the the averaged adjacency matrix loses its shape on the last semester of the first year and returns to the behavior in the coming semester. However the same behavior can not be seen in the last semester of de congress (last semester of the second year). The number of laws proposed in the last semester of first year is lower as well as we saw in `analysis-historical-votes.ipynb`- Our hypothesis that the senators from democrats and republicans change their behavior by the end of the period does not hold if we analyse the averaged adjacency matrices for all congresses. On the other had, it is possible see that the connected components increases quite consistenly during the period Further AnalysisThe analysis of adjacency matrices and votes by subject and party that proposed it is available in `analysis-historical-by-subject.ipynb`<jupyter_code>import os
import pandas as pd
import numpy as np

import pickle
import networkx as nx

import altair as alt
import seaborn as sns
import matplotlib.pyplot as plt
%matplotlib inline 
ALL_LAWS_PATH = '../data/all-votes/laws_20years.csv'
ALL_ADJACENCY_PATH = '../data/all-votes/all_adjacency.pickle'
pal = {'republican': '#ff0000', 'democrats': '#0000ff'}
sns.set(context='notebook', style='darkgrid', palette='deep', font='sans-serif', font_scale=2, color_codes=True, rc=None)
all_laws = pd.read_csv(ALL_LAWS_PATH)
all_laws['law_id'] = all_laws['vote_uri'].apply(lambda x: '_'.join(x.split('/')[4:]).split('.')[0])
all_laws['q'] = pd.to_datetime(all_laws.date).dt.quarter
all_laws['quarter'] = all_laws.apply(lambda x: '1' '-' + str(x['q']) if x['year'] % 2==0 else '2' + '-' + str(x['q']), axis=1)
print(all_laws.shape)

with open(ALL_ADJACENCY_PATH, 'rb') as handle:
    adjacencies = pickle.load(handle)
print(len(adjacencies))
def get_metrics(a):
    m = {}
    
    m['shape'] = a.shape
    m['L'] = a.sum(axis=1).sum(axis=0)
#     m['degree'] = a.sum(axis=1)

    G = nx.Graph(a)
    eigenvalues = nx.laplacian_spectrum(G)
    m['eigenvalues'] = eigenvalues
    m['degree_centality'] = nx.degree_centrality(G)
    m['connected_components'] = len(list(nx.connected_component_subgraphs(G)))
    m['density'] = nx.density(G)
    
    return m
all_metrics = [(k, get_metrics(v)) for k, v in adjacencies.items()]
df_metrics = pd.DataFrame([x[1] for x in all_metrics])
df_metrics['law_id'] = [x[0] for x in all_metrics]
df_metrics = df_metrics.merge(all_laws)
pd.DataFrame(df_metrics.iloc[0].T)
df_metrics.to_csv('../data/all-votes/all_adjacency_metric.csv', index=False)<jupyter_output><empty_output><jupyter_text>Heatmap of adjacency matricies over time by congressps: matrices are ordered by Democrats -> Indepedent -> Republicans<jupyter_code>6%5
by = 'quarter'
quarters = sorted(all_laws[by].unique())
congress = all_laws['congress'].unique()

f, ax = plt.subplots(3, 4, figsize=(20, 10), squeeze=True )
for i, c in enumerate(congress):
    
    laws = list(all_laws[(all_laws.congress == c)]['law_id'])
    n = len(laws)
    sum_adjacencies = adjacencies[laws[0]]
    for l in laws[1:]:
        sum_adjacencies += adjacencies[l]

    sum_adjacencies /= n
#     sum_adjacencies = 1 - sum_adjacencies # xunxo to get right color pattern

    ax[i//4, i%4].set_title('{}'.format(c))
    ax[i//4, i%4].imshow(sum_adjacencies, cmap = 'Blues')
    ax[i//4, i%4].set_xticks([])
    ax[i//4, i%4].set_yticks([])
f.delaxes(ax[2,3])
plt.subplots_adjust(wspace=-.7, hspace=0.3)
plt.savefig('figs/adj_matrix.pdf', bbox_inches='tight')<jupyter_output><empty_output><jupyter_text>General overview of metrics of adjacency matrices by congress<jupyter_code>df_metrics['avg_degree_centrality'] = df_metrics['degree_centality'].apply(lambda x: np.mean(list(x.values())))
sns.set(style="whitegrid")
f, ax = plt.subplots(3,1, figsize=(8,7))
sns.boxplot(x='congress', y='L', data=df_metrics, boxprops=dict(alpha=.5), ax = ax[0], palette = sns.color_palette("Blues", 11))
sns.boxplot(x='congress', y='avg_degree_centrality', data=df_metrics, boxprops=dict(alpha=.5), ax = ax[1], palette = sns.color_palette("Reds", 11))
sns.boxplot(x='congress', y='density', data=df_metrics, ax = ax[2], boxprops=dict(alpha=.5), palette = sns.color_palette("Oranges", 11))

ax[0].set_xticks([])
ax[0].set_xlabel(None)
ax[1].set_xticks([])
ax[1].set_xlabel(None)
ax[2].set_xlabel('Senate')
ax[0].set_ylabel('Nb of edges')
ax[1].set_ylabel('Avg deg centrality')
ax[2].set_ylabel('Density')
ax[0].set_yticklabels(['', '2k', '4k', '6k', '8k', '10k'])
plt.subplots_adjust(wspace=0, hspace=0.1)
plt.savefig('figs/graph_stats.pdf', bbox_inches='tight')
plt.figure(figsize=(20, 8))
ax = sns.boxplot(x='congress', y='L', hue='quarter', data=df_metrics, boxprops=dict(alpha=.5))
ax.legend_.remove()
plt.figure(figsize=(20, 8))
ax = sns.boxplot(x='congress', y='connected_components', data=df_metrics, boxprops=dict(alpha=.5))
plt.figure(figsize=(20, 8))
ax = sns.boxplot(x='congress', y='connected_components', hue='quarter', data=df_metrics, boxprops=dict(alpha=.5))
ax.legend_.remove()
plt.figure(figsize=(20, 8))
ax = sns.boxplot(x='congress', y='density', hue='quarter', data=df_metrics, boxprops=dict(alpha=.5))
ax.legend_.remove()
plt.figure(figsize=(20, 8))
ax = sns.boxplot(x='congress', y='avg_degree_centrality', hue='quarter', data=df_metrics, boxprops=dict(alpha=.5))
ax.legend_.remove()
range_evals = 10
for i in range(0, range_evals):
    name = '{}_evalue'.format(i + 1)
    df_metrics[name] = df_metrics['eigenvalues'].apply(lambda x: x[i])
(df_metrics[['{}_evalue'.format(i + 1) for i in range(0, range_evals)]]
 .describe()
 .T[['mean', 'min', '25%', '50%', '75%', 'max']]
 .applymap(lambda x: round(x, 2)))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1611295681063123, 'fraction_numerical': 0.019601328903654486, 'mean_word_length': 5.912743972445465, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '285415', 'license': 'MIT', 'n_tokens_mistral': 2314, 'n_tokens_neox': 2178, 'n_words': 519, 'path': 'notebooks/analysis-historical-adjacency-REPORT.ipynb', 'repo_name': 'roman-bachmann/US-Senators', 'size': '6020'}