<jupyter_start><jupyter_text>GoalThis notebook is just an extension of `cnn-congress-115.ipynb` for all congresses. In this notebook, we fitted the neural network for all congresses separately and also all 20 years together (since there are 231 senators along 20 years, so we might expect get a performance improvement) Main Takeaways- As expected, congresses 110 and 111 had the lowest performance among all congresses- The best performance was the last congress with 97% of accuracy (78% is the baseline)- The model with all 20 year give us an accuracy of 89% (63% is the baseline) Further AnalysisFor each initiative, there are two sessions. I might be interesting build a model based on session 1 to predict session 2.<jupyter_code>import os
import pandas as pd
import numpy as np

import matplotlib.pyplot as plt
%matplotlib inline 

from lib import models, graph, coarsening, utils
ALL_LAWS_PATH = '../data/all-votes/laws_20years.csv'
ALL_ADJACENCY_PATH = '../data/all-votes/all_adjacency.pickle'
VOTES_POSITIONS = '../data/all-votes/all_votes_positions.csv'
ALL_SUBJECT_PATH = '../data/all-votes/bills_20years.csv'
ALL_ADJACENCY_METRIS_PATH = '../data/all-votes/all_adjacency_metric.csv'
pal = {'republican': '#ff0000', 'democrats': '#0000ff'}
all_laws = pd.read_csv(ALL_LAWS_PATH)
all_bills = pd.read_csv(ALL_SUBJECT_PATH)
all_adjacency_metrics = pd.read_csv(ALL_ADJACENCY_METRIS_PATH)
all_laws['law_id'] = all_laws['vote_uri'].apply(lambda x: '_'.join(x.split('/')[4:]).split('.')[0])
all_laws = all_laws.merge(all_bills, on='law_id', how='left')
all_laws = all_adjacency_metrics.merge(all_bills, on='law_id', how='left')

LOOKUP_RESULT = {
    'Agreed to': 'passed', 'Rejected': 'rejected', 'Nomination Confirmed': 'passed',
    'Cloture Motion Agreed to': 'passed', 'Amendment Rejected': 'rejected', 'Confirmed': 'passed',
    'Passed': 'passed', 'Amendment Agreed to': 'passed', 'Motion Rejected': 'rejected',
    'Motion Agreed to': 'passed', 'Bill Passed': 'passed', 'Motion to Table Agreed to': 'passed',
    'Cloture Motion Rejected': 'rejected', 'Motion to Proceed Agreed to': 'passed',
    'Cloture on the Motion to Proceed Agreed to': 'passed', 'Cloture on the Motion to Proceed Rejected': 'rejected'
}
all_laws['result_ok'] = all_laws.result.apply(lambda x: LOOKUP_RESULT.get(x, 'undefinied'))

votes_positions = pd.read_csv(VOTES_POSITIONS)
    
print(all_laws.shape)
print(votes_positions.shape)
all_laws = all_laws[all_laws['result_ok'].apply(lambda x: x in ['passed', 'rejected'])]
def numeric_votes(x):
    if x == 'Yes':
        return 1
    elif x == 'Not Voting' or pd.isnull(x):
        return 0.5 # we can put np.nan and then fill with the median or mean, whatever for now
    else:
        return 0

datas = {}

for c in list(all_laws.congress.unique()):
    filtered = all_laws[all_laws.congress == c]
    laws_ids = list(filtered['vote_uri'])
    filtered_votes = votes_positions[votes_positions.vote_uri.apply(lambda x: x in laws_ids)]
    members = pd.DataFrame(list(filtered_votes['member_id'].unique())).rename(columns={0: 'member_id'})
    n = len(laws_ids)
    d = len(members)
    y = filtered.result_ok.apply(lambda x: 0 if x=='rejected' else 1).values
    X = np.zeros((n, d))
    
    for i, l in enumerate(laws_ids):
        votes = (filtered_votes[filtered_votes.vote_uri == l]
                 .merge(members, how='right')
                 .sort_values(['party', 'member_id']))
        feature = votes.vote_position.apply(numeric_votes).values
        X[i] = feature
        
    # it will select by time and it is on purpose because our main is predict the future based on the past
    n_train = n // 2
    n_val = n // 5

    X_train = X[:n_train]
    X_val   = X[n_train:n_train+n_val]
    X_test  = X[n_train+n_val:]

    y_train = y[:n_train]
    y_val   = y[n_train:n_train+n_val]
    y_test  = y[n_train+n_val:]

    datas[c] = {'X_train': X_train, 'y_train': y_train,
                'X_val': X_val, 'y_val': y_val,
                'X_test': X_test, 'y_test': y_test}
    
params = dict()
params['dir_name']       = 'demo'
params['num_epochs']     = 40
params['batch_size']     = 10 # changed
params['eval_frequency'] = 100 # changed

# Building blocks.
params['filter']         = 'chebyshev5'
params['brelu']          = 'b1relu'
params['pool']           = 'apool1'

# Number of classes.
C = y.max() + 1
assert C == np.unique(y).size

# Architecture.
params['F']              = [32, 64]  # Number of graph convolutional filters.
params['K']              = [20, 20]  # Polynomial orders.
params['p']              = [4, 2]    # Pooling sizes.
params['M']              = [512, C]  # Output dimensionality of fully connected layers.

# Optimization.
params['regularization'] = 5e-4
params['dropout']        = 0.5 # changed
params['learning_rate']  = 1e-3
params['decay_rate']     = 0.95
params['momentum']       = 0.9
params['decay_steps']    = n_train / params['batch_size']

def fit(X_train, X_val, X_test, y_train, y_val, y_test):     
    min_acc = y_train.sum() / y_train.shape[0]
    
    dist, idx = graph.distance_scipy_spatial(X_train.T, k=10, metric='euclidean')
    A = graph.adjacency(dist, idx).astype(np.float32)
    graphs, perm = coarsening.coarsen(A, levels=3, self_connections=False)
    X_train = coarsening.perm_data(X_train, perm)
    X_val = coarsening.perm_data(X_val, perm)
    X_test = coarsening.perm_data(X_test, perm)
    L = [graph.laplacian(A, normalized=True) for A in graphs]
#     graph.plot_spectrum(L)
    
    model = models.cgcnn(L, **params)
    accuracy, loss, t_step = model.fit(X_train, y_train, X_val, y_val)
    res = model.evaluate(X_test, y_test)
    return {'min_acc': min_acc, 'graph': graph, 'acc': res, 'A': A}<jupyter_output><empty_output><jupyter_text>Models by Congress<jupyter_code>results = dict([(k, fit(**d)) for k, d in datas.items()])
f, ax = plt.subplots(4, 3, figsize=(20, 20))
positions = np.array(range(12)).reshape(4, 3)

for i, r in enumerate(results):
    pos = np.where(positions==i)
    A = results[r]['A']
    
    ax[pos[0][0], pos[1][0]].spy(A, markersize=2, color='black');
    ax[pos[0][0], pos[1][0]].set_title('Congress {} ~ acc: {} -> {}'.format(r, round(results[r]['min_acc'],2)*100, round(results[r]['acc'][1], 3)))
    ax[pos[0][0], pos[1][0]].axis('off')<jupyter_output><empty_output><jupyter_text>Model with all congressesThere are 231 unique members over the 20 years, it means that one senator participates in many congresses. Let's try to build a model with all senator at once<jupyter_code>laws_ids = list(all_laws['vote_uri'])
filtered_votes = votes_positions[votes_positions.vote_uri.apply(lambda x: x in laws_ids)]
members = pd.DataFrame(list(filtered_votes['member_id'].unique())).rename(columns={0: 'member_id'})
n = len(laws_ids)
d = len(members)
y = all_laws.result_ok.apply(lambda x: 0 if x=='rejected' else 1).values
X = np.zeros((n, d))
    
for i, l in enumerate(laws_ids):
    votes = (filtered_votes[filtered_votes.vote_uri == l].merge(members, how='right')
                 .sort_values(['party', 'member_id']))
    feature = votes.vote_position.apply(numeric_votes).values
    X[i] = feature
        
    # it will select by time and it is on purpose because our main is predict the future based on the past
    n_train = n // 2
    n_val = n // 5

    X_train = X[:n_train]
    X_val   = X[n_train:n_train+n_val]
    X_test  = X[n_train+n_val:]

    y_train = y[:n_train]
    y_val   = y[n_train:n_train+n_val]
    y_test  = y[n_train+n_val:]
all_result = fit(X_train, X_val, X_test, y_train, y_val, y_test)
print(all_result['acc'][1])
all_result<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14280094413847363, 'fraction_numerical': 0.01730920535011802, 'mean_word_length': 3.984967320261438, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '162517', 'license': 'MIT', 'n_tokens_mistral': 2909, 'n_tokens_neox': 2716, 'n_words': 681, 'path': 'notebooks/cnn-congress-all.ipynb', 'repo_name': 'roman-bachmann/US-Senators', 'size': '7626'}