<jupyter_start><jupyter_text>Matched Nodes StatisticsIn this notebook, we compute the percentage of nodes that we managed to map between snapshotsusing on our multi-pass node alignment heuristic, as reported in the text of the paper.  Preparations<jupyter_code>import networkx as nx
import os
import json
import pandas as pd
from quantlaw.utils.networkx import get_leaves
data = []
for dataset in ['us', 'de']:
    folder = f'../../legal-networks-data/{dataset}/4_crossreference_graph/subseqitems/'
    leaves_dict = {}
    for file in sorted(os.listdir(folder)):
        if file.endswith('.gpickle.gz'):
            G = nx.read_gpickle(folder+file)
            leaves = get_leaves(G)
            leaves_dict[file.split('.')[0]] = len(leaves)
            print(file, 'done')
    
    folder = f'../../legal-networks-data/{dataset}/5_snapshot_mapping_edgelist/subseqitems/'
    for file in sorted(os.listdir(folder)):
        if file.endswith('.json'):
            with open(folder+file) as f:
                mappings = json.load(f)
            file_base = os.path.splitext(file)[0]
            snapshot_1, snapshot_2 = file_base.split('_')
            data.append({
                'dataset': dataset,
                'year1': snapshot_1,
                'year2': snapshot_2,
                'count1': leaves_dict[snapshot_1],
                'count2': leaves_dict[snapshot_2],
                'mapped': len(mappings)
            })
df = pd.DataFrame(data).sort_values(['dataset', 'year1'])
df['Mappend Ratio'] = [t.mapped / min(t.count1, t.count2) for t in df.itertuples()]
df.to_csv('../graphics/matched_stats.csv')
df = pd.read_csv('../graphics/matched_stats.csv')
df.groupby('dataset')['Mappend Ratio'].describe()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13558352402745996, 'fraction_numerical': 0.009725400457665904, 'mean_word_length': 2.777537796976242, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '665369', 'license': 'CC-BY-4.0', 'n_tokens_mistral': 557, 'n_tokens_neox': 549, 'n_words': 120, 'path': 'notebooks/si_02_matched_nodes_statistics.ipynb', 'repo_name': 'QuantLaw/Complex-Societies-and-Growth', 'size': '1748'}