<jupyter_start><jupyter_text>Summary of Bayesian parameters searchRead all the parameters' search results and save the optimal ones.Read all the sensitivity test results and save for the further visualisation.<jupyter_code>%load_ext autoreload
%autoreload 2
import json
import pandas as pd
import validation
from pprint import pprint<jupyter_output><empty_output><jupyter_text>1 Read the logs of validation and save to a file<jupyter_code>region_list = ['sweden', 'netherlands', 'saopaulo']
for region in region_list:
    file = f'../../results/para-search-r1/parasearch-n_{region}.txt'
    list_lines = []
    with open(file) as f:
        for jsonObj in f:
            line = json.loads(jsonObj)
            list_lines.append(line)
    df = pd.DataFrame(list_lines)
    df = df.loc[df['kl'] > 0, :]
    with open('../../results/para-search-r1/parasearch.txt', 'a') as outfile:
        json.dump(df.loc[df['kl'] == min(df['kl']), :].to_dict('records')[0], outfile)
        outfile.write('\n')<jupyter_output><empty_output><jupyter_text>2 Run src\py\parameters-validation.pyThis step generates and saves the visits using the optimal parameters on both calibration and validation datasets.  3 Compute KL divergence measure of the validation results<jupyter_code>with open('../../results/para-search-r1/parasearch.txt', 'r') as f:
    for jsonObj in f:
        line = json.loads(jsonObj)
        region = line['region']
        # Add validation performance
        file = f'../../results/para-search-r1/{region}_validation_distances.csv'
        distances = pd.read_csv(file)
        kl = validation.DistanceMetrics().kullback_leibler(distances,
                                                           titles=['groundtruth', 'model'])
        line['kl-v'] = kl

        # Add validation performance for benchmark
        kl_baseline = validation.DistanceMetrics().kullback_leibler(distances,
                                                           titles=['groundtruth', 'benchmark'])
        line['kl-v-baseline'] = kl_baseline
        with open('../../results/summary.txt', 'a') as outfile:
            json.dump(line, outfile)
            outfile.write('\n')
with open('../../results/summary.txt', 'r') as outfile:
    for line in outfile:
        pprint(line)<jupyter_output>('{"region": "sweden", "p": 0.9837273303214517, "beta": 0.017503751178316866, '
 '"gamma": 0.11895253662991521, "kl-baseline": 1.2950978492258307, "kl": '
 '0.01585126958256988, "kl-v": 0.02354793316813772}\n')
('{"region": "netherlands", "p": 0.7763441613832662, "beta": '
 '0.22682873676239296, "gamma": 0.012171103323168245, "kl-baseline": '
 '1.2757839552588923, "kl": 0.017479270659420572, "kl-v": '
 '0.019721853782483253}\n')
('{"region": "saopaulo", "p": 0.45879896443898016, "beta": '
 '0.17666806537151383, "gamma": 0.011859780066996593, "kl-baseline": '
 '0.8589528848493738, "kl": 0.0027090426934729584, "kl-v": '
 '0.00308760109916755}\n')
('{"region": "sweden", "p": 0.9810923928598705, "beta": 0.014032698838826095, '
 '"gamma": 0.23585684808119195, "kl-baseline": 0.09081822690812881, '
 '"kl-deviation": 0.008582467654668042, "kl": 0.006669178646773053, "kl-v": '
 '0.010764955875287098, "kl-v-baseline": 0.07077894402988541}\n')
('{"region": "netherlands", "p": 0.7977081692955302, [...]<jupyter_text>4 Compute KL divergence measure of the sensitivity test results<jupyter_code>file = '../../results/para-search-r1/parasearch.txt'
list_lines = []
with open(file) as f:
    for jsonObj in f:
        line = json.loads(jsonObj)
        list_lines.append(line)
df_av = pd.DataFrame(list_lines)
df_av.set_index('region', inplace=True)
df_av.loc['average'] = df_av.mean()  # Get the average value of parameters
df_av.reset_index(inplace=True)
df_av<jupyter_output><empty_output><jupyter_text>4.1 R1 version<jupyter_code>res_list = []
for region in region_list:
    for region2cross in [x for x in region_list if x != region] + ['average']:
        for type in ['calibration', 'validation']:
            file = f'../../results/para-search-r1/sensitivity/{region}_{type}_{region2cross}_distances.csv'
            distances = pd.read_csv(file)
            kl = validation.DistanceMetrics().kullback_leibler(distances, titles=['groundtruth', 'model'])
            kl_b = validation.DistanceMetrics().kullback_leibler(distances, titles=['groundtruth', 'benchmark'])
            gain = (kl_b - kl) / kl_b * 100 # Improvement (decrease) in %
            res_list.append((region, region2cross, type, kl, kl_b, gain))
df_sens = pd.DataFrame(res_list, columns=['region', 'region2cross', 'type', 'kl', 'kl_b', 'gain'])
# Add its own performance
res_self_list = []
with open('../../results/summary.txt', 'r') as outfile:
    for jsonObj in outfile:
        line = json.loads(jsonObj)
        gain_c = (line['kl-baseline'] - line['kl']) / line['kl-baseline'] * 100
        res_self_list.append((line['region'], line['region'], 'calibration', line['kl'], line['kl-baseline'], gain_c))

        gain_v = (line['kl-v-baseline'] - line['kl-v']) / line['kl-v-baseline'] * 100
        res_self_list.append((line['region'], line['region'], 'validation', line['kl-v'], line['kl-v-baseline'], gain_v))
df_sens = pd.concat([pd.DataFrame(res_self_list, columns=['region', 'region2cross', 'type', 'kl', 'kl_b', 'gain']),
                     df_sens])
df_sens<jupyter_output><empty_output><jupyter_text>4.2 R2 version<jupyter_code>df_sens_diagonal = df_sens.loc[df_sens['region'] == df_sens['region2cross'], ['region', 'type', 'gain']]
def relative_kl_change(row):
    kl_self = df_sens_diagonal.loc[(df_sens_diagonal['region'] == row['region']) & \
                                   (df_sens_diagonal['type'] == row['type']), 'gain'].values[0]
    change = row['gain'] / kl_self * 100 # Performance (increase) in %
    return change
df_sens.loc[:, 'gain_ratio'] = df_sens.apply(lambda row: relative_kl_change(row), axis=1)
df_sens
df_sens_diagonal = df_sens.loc[df_sens['region'] == df_sens['region2cross'], ['region', 'type', 'kl']]
def relative_kl_change(row):
    kl_self = df_sens_diagonal.loc[(df_sens_diagonal['region'] == row['region']) & \
                                   (df_sens_diagonal['type'] == row['type']), 'kl'].values[0]
    change = (row['kl'] - kl_self) / kl_self * 100 # Performance (loss) in %
    return change
df_sens.loc[:, 'kl_ratio'] = df_sens.apply(lambda row: relative_kl_change(row), axis=1)
df_sens
df_sens.to_csv('../../results/para-search-r1/sensitivity_summary.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16582297898450682, 'fraction_numerical': 0.08222119957048626, 'mean_word_length': 3.808259587020649, 'pattern_counts': {'":': 32, '<': 24, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '630155', 'license': 'MIT', 'n_tokens_mistral': 2594, 'n_tokens_neox': 2229, 'n_words': 437, 'path': 'src/py/2-parasearch-summary.ipynb', 'repo_name': 'TheYuanLiao/individual_mobility_model', 'size': '6519'}