<jupyter_start><jupyter_text>Setup<jupyter_code>import sys
import os
madminer_src_path = "/home/shomiller/madminer"
sys.path.append(madminer_src_path)

from __future__ import absolute_import, division, print_function, unicode_literals
import logging

import numpy as np
import math
import matplotlib
from matplotlib import pyplot as plt
from scipy.optimize import curve_fit
% matplotlib inline
from matplotlib.patches import Patch
from matplotlib.lines import Line2D

from madminer.fisherinformation import FisherInformation
from madminer.fisherinformation import project_information, profile_information

from pandas import DataFrame

import madminer.__version__
print( 'MadMiner version: {}'.format(madminer.__version__) )
# MadMiner output
logging.basicConfig(
    format='%(asctime)-5.5s %(name)-20.20s %(levelname)-7.7s %(message)s',
    datefmt='%H:%M',
    level=logging.INFO
)

# Output of all other modules (e.g. matplotlib)
for key in logging.Logger.manager.loggerDict:
    if "madminer" not in key:
        logging.getLogger(key).setLevel(logging.WARNING)
uselumi=300.
labels = [ 'cHbox', 'cHDD', 'cHW', 'cHq3' ]
labels_rot = [ 'cHK', 'cHW', 'cHq3' ]<jupyter_output><empty_output><jupyter_text>Data Files:Saving the paths for all our lhe data files, for easy use later on.Note, lhedata are saved in the format:`data/{channel}_lhedata_{observables}.h5`while the `SALLY` models are saved in:`models/sally_ensemble_{channel}_{observables}/`here `channel` can be e.g, `wph_mu_wbkgs` or `wmh_e_wbkgs` (or take out the backgrounds for the background free runs), while `observables` can be `full`, `met`, `ptw` or `2d`  Signal + Background with Signal Systematics Only<jupyter_code>lhedatafile_wph_mu_wbkgs_met = 'data/met/wph_mu_wbkgs_sigsystonly_lhedata_met.h5'
lhedatafile_wph_e_wbkgs_met = 'data/met/wph_e_wbkgs_sigsystonly_lhedata_met.h5'
lhedatafile_wmh_mu_wbkgs_met = 'data/met/wmh_mu_wbkgs_sigsystonly_lhedata_met.h5'
lhedatafile_wmh_e_wbkgs_met = 'data/met/wmh_e_wbkgs_sigsystonly_lhedata_met.h5'<jupyter_output><empty_output><jupyter_text>Setup FisherInfo Objects<jupyter_code>fisher_wph_mu_wbkgs_met = FisherInformation(lhedatafile_wph_mu_wbkgs_met, include_nuisance_parameters=True)
fisher_wph_e_wbkgs_met = FisherInformation(lhedatafile_wph_e_wbkgs_met, include_nuisance_parameters=True)
fisher_wmh_mu_wbkgs_met = FisherInformation(lhedatafile_wmh_mu_wbkgs_met, include_nuisance_parameters=True)
fisher_wmh_e_wbkgs_met = FisherInformation(lhedatafile_wmh_e_wbkgs_met, include_nuisance_parameters=True)<jupyter_output>13:30 madminer.analysis    INFO    Loading data from data/met/wmh_e_wbkgs_sigsystonly_lhedata_met.h5
13:30 madminer.analysis    INFO    Found 4 parameters
13:30 madminer.analysis    INFO    Found 33 nuisance parameters
13:30 madminer.analysis    INFO    Found 54 benchmarks, of which 15 physical
13:30 madminer.analysis    INFO    Found 48 observables
13:30 madminer.analysis    INFO    Found 5000000 events
13:30 madminer.analysis    INFO      1000000 signal events sampled from benchmark sm
13:30 madminer.analysis    INFO      4000000 background events
13:30 madminer.analysis    INFO    Found morphing setup with 15 components
13:30 madminer.analysis    INFO    Found nuisance morphing setup<jupyter_text>Information in Histograms  1D Histograms  Simplified Template Cross SectionsThe Info in the STXS defined in https://cds.cern.ch/record/2669925 (with and without the extra bin)  3 BinsThe original STXS defined in The LHC Higgs Cross Section Working Group Report, Vol. 4 (https://arxiv.org/abs/1610.07922)<jupyter_code>fi_stxs_3bins_wph_mu_mean, fi_stxs_3bins_wph_mu_covariance = fisher_wph_mu_wbkgs_met.calculate_fisher_information_hist1d(
    theta=[0.,0.,0.,0.],
    luminosity=uselumi*1000.,
    observable='pt_w',
    bins=[0.,150.,250.],
    histrange=[0.,250.]
)

print( DataFrame(
    data = project_information(fi_stxs_3bins_wph_mu_mean, [0,1,2,3]),
    columns = labels,
    index = labels)
)
fi_stxs_3bins_wph_e_mean, fi_stxs_3bins_wph_e_covariance = fisher_wph_e_wbkgs_met.calculate_fisher_information_hist1d(
    theta=[0.,0.,0.,0.],
    luminosity=uselumi*1000.,
    observable='pt_w',
    bins=[0.,150.,250.],
    histrange=[0.,250.]
)
fi_stxs_3bins_wmh_mu_mean, fi_stxs_3bins_wmh_mu_covariance = fisher_wmh_mu_wbkgs_met.calculate_fisher_information_hist1d(
    theta=[0.,0.,0.,0.],
    luminosity=uselumi*1000.,
    observable='pt_w',
    bins=[0.,150.,250.],
    histrange=[0.,250.]
)
fi_stxs_3bins_wmh_e_mean, fi_stxs_3bins_wmh_e_covariance = fisher_wmh_e_wbkgs_met.calculate_fisher_information_hist1d(
    theta=[0.,0.,0.,0.],
    luminosity=uselumi*1000.,
    observable='pt_w',
    bins=[0.,150.,250.],
    histrange=[0.,250.]
)
fi_stxs_3bins_mean = fi_stxs_3bins_wph_mu_mean + fi_stxs_3bins_wph_e_mean + fi_stxs_3bins_wmh_mu_mean + fi_stxs_3bins_wmh_e_mean
fi_stxs_3bins_covariance = fi_stxs_3bins_wph_mu_covariance + fi_stxs_3bins_wph_e_covariance + fi_stxs_3bins_wmh_mu_covariance + fi_stxs_3bins_wmh_e_covariance<jupyter_output><empty_output><jupyter_text>4 Bins (Including the Extra Division at $75\,\mathrm{GeV}$ as in https://cds.cern.ch/record/2669925)(We're not computing this for now, just setting the values to zero as placeholders)<jupyter_code>fi_stxs_4bins_wph_mu_mean, fi_stxs_4bins_wph_mu_covariance = 0., 0.
fi_stxs_4bins_wph_e_mean, fi_stxs_4bins_wph_e_covariance = 0., 0.
fi_stxs_4bins_wmh_mu_mean, fi_stxs_4bins_wmh_mu_covariance = 0., 0.
fi_stxs_4bins_wmh_e_mean, fi_stxs_4bins_wmh_e_covariance = 0., 0.<jupyter_output><empty_output><jupyter_text>5 BinsIncluding all the bins in the Stage 1.1 STXS defined in https://cds.cern.ch/record/2669925Beyond the bins at 0, 150, and 250 GeV, this includes additional bin divisions at $75$ and $400\,\mathrm{GeV}$.<jupyter_code>fi_stxs_5bins_wph_mu_mean, fi_stxs_5bins_wph_mu_covariance = fisher_wph_mu_wbkgs_met.calculate_fisher_information_hist1d(
    theta=[0.,0.,0.,0.],
    luminosity=uselumi*1000.,
    observable='pt_w',
    bins=[0.,75.,150.,250.,400.],
    histrange=[0.,400.]
)

print( DataFrame(
    data = project_information(fi_stxs_5bins_wph_mu_mean, [0,1,2,3]),
    columns = labels,
    index = labels)
)
fi_stxs_5bins_wph_e_mean, fi_stxs_5bins_wph_e_covariance = fisher_wph_e_wbkgs_met.calculate_fisher_information_hist1d(
    theta=[0.,0.,0.,0.],
    luminosity=uselumi*1000.,
    observable='pt_w',
    bins=[0.,75.,150.,250.,400.],
    histrange=[0.,400.]
)
fi_stxs_5bins_wmh_mu_mean, fi_stxs_5bins_wmh_mu_covariance = fisher_wmh_mu_wbkgs_met.calculate_fisher_information_hist1d(
    theta=[0.,0.,0.,0.],
    luminosity=uselumi*1000.,
    observable='pt_w',
    bins=[0.,75.,150.,250.,400.],
    histrange=[0.,400.]
)
fi_stxs_5bins_wmh_e_mean, fi_stxs_5bins_wmh_e_covariance = fisher_wmh_e_wbkgs_met.calculate_fisher_information_hist1d(
    theta=[0.,0.,0.,0.],
    luminosity=uselumi*1000.,
    observable='pt_w',
    bins=[0.,75.,150.,250.,400.],
    histrange=[0.,400.]
)
fi_stxs_5bins_mean = fi_stxs_5bins_wph_mu_mean + fi_stxs_5bins_wph_e_mean + fi_stxs_5bins_wmh_mu_mean + fi_stxs_5bins_wmh_e_mean
fi_stxs_5bins_covariance = fi_stxs_5bins_wph_mu_covariance + fi_stxs_5bins_wph_e_covariance + fi_stxs_5bins_wmh_mu_covariance + fi_stxs_5bins_wmh_e_covariance<jupyter_output><empty_output><jupyter_text>6 BinsAs a step towards "Improved STXS", we can also include an additional bin at $600\,\mathrm{GeV}$<jupyter_code>fi_stxs_6bins_wph_mu_mean, fi_stxs_6bins_wph_mu_covariance = fisher_wph_mu_wbkgs_met.calculate_fisher_information_hist1d(
    theta=[0.,0.,0.,0.],
    luminosity=uselumi*1000.,
    observable='pt_w',
    bins=[0.,75.,150.,250.,400.,600.],
    histrange=[0.,600.]
)

print( DataFrame(
    data = project_information(fi_stxs_6bins_wph_mu_mean, [0,1,2,3]),
    columns = labels,
    index = labels)
)
fi_stxs_6bins_wph_e_mean, fi_stxs_6bins_wph_e_covariance = fisher_wph_e_wbkgs_met.calculate_fisher_information_hist1d(
    theta=[0.,0.,0.,0.],
    luminosity=uselumi*1000.,
    observable='pt_w',
    bins=[0.,75.,150.,250.,400.,600.],
    histrange=[0.,600.]
)
fi_stxs_6bins_wmh_mu_mean, fi_stxs_6bins_wmh_mu_covariance = fisher_wmh_mu_wbkgs_met.calculate_fisher_information_hist1d(
    theta=[0.,0.,0.,0.],
    luminosity=uselumi*1000.,
    observable='pt_w',
    bins=[0.,75.,150.,250.,400.,600.],
    histrange=[0.,600.]
)
fi_stxs_6bins_wmh_e_mean, fi_stxs_6bins_wmh_e_covariance = fisher_wmh_e_wbkgs_met.calculate_fisher_information_hist1d(
    theta=[0.,0.,0.,0.],
    luminosity=uselumi*1000.,
    observable='pt_w',
    bins=[0.,75.,150.,250.,400.,600.],
    histrange=[0.,600.]
)
fi_stxs_6bins_mean = fi_stxs_6bins_wph_mu_mean + fi_stxs_6bins_wph_e_mean + fi_stxs_6bins_wmh_mu_mean + fi_stxs_6bins_wmh_e_mean
fi_stxs_6bins_covariance = fi_stxs_6bins_wph_mu_covariance + fi_stxs_6bins_wph_e_covariance + fi_stxs_6bins_wmh_mu_covariance + fi_stxs_6bins_wmh_e_covariance<jupyter_output><empty_output><jupyter_text>Save the Arrays for Later<jupyter_code>fi_hist_stxs_outfile = './fisher_info/fi_hist_stxs.npz'
fi_hist_stxs_cov_outfile = './fisher_info/fi_hist_stxs_cov.npz'

fi_hist_stxs_list = [
    fi_stxs_3bins_wph_mu_mean,
    fi_stxs_3bins_wph_e_mean,
    fi_stxs_3bins_wmh_mu_mean,
    fi_stxs_3bins_wmh_e_mean,
    fi_stxs_4bins_wph_mu_mean,
    fi_stxs_4bins_wph_e_mean,
    fi_stxs_4bins_wmh_mu_mean,
    fi_stxs_4bins_wmh_e_mean,
    fi_stxs_5bins_wph_mu_mean,
    fi_stxs_5bins_wph_e_mean,
    fi_stxs_5bins_wmh_mu_mean,
    fi_stxs_5bins_wmh_e_mean,
    fi_stxs_6bins_wph_mu_mean,
    fi_stxs_6bins_wph_e_mean,
    fi_stxs_6bins_wmh_mu_mean,
    fi_stxs_6bins_wmh_e_mean,
]

fi_hist_stxs_cov_list = [
    fi_stxs_3bins_wph_mu_covariance,
    fi_stxs_3bins_wph_e_covariance,
    fi_stxs_3bins_wmh_mu_covariance,
    fi_stxs_3bins_wmh_e_covariance,
    fi_stxs_4bins_wph_mu_covariance,
    fi_stxs_4bins_wph_e_covariance,
    fi_stxs_4bins_wmh_mu_covariance,
    fi_stxs_4bins_wmh_e_covariance,
    fi_stxs_5bins_wph_mu_covariance,
    fi_stxs_5bins_wph_e_covariance,
    fi_stxs_5bins_wmh_mu_covariance,
    fi_stxs_5bins_wmh_e_covariance,
    fi_stxs_6bins_wph_mu_covariance,
    fi_stxs_6bins_wph_e_covariance,
    fi_stxs_6bins_wmh_mu_covariance,
    fi_stxs_6bins_wmh_e_covariance,
]
np.savez(fi_hist_stxs_outfile, fi_hist_stxs_list, allow_pickle=False)
np.savez(fi_hist_stxs_cov_outfile, fi_hist_stxs_cov_list, allow_pickle=False)<jupyter_output><empty_output><jupyter_text>2D Histograms  Compute FI in 2D Histograms  With 3 bins in $m_{T,\mathrm{tot}}$<jupyter_code>fi_imp_stxs_wph_mu_mean, fi_imp_stxs_wph_mu_covariance = fisher_wph_mu_wbkgs_met.calculate_fisher_information_hist2d(
    theta=[0.,0.,0.,0.],
    luminosity=uselumi*1000.,
    observable1='pt_w',
    bins1=[0.,75.,150.,250.,400.,600.],
    histrange1=[0.,600.],
    observable2='mt_tot',
    bins2=[0.,400.,800.],
    histrange2=[0.,800.],
)
fi_imp_stxs_wph_e_mean, fi_imp_stxs_wph_e_covariance = fisher_wph_e_wbkgs_met.calculate_fisher_information_hist2d(
    theta=[0.,0.,0.,0.],
    luminosity=uselumi*1000.,
    observable1='pt_w',
    bins1=[0.,75.,150.,250.,400.,600.],
    histrange1=[0.,600.],
    observable2='mt_tot',
    bins2=[0.,400.,800.],
    histrange2=[0.,800.],
)
fi_imp_stxs_wmh_mu_mean, fi_imp_stxs_wmh_mu_covariance = fisher_wmh_mu_wbkgs_met.calculate_fisher_information_hist2d(
    theta=[0.,0.,0.,0.],
    luminosity=uselumi*1000.,
    observable1='pt_w',
    bins1=[0.,75.,150.,250.,400.,600.],
    histrange1=[0.,600.],
    observable2='mt_tot',
    bins2=[0.,400.,800.],
    histrange2=[0.,800.],
)
fi_imp_stxs_wmh_e_mean, fi_imp_stxs_wmh_e_covariance = fisher_wmh_e_wbkgs_met.calculate_fisher_information_hist2d(
    theta=[0.,0.,0.,0.],
    luminosity=uselumi*1000.,
    observable1='pt_w',
    bins1=[0.,75.,150.,250.,400.,600.],
    histrange1=[0.,600.],
    observable2='mt_tot',
    bins2=[0.,400.,800.],
    histrange2=[0.,800.],
)
fi_imp_stxs_mean = fi_imp_stxs_wph_mu_mean + fi_imp_stxs_wph_e_mean + fi_imp_stxs_wmh_mu_mean + fi_imp_stxs_wmh_e_mean
fi_imp_stxs_covariance = fi_imp_stxs_wph_mu_covariance + fi_imp_stxs_wph_e_covariance + fi_imp_stxs_wmh_mu_covariance + fi_imp_stxs_wmh_e_covariance<jupyter_output><empty_output><jupyter_text>Save the Arrays for Later<jupyter_code>fi_hist_2d_outfile = './fisher_info/fi_hist_2d.npz'
fi_hist_2d_cov_outfile = './fisher_info/fi_hist_2d_cov.npz'

fi_hist_2d_list = [
    fi_imp_stxs_wph_mu_mean, 
    fi_imp_stxs_wph_e_mean, 
    fi_imp_stxs_wmh_mu_mean, 
    fi_imp_stxs_wmh_e_mean, 
]

fi_hist_2d_cov_list = [
    fi_imp_stxs_wph_mu_covariance,
    fi_imp_stxs_wph_e_covariance,
    fi_imp_stxs_wmh_mu_covariance,
    fi_imp_stxs_wmh_e_covariance,
]
np.savez(fi_hist_2d_outfile, fi_hist_2d_list, allow_pickle=False)
np.savez(fi_hist_2d_cov_outfile, fi_hist_2d_cov_list, allow_pickle=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1071820605300298, 'fraction_numerical': 0.05880508075897758, 'mean_word_length': 6.09794101279911, 'pattern_counts': {'":': 0, '<': 40, '<?xml version=': 0, '>': 40, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '302111', 'license': 'MIT', 'n_tokens_mistral': 6033, 'n_tokens_neox': 5487, 'n_words': 655, 'path': '5b-WH-Compute-FisherInfo-Histograms.ipynb', 'repo_name': 'diemort/wh-madminer', 'size': '12754'}