<jupyter_start><jupyter_text>Data exploration and visusalisation with pandasThis is an example notebook to show how to use pandas for quick data exploration and results visualition based on a results hdf5 file.  Imports and data download for cta-lstchain-extra<jupyter_code>import os
if not 'gamma_events_point_tiny.h5' in os.listdir('.'):
    !wget https://github.com/cta-observatory/cta-lstchain-extra/raw/master/reco/sample_data/dl1/gamma_events_point_tiny.h5
if not 'proton_events_tiny.h5' in os.listdir('.'):
    !wget https://github.com/cta-observatory/cta-lstchain-extra/raw/master/reco/sample_data/dl1/proton_events_tiny.h5
        
!ls gamma_events_point_tiny.h5
!ls proton_events_tiny.h5
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np

%matplotlib inline

from IPython.core.display import display, HTML
display(HTML("<style>.container { width:98% !important; }</style>"))
gammas_table = pd.read_hdf('gamma_events_point_tiny.h5')
protons_table = pd.read_hdf('proton_events_tiny.h5')
gammas_table.describe()
protons_table.describe()
hist_opt = {'figsize':(18,18), 'density':True, 'alpha':0.6, 'bins':80}
axes = gammas_table.hist(label='diffuse gammas', **hist_opt);
axes = protons_table.hist(ax=axes.ravel()[:len(gammas_table.columns)], label='protons', **hist_opt);<jupyter_output><empty_output><jupyter_text>Correlation matrices<jupyter_code>import seaborn as sns
# Gammas
f, ax = plt.subplots(figsize=(10, 8))
features = gammas_table.columns
corr = gammas_table[features].corr()
sns.heatmap(corr, 
            mask=np.zeros_like(corr, dtype=np.bool),
            cmap=sns.diverging_palette(220, 10, as_cmap=True),
            square=True, 
            ax=ax);
# Protons
f, ax = plt.subplots(figsize=(10, 8))
features = protons_table.columns
corr = protons_table[features].corr()
sns.heatmap(corr, 
            mask=np.zeros_like(corr, dtype=np.bool),
            cmap=sns.diverging_palette(220, 10, as_cmap=True),
            square=True, 
            ax=ax);<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12895256916996048, 'fraction_numerical': 0.018280632411067192, 'mean_word_length': 4.9734513274336285, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '401413', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 771, 'n_tokens_neox': 730, 'n_words': 134, 'path': 'notebooks/data_vis.ipynb', 'repo_name': 'pawel21/cta-lstchain', 'size': '2024'}