<commit_before>'''Analysis helper routines for HANDE data.'''

from os import path
import pandas as pd
import sys

sys.path.append(path.join(path.abspath(path.dirname(__file__)), '../pyblock'))
import pyblock.error
import pyblock.pd_utils

def projected_energy(reblock_data, covariance, data_length,
                     sum_key='\<KEY>', ref_key='N_0'):
    '''Calculate the projected energy estimator and associated error.

The projected energy estimator is given by

.. math::

    E = \\frac{\\sum_H_0j N_j}{N_0}

The numerator and denominator are correlated and so their covariance must be
taken into account.

Parameters
----------
reblock_data : :class:`pandas.DataFrame`
    reblock data for (at least) the numerator and denominator in the
    projected energy estimator.
covariance: :class:`pandas.DataFrame`
    covariance at each reblock iteration between (at least) the numerator
    and denominator in the projected energy estimator.
data_length: :class:`pandas.DataFrame`
    number of data points in each reblock iteration.
sum_key : string
    column name in reblock_data containing :math:`\\sum_H_0j N_j``, i.e. the sum
    of the population weighted by the Hamiltonian matrix element with the trial
    wavefunction.
ref_key : string
    column name in reblock_data containing :math:`N_0``, i.e. the population of
    the trial wavefunction (often/originally just a single determinant).

Returns
-------
proje : :class:`pandas.DataFrame`
    The projected energy estimator at each reblock iteration.

See also
--------
:func:`pyblock.pd_utils.reblock` for producing the input parameters.
'''

    proje_sum = reblock_data.ix[:, sum_key]
    ref_pop = reblock_data.ix[:, ref_key]
    proje_ref_cov = covariance.xs(ref_key, level=1)[sum_key]
    proje = pyblock.error.ratio(proje_sum, ref_pop, proje_ref_cov, data_length)
    proje.columns = pd.MultiIndex.from_tuples(
            [('Proj. Energy', col) for col in proje.columns])
    return proje

def qmc_summary(data, keys=('Shift', '\sum H_0j N_j', 'N_0',
                            'Proj. Energy'), summary_tuple=None):
    '''Summarise a reblocked data set by the optimal block.

Parameters
----------
data : :class:`pandas.DataFrame`
    reblocked data (i.e. data with the reblock iteration as the index).
keys : list of strings
    columns (by top-level index) of the data table to inspect.  Each top-level
    column must contain an optimal block column.
summary_tuple :  (:class:`pandas.DataFrame`, list of strings)
    Optionally append summary data to this tuple. Allows repeated calling of 
    this function.

Returns
-------
opt_data : :class:`pandas.DataFrame`
    Data for each column from the optimal block size of that column.
no_opt : list of strings
    list of columns for which no optimal block size was found.
'''

    opt_data = []
    no_opt = []
    if summary_tuple != None:
        (old_opt_data, no_opt) = summary_tuple
    for col in keys:
        if col in data:
            summary = pyblock.pd_utils.reblock_summary(data.ix[:, col])
            if summary.empty:
                no_opt.append(col)
            else:
                summary.index = [col]
            opt_data.append(summary)
    opt_data = pd.concat(opt_data)
    if summary_tuple != None:
        opt_data = opt_data.append(old_opt_data)
    return (opt_data, no_opt)<commit_msg>Clean up of code for extending a previous QMC summary.<commit_after>'''Analysis helper routines for HANDE data.'''

from os import path
import pandas as pd
import sys

sys.path.append(path.join(path.abspath(path.dirname(__file__)), '../pyblock'))
import pyblock.error
import pyblock.pd_utils

def projected_energy(reblock_data, covariance, data_length,
                     sum_key='\sum H_0j N_j', ref_key='N_0'):
    '''Calculate the projected energy estimator and associated error.

The projected energy estimator is given by

.. math::

    E = \\frac{\\sum_H_0j N_j}{N_0}

The numerator and denominator are correlated and so their covariance must be
taken into account.

Parameters
----------
reblock_data : :class:`pandas.DataFrame`
    reblock data for (at least) the numerator and denominator in the
    projected energy estimator.
covariance: :class:`pandas.DataFrame`
    covariance at each reblock iteration between (at least) the numerator
    and denominator in the projected energy estimator.
data_length: :class:`pandas.DataFrame`
    number of data points in each reblock iteration.
sum_key : string
    column name in reblock_data containing :math:`\\sum_H_0j N_j``, i.e. the sum
    of the population weighted by the Hamiltonian matrix element with the trial
    wavefunction.
ref_key : string
    column name in reblock_data containing :math:`N_0``, i.e. the population of
    the trial wavefunction (often/originally just a single determinant).

Returns
-------
proje : :class:`pandas.DataFrame`
    The projected energy estimator at each reblock iteration.

See also
--------
:func:`pyblock.pd_utils.reblock` for producing the input parameters.
'''

    proje_sum = reblock_data.ix[:, sum_key]
    ref_pop = reblock_data.ix[:, ref_key]
    proje_ref_cov = covariance.xs(ref_key, level=1)[sum_key]
    proje = pyblock.error.ratio(proje_sum, ref_pop, proje_ref_cov, data_length)
    proje.columns = pd.MultiIndex.from_tuples(
            [('Proj. Energy', col) for col in proje.columns])
    return proje

def qmc_summary(data, keys=('Shift', '\sum H_0j N_j', 'N_0',
                            'Proj. Energy'), summary_tuple=None):
    '''Summarise a reblocked data set by the optimal block.

Parameters
----------
data : :class:`pandas.DataFrame`
    reblocked data (i.e. data with the reblock iteration as the index).
keys : list of strings
    columns (by top-level index) of the data table to inspect.  Each top-level
    column must contain an optimal block column.
summary_tuple :  (:class:`pandas.DataFrame`, list of strings)
    Optionally append summary data to this tuple. Allows repeated calling of 
    this function.

Returns
-------
opt_data : :class:`pandas.DataFrame`
    Data for each column from the optimal block size of that column.
no_opt : list of strings
    list of columns for which no optimal block size was found.
'''

    if summary_tuple:
        (opt_data, no_opt) = ([summary_tuple[0]], summary_tuple[1])
    else:
        (opt_data, no_opt) = ([], [])
    for col in keys:
        if col in data:
            summary = pyblock.pd_utils.reblock_summary(data.ix[:, col])
            if summary.empty:
                no_opt.append(col)
            else:
                summary.index = [col]
            opt_data.append(summary)
    opt_data = pd.concat(opt_data)
    return (opt_data, no_opt)
{'fraction_non_alphanumeric': 0.10119581464872944, 'fraction_numerical': 0.0028400597907324364, 'mean_word_length': 3.1559006211180125, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6783421', 'n_tokens_mistral': 2057, 'n_tokens_neox': 1998, 'n_words': 680}