<commit_before>"""
.. module:: analyze
   :synopsis: Extract data from chains and produce plots

.. moduleauthor:: <NAME> <<EMAIL>>
.. moduleauthor:: <NAME> <<EMAIL>>

Collection of functions needed to analyze the Markov chains.

This module defines as well a class :class:`Information`, that stores useful
quantities, and shortens the argument passing between the functions.

.. note::
    Some of the methods used in this module are directly adapted from the
    `CosmoPmc <http://www.cosmopmc.info>`_ code from Kilbinger et. al.

"""
import os
import math
import numpy as np
# The root plotting module, to change options like font sizes, etc...
import matplotlib
# The following line suppresses the need for an X server
matplotlib.use("Agg")
# Module for handling display
import matplotlib.pyplot as plt
# Module to handle warnings from matplotlib
import warnings
import io_mp

from time import time

def analyze(command_line):
    """
    Main function, does the entire analysis.

    It calls in turn all the other routines from this module. To limit the
    arguments of each function to a reasonnable size, a :class:`Information`
    instance is used. This instance is initialized in this function, then
    appended by the other routines.

    """
    # Create an instance of the Information class, that will hold all relevant
    # information, and be used as a compact way of exchanging information
    # between functions
    info = Information()

    # Check if the scipy module has the interpolate method correctly
    # installed (should be the case on every linux distribution with
    # standard numpy)
    try:
        from scipy.interpolate import interp1d
        info.has_interpolate_module = True
    except ImportError:
        info.has_interpolate_module = False
        warnings.warn(
            'No cubic interpolation done (no interpolate method found ' +
            'in scipy), only linear')

    # At this points, `files` could contain either a list of files (that
    # could be only one) or a folder. This is the reason why it is not yet
    # copied to the info class.
    files = command_line.files
    binnumber = command_line.bins

    # Save the extension to output files
    info.extension = command_line.extension

    # Read a potential file describing changes to be done for the parameter
    # names, and number of paramaters plotted (can be let empty, all will
    # then be plotted).
    if command_line.optional_plot_file is not None:
        for line in open(command_line.optional_plot_file[0], 'r'):
            exec(line)

    # Prepare the files, according to the case, load the log.param, and
    # prepare the output (plots folder, .covmat, .info and .log files).
    # After this step, info.files will contain all chains.
    status = prepare(info, files)

    if not status:
        return

    # Compute the mean, maximum of likelihood, 1-sigma variance for this
    # main folder. This will create the info.spam chain
    convergence(info)

    # Create the main chain, which consists in all elements of info.spam
    # put together. This will serve for the plotting.
    chain = np.copy(info.spam[0])
    for i in range(len(info.spam)-1):
        chain = np.append(chain, info.spam[i+1], axis=0)

    # In case of comparison, launch the prepare and convergence methods,
    # with an additional flag: is_main_chain=False. This will ensure that
    # all the information will not be stored to info.files, info.covmat...
    # but to the specified output, respectively comp_files, comp_spam...
    if command_line.comp is not None:
        comp_files, comp_folder, comp_param = \
            prepare(info, command_line.comp, is_main_chain=False)
        comp_spam, comp_ref_names, comp_tex_names, comp_backup_names, \
            comp_plotted_parameters, comp_boundaries, comp_mean = \
            convergence(info, is_main_chain=False, files=comp_files,
                        param=comp_param)
        comp_mean = comp_mean[0]

        # Create comp_chain
        comp_chain = np.copy(comp_spam[0])
        for i in range(len(comp_spam)-1):
            comp_chain = np.append(comp_chain, comp_spam[i+1], axis=0)

    # Total number of steps.
    weight = np.sum(chain[:, 0])

    # Covariance matrix computation (for the whole chain)<commit_msg>Further analyze optimisations<commit_after>"""
.. module:: analyze
   :synopsis: Extract data from chains and produce plots

.. moduleauthor:: <NAME> <<EMAIL>>
.. moduleauthor:: <NAME> <<EMAIL>>

Collection of functions needed to analyze the Markov chains.

This module defines as well a class :class:`Information`, that stores useful
quantities, and shortens the argument passing between the functions.

.. note::
    Some of the methods used in this module are directly adapted from the
    `CosmoPmc <http://www.cosmopmc.info>`_ code from Kilbinger et. al.

"""
import os
import math
import numpy as np
# The root plotting module, to change options like font sizes, etc...
import matplotlib
# The following line suppresses the need for an X server
matplotlib.use("Agg")
# Module for handling display
import matplotlib.pyplot as plt
# Module to handle warnings from matplotlib
import warnings
import io_mp


def analyze(command_line):
    """
    Main function, does the entire analysis.

    It calls in turn all the other routines from this module. To limit the
    arguments of each function to a reasonnable size, a :class:`Information`
    instance is used. This instance is initialized in this function, then
    appended by the other routines.

    """
    # Create an instance of the Information class, that will hold all relevant
    # information, and be used as a compact way of exchanging information
    # between functions
    info = Information()

    # Check if the scipy module has the interpolate method correctly
    # installed (should be the case on every linux distribution with
    # standard numpy)
    try:
        from scipy.interpolate import interp1d
        info.has_interpolate_module = True
    except ImportError:
        info.has_interpolate_module = False
        warnings.warn(
            'No cubic interpolation done (no interpolate method found ' +
            'in scipy), only linear')

    # At this points, `files` could contain either a list of files (that
    # could be only one) or a folder. This is the reason why it is not yet
    # copied to the info class.
    files = command_line.files
    binnumber = command_line.bins

    # Save the extension to output files
    info.extension = command_line.extension

    # Read a potential file describing changes to be done for the parameter
    # names, and number of paramaters plotted (can be let empty, all will
    # then be plotted).
    if command_line.optional_plot_file is not None:
        for line in open(command_line.optional_plot_file[0], 'r'):
            exec(line)

    # Prepare the files, according to the case, load the log.param, and
    # prepare the output (plots folder, .covmat, .info and .log files).
    # After this step, info.files will contain all chains.
    status = prepare(info, files)

    if not status:
        return

    # Compute the mean, maximum of likelihood, 1-sigma variance for this
    # main folder. This will create the info.spam chain
    convergence(info)

    # Create the main chain, which consists in all elements of info.spam
    # put together. This will serve for the plotting.
    chain = np.vstack(info.spam)

    # In case of comparison, launch the prepare and convergence methods,
    # with an additional flag: is_main_chain=False. This will ensure that
    # all the information will not be stored to info.files, info.covmat...
    # but to the specified output, respectively comp_files, comp_spam...
    if command_line.comp is not None:
        comp_files, comp_folder, comp_param = \
            prepare(info, command_line.comp, is_main_chain=False)
        comp_spam, comp_ref_names, comp_tex_names, comp_backup_names, \
            comp_plotted_parameters, comp_boundaries, comp_mean = \
            convergence(info, is_main_chain=False, files=comp_files,
                        param=comp_param)
        comp_mean = comp_mean[0]

        # Create comp_chain
        comp_chain = np.copy(comp_spam[0])
        for i in range(len(comp_spam)-1):
            comp_chain = np.append(comp_chain, comp_spam[i+1], axis=0)

    # Total number of steps.
    weight = chain[:, 0].sum()

    # Covariance matrix computation (for the whole chain)
{'fraction_non_alphanumeric': 0.07152941176470588, 'fraction_numerical': 0.0025882352941176473, 'mean_word_length': 3.055820610687023, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5794412', 'n_tokens_mistral': 2368, 'n_tokens_neox': 2292, 'n_words': 981}