<commit_before>#! /usr/bin/env python
# coding: utf-8

# Returns arrays of post-processed data from a list of data files.

from __future__ import absolute_import

__version__ = "$Revision$"
# $URL$
# $Date$

import scipy
import scipy.io
from pdb import set_trace

from . import bootstrap

__all__ = ['main','quantile','postprocess']

# Define global variables.
prctilesTab = (0., 0.1, 0.5, 0.9, 1.)
# These depends on the layout of the data files.
funcEvalsIndex = 0
fitValIndex = 2
maxEvalsFactor = 1e4
valuesOfInterest = (1.0, 1.0e-2, 1.0e-4, 1.0e-6, 1.0e-8) #has to be sorted
valuesForDataProf = (1.0, 1.0e-2, 1.0e-4, 1.0e-6, 1.0e-8) #has to be sorted


class Error(Exception):
    """ Base class for errors. """
    #TODO: what the?
    pass


class MissingValueError(Error):
    """ Error if a mandatory value is not found within a file.
        Returns a message with the missing value and the respective
        file."""

    def __init__(self,value, filename):
        self.value = value
        self.filename = filename

    def __str__(self):
        message = 'The value %s was not found in file %s!' % \
                  (self.value, self.filename)
        return repr(message)


# Remarks:
# Need for the optimum of the function!
# Let's store everything into memory.
# TODO: aligned with function values or function evaluations should be a choice.

def postprocess(dataFiles, fvalueToReach, maxEvals):
    """Post processes raw data files, returns post-processed data in arrays."""
    #TODO this function is long and ugly.
    #set_trace()
    dataSets = split(dataFiles)
    # Is it ok to store all the data in memory?

    measuredMaxEvals = []
    for i in dataSets:
        measuredMaxEvals.append(i.userDefMaxEvals)

    measuredMaxEvals = max(measuredMaxEvals)
    maxEvals = min((measuredMaxEvals, maxEvals))

    # Parallel construction of the post-processed arrays:
    currentFuncEvals = 1.
    currentFitValue = scipy.inf # Minimization

    isFinished = len(dataSets) * [False]
    res = len(dataSets) * [0]
    vals = len(dataSets) * [0]
    previousVals = len(dataSets) * [0]
    arrayFullTab = [] #aligned by function values.
    arrayTab = {} #aligned by function values.
    rlDistr = {} #aligned by function values.
    iValuesOfInterest = 0
    iValuesForDataProf = 0
    #arrayFig = [] #aligned by function evaluations.
    #currentLines = []
    #dataSetsToRead = dataSets
    #dataSetsToRemove = []
    #res = []

    while not all(isFinished) and iValuesOfInterest < len(valuesOfInterest):
        #CurrentLines is a list of arrays with lines from the dataSets
        #set_trace()
        # Align data
        for i in range(len(dataSets)):
            res[i] = dataSets[i].set[dataSets[i].currentPos, funcEvalsIndex]
            vals[i] = (dataSets[i].set[dataSets[i].currentPos, fitValIndex] -
                       fvalueToReach + 1.0e-8)
            if not isFinished[i]:
                while (dataSets[i].currentPos < len(dataSets[i].set)-1 and
                       dataSets[i].set[dataSets[i].currentPos, 0] < maxEvals and
                       vals[i] > currentFitValue):
                    #Get the function values from the last valid position
                    previousVals[i] = vals[i]
                    dataSets[i].currentPos += 1
                    res[i] = dataSets[i].set[dataSets[i].currentPos,
                                             funcEvalsIndex]
                    vals[i] = dataSets[i].set[dataSets[i].currentPos,
                                              fitValIndex]
                if (dataSets[i].currentPos == len(dataSets[i].set) - 1 or 
                    dataSets[i].set[dataSets[i].currentPos, 0] >= maxEvals):
                    isFinished[i] = True

        # Process currentLines:
        # 1) Check that the current lines correspond to the current position
        # either for function values or function evaluations.
        # 2) compute what needs to be computed.
        #if isFuncEvalsAligned:
        #arrayTab = scipy.append(scipy.floor(currentFunctionEvals),
                               #bootstrap.prctile(res))
        #if currentFunctionEvals < 10.:
            #currentFunctionEvals += 1.
        #else:
            #currentFunctionEvals *= 10.**0.05
        #if isFitValAligned:
        if scipy.isinf(currentFitValue):
            currentFitValue = max(valuesOfInterest[0],
                                  10**(scipy.ceil(scipy.log10(max(vals))*5)/5))
        tmp = scipy.append(currentFitValue, computevalues(res,maxEvals))
        for j in range(2,7):
            if tmp[j] == maxEvals:
                tmp[j] = scipy.nan
        arrayFullTab.append(tmp[:])

        #arrayFullTab is displayed in the figures. As opposed to the tables,
        #we are not constrained to use the default maxEvals for comparison
        #sake. We can then use the user-defined maximum number of function
        #evaluations.

        # Tests if we have covered all of the valuesOfInterest (using the
        # iterator iValuesOfInterest and then if 
        # currentFitValue == valuesOfInterest[iValuesOfInterest].
        if (iValuesOfInterest < len(valuesOfInterest) and
            (currentFitValue-valuesOfInterest[iValuesOfInterest] <
             max([currentFitValue * (1.0 - 10.**-0.1),
                  scipy.finfo('float64').resolution]))):
            arrayTab[valuesOfInterest[iValuesOfInterest]] = computevalues(res,
                                                     maxEvals, dispersion=True)
            previousVals.sort()
            ranksOfInterest = [previousVals[0], previousVals[2],
                               bootstrap.prctile(previousVals, 50)[0],
                               previousVals[-3], previousVals[-1]]
            #Gets the last valid function values. Might not be corresponding
            #to maxEvals.
            # minimization
            #set_trace()
            for j in range(4,9): #index of the percentiles
                if arrayTab[valuesOfInterest[iValuesOfInterest]][j] >= maxEvals:
                    arrayTab[valuesOfInterest[iValuesOfInterest]][j] = ranksOfInterest[j-4]
                    #TODO rewrite
                    #use vals either here or in computevalues.
            iValuesOfInterest += 1

        if (iValuesForDataProf < len(valuesForDataProf) and
            (currentFitValue-valuesForDataProf[iValuesForDataProf] <
             max([currentFitValue*(1.0-10.**-0.1),
                  scipy.finfo('float64').resolution]))):
            rlDistr[valuesForDataProf[iValuesForDataProf]]= res[:]
            iValuesForDataProf += 1

        currentFitValue *= 10.**(-0.2)
    arrayFullTab = scipy.vstack(arrayFullTab)
    tmp = []
    for i in valuesOfInterest:
        if i in arrayTab:
            #set_trace()
            tmp.append(scipy.append(scipy.array([i]), arrayTab[i])) #Weird.
        else:
            vals.sort()
            ranksOfInterest = []
            tmp2 = scipy.array((i, scipy.inf, scipy.inf, scipy.inf, 0.0,
                                vals[0], vals[2],
                                bootstrap.prctile(vals, 50)[0], vals[-3],
                                vals[-1])) # minimization
            tmp.append(tmp2)
            #tmp.append(scipy.array((i,scipy.inf,scipy.inf,scipy.inf,len(dataSets),
            #                        maxEvals,maxEvals,maxEvals,maxEvals,
            #                        maxEvals)))
            #Raymond: Why this line? It makes the computation of the probability
            #of success wrong.

    #set_trace()
    arrayTab = scipy.vstack(tmp)
    #set_trace()
    return (arrayFullTab, arrayTab, rlDistr, len(dataSets), maxEvals)


def testpostprocess():
    import glob
    files = ['/users/dsa/ros/Desktop/coco/BBOB/demo/matlab/datafminunc_worestart/data_f1/datafminunc0_f1_DIM2.dat']<commit_msg>BUG FIX: correct the display of CMA results.<commit_after>#! /usr/bin/env python
# coding: utf-8

# Returns arrays of post-processed data from a list of data files.

from __future__ import absolute_import

import scipy
import scipy.io
from pdb import set_trace

from bbob_pproc import bootstrap

__version__ = "$Revision$"
# $URL$
# $Date$

__all__ = ['main', 'postprocess']

# Define global variables.
prctilesTab = (0., 0.1, 0.5, 0.9, 1.)
# These depends on the layout of the data files.
funcEvalsIndex = 0
fitValIndex = 2
maxEvalsFactor = 1e4
valuesOfInterest = (1.0, 1.0e-2, 1.0e-4, 1.0e-6, 1.0e-8) #has to be sorted
valuesForDataProf = (1.0, 1.0e-2, 1.0e-4, 1.0e-6, 1.0e-8) #has to be sorted


class Error(Exception):
    """ Base class for errors. """
    #TODO: what the?
    pass


class MissingValueError(Error):
    """ Error if a mandatory value is not found within a file.
        Returns a message with the missing value and the respective
        file."""

    def __init__(self,value, filename):
        self.value = value
        self.filename = filename

    def __str__(self):
        message = 'The value %s was not found in file %s!' % \
                  (self.value, self.filename)
        return repr(message)


# Remarks:
# Need for the optimum of the function!
# Let's store everything into memory.
# TODO: aligned with function values or function evaluations should be a choice.

def postprocess(dataFiles, fvalueToReach, maxEvals):
    """Post processes raw data files, returns post-processed data in arrays."""
    #TODO this function is long and ugly.
    #set_trace()
    dataSets = split(dataFiles)
    # Is it ok to store all the data in memory?

    measuredMaxEvals = []
    for i in dataSets:
        measuredMaxEvals.append(i.userDefMaxEvals)

    measuredMaxEvals = max(measuredMaxEvals)
    maxEvals = min((measuredMaxEvals, maxEvals))

    # Parallel construction of the post-processed arrays:
    currentFuncEvals = 1.
    currentFitValue = scipy.inf # Minimization

    isFinished = len(dataSets) * [False]
    res = len(dataSets) * [0]
    vals = len(dataSets) * [0]
    previousVals = len(dataSets) * [0]
    arrayFullTab = [] #aligned by function values.
    arrayTab = {} #aligned by function values.
    rlDistr = {} #aligned by function values.
    iValuesOfInterest = 0
    iValuesForDataProf = 0
    #arrayFig = [] #aligned by function evaluations.
    #currentLines = []
    #dataSetsToRead = dataSets
    #dataSetsToRemove = []
    #res = []

    while not all(isFinished) and iValuesOfInterest < len(valuesOfInterest):
        #CurrentLines is a list of arrays with lines from the dataSets
        #set_trace()
        # Align data
        for i in range(len(dataSets)):
            res[i] = dataSets[i].set[dataSets[i].currentPos, funcEvalsIndex]
            vals[i] = (dataSets[i].set[dataSets[i].currentPos, fitValIndex] -
                       fvalueToReach + 1.0e-8)
            if not isFinished[i]:
                while (dataSets[i].currentPos < len(dataSets[i].set)-1 and
                       dataSets[i].set[dataSets[i].currentPos, 0] < maxEvals and
                       vals[i] > currentFitValue):
                    #Get the function values from the last valid position
                    previousVals[i] = vals[i]
                    dataSets[i].currentPos += 1
                    res[i] = dataSets[i].set[dataSets[i].currentPos,
                                             funcEvalsIndex]
                    vals[i] = (dataSets[i].set[dataSets[i].currentPos,
                                               fitValIndex] - fvalueToReach + 
                               1.0e-8)
                if (dataSets[i].currentPos == len(dataSets[i].set) - 1 or 
                    dataSets[i].set[dataSets[i].currentPos, 0] >= maxEvals):
                    isFinished[i] = True

        # Process currentLines:
        # 1) Check that the current lines correspond to the current position
        # either for function values or function evaluations.
        # 2) compute what needs to be computed.
        #if isFuncEvalsAligned:
        #arrayTab = scipy.append(scipy.floor(currentFunctionEvals),
                               #bootstrap.prctile(res))
        #if currentFunctionEvals < 10.:
            #currentFunctionEvals += 1.
        #else:
            #currentFunctionEvals *= 10.**0.05
        #if isFitValAligned:
        if scipy.isinf(currentFitValue):
            currentFitValue = max(valuesOfInterest[0],
                                  10**(scipy.ceil(scipy.log10(max(vals))*5)/5))
            #What does this mean?
        tmp = scipy.append(currentFitValue, computevalues(res,maxEvals))
        for j in range(2,7):
            if tmp[j] == maxEvals:
                tmp[j] = scipy.nan
        arrayFullTab.append(tmp[:])

        #arrayFullTab is displayed in the figures. As opposed to the tables,
        #we are not constrained to use the default maxEvals for comparison
        #sake. We can then use the user-defined maximum number of function
        #evaluations.

        # Tests if we have covered all of the valuesOfInterest (using the
        # iterator iValuesOfInterest and then if 
        # currentFitValue == valuesOfInterest[iValuesOfInterest].
        if (iValuesOfInterest < len(valuesOfInterest) and
            (currentFitValue-valuesOfInterest[iValuesOfInterest] <
             max([currentFitValue * (1.0 - 10.**-0.1),
                  scipy.finfo('float64').resolution]))):
            arrayTab[valuesOfInterest[iValuesOfInterest]] = computevalues(res,
                                                     maxEvals, dispersion=True)
            previousVals.sort()
            ranksOfInterest = [previousVals[0], previousVals[2],
                               bootstrap.prctile(previousVals, 50)[0],
                               previousVals[-3], previousVals[-1]]
            #Gets the last valid function values. Might not be corresponding
            #to maxEvals.
            # minimization
            #set_trace()
            for j in range(4,9): #index of the percentiles
                if arrayTab[valuesOfInterest[iValuesOfInterest]][j] >= maxEvals:
                    arrayTab[valuesOfInterest[iValuesOfInterest]][j] = ranksOfInterest[j-4]
                    #TODO rewrite
                    #use vals either here or in computevalues.
            iValuesOfInterest += 1

        if (iValuesForDataProf < len(valuesForDataProf) and
            (currentFitValue-valuesForDataProf[iValuesForDataProf] <
             max([currentFitValue*(1.0-10.**-0.1),
                  scipy.finfo('float64').resolution]))):
            rlDistr[valuesForDataProf[iValuesForDataProf]]= res[:]
            iValuesForDataProf += 1

        currentFitValue *= 10.**(-0.2)
    arrayFullTab = scipy.vstack(arrayFullTab)
    tmp = []
    for i in valuesOfInterest:
        if i in arrayTab:
            #set_trace()
            tmp.append(scipy.append(scipy.array([i]), arrayTab[i])) #Weird.
        else:
            vals.sort()
            ranksOfInterest = []
            tmp2 = scipy.array((i, scipy.inf, scipy.inf, scipy.inf, 0.0,
                                vals[0], vals[2],
                                bootstrap.prctile(vals, 50)[0], vals[-3],
                                vals[-1])) # minimization
            tmp.append(tmp2)
            #tmp.append(scipy.array((i,scipy.inf,scipy.inf,scipy.inf,len(dataSets),
            #                        maxEvals,maxEvals,maxEvals,maxEvals,
            #                        maxEvals)))
            #Raymond: Why this line? It makes the computation of the probability
            #of success wrong.

    #set_trace()
    arrayTab = scipy.vstack(tmp)
    #set_trace()
    return (arrayFullTab, arrayTab, rlDistr, len(dataSets), maxEvals)


def testpostprocess():
    files = ['/users/dsa/ros/Desktop/coco/BBOB/demo/matlab/datafminunc_worestart/data_f1/datafminunc0_f1_DIM2.dat']
{'fraction_non_alphanumeric': 0.10319068118511016, 'fraction_numerical': 0.01551025576095214, 'mean_word_length': 2.0667831489031254, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1928244', 'n_tokens_mistral': 4951, 'n_tokens_neox': 4593, 'n_words': 1178}