<commit_before># Licensed under an MIT open source license - see LICENSE


'''
Script to create final output form of the HDF5 results files.
'''

import numpy as np
from pandas import HDFStore, DataFrame, concat, read_csv, Series
import os


def convert_format(path, face1, face2=None, design=None, output_type="csv",
                   parameters=None, decimal_places=8, append_comp=True):
    '''
    Takes all HDF5 files in given path comparing face1 to face2 and combines
    them into a single file.

    Parameters
    ----------
    path : str
        Path where files are located.
    face1 : int
        Face of the cube.
    face2: int, optional
        Face of the cube compared to. Disabled for observational comparison.
    design : str or pandas.DataFrame, optional
        If str, assumes a 'csv' file. Disabled for observational
        comparison.
    output_type : str, optional
        Type of file to output.
    parameters : list, optional
        Contains column names of design that are the parameters
        varied in the set. If None, all columns are appended to
        the output file.
    decimal_places : int, optional
        Specify the number of decimal places to keep.
    append_comp : bool, optional
        Append on columns with fiducial numbers copy
    '''

    if path[-1] != "/":
        path += "/"

    if face2 is not None:
        files = [path + f for f in os.listdir(path)
                 if os.path.isfile(path + f)
                 and "_"+str(face1)+"_"+str(face2)+"_" in f
                 and "comparisons" not in f]
    else:
        # Observational comparisons explicitly have 'face' in filename
        files = [path + f for f in os.listdir(path)
                 if os.path.isfile(path + f)
                 and "face_"+str(face1) in f
                 and "comparisons" not in f]
    files.sort()
    print "Files used: %s" % (files)

    if design is not None:
        if isinstance(design, str):
            design = read_csv(design)

        if isinstance(parameters, list):
            design_df = {}
            for param in parameters:
                design_df[param] = Series(design[param])
            design_df = DataFrame(design_df)
        else:
            design_df = design

    for i, f in enumerate(files):
        store = HDFStore(f)
        data_columns = {}
        # Get data from HDF5
        for key in store.keys():
            data = store[key].sort(axis=0).sort(axis=1)
            index = data.index
            mean_data = data.mean(axis=1)
            data_columns[key[1:]] = trunc_float(mean_data, decimal_places)
        store.close()

        # Add on design matrix
        if design is not None:
            for key in design_df:
                # can get nans if the file was made in excel
                design_df = design_df.dropna()
                design_df.index = index
                data_columns[key] = design_df[key]

        data_columns = DataFrame(data_columns)

        if append_comp:
            data_columns["Fiducial"] = \
                Series(np.asarray([i]*len(index)).T, index=index)
            data_columns["Designs"] = Series(index.T, index=index)

        if i == 0:  # Create dataframe
            df = data_columns
        else:  # Add on to dataframe
            df = concat([df, data_columns])

    if face2 is not None:
        filename = "distances_"+str(face1)+"_"+str(face2)
    else:
        filename = "complete_distances_face_"+str(face1)

    if "Name" in df.keys():
        del df["Name"]

    if output_type == "csv":
        df.to_csv(path+filename+".csv")


def convert_fiducial(filename, output_type="csv", decimal_places=8,
                     append_comp=True, num_fids=5):<commit_msg>Optionally keep the original index in the appended df<commit_after># Licensed under an MIT open source license - see LICENSE


'''
Script to create final output form of the HDF5 results files.
'''

import numpy as np
from pandas import HDFStore, DataFrame, concat, read_csv, Series
import os


def convert_format(path, face1, face2=None, design=None, output_type="csv",
                   parameters=None, decimal_places=8, append_comp=True,
                   keep_index=True):
    '''
    Takes all HDF5 files in given path comparing face1 to face2 and combines
    them into a single file.

    Parameters
    ----------
    path : str
        Path where files are located.
    face1 : int
        Face of the cube.
    face2: int, optional
        Face of the cube compared to. Disabled for observational comparison.
    design : str or pandas.DataFrame, optional
        If str, assumes a 'csv' file. Disabled for observational
        comparison.
    output_type : str, optional
        Type of file to output.
    parameters : list, optional
        Contains column names of design that are the parameters
        varied in the set. If None, all columns are appended to
        the output file.
    decimal_places : int, optional
        Specify the number of decimal places to keep.
    append_comp : bool, optional
        Append on columns with fiducial numbers copy
    '''

    if path[-1] != "/":
        path += "/"

    if face2 is not None:
        files = [path + f for f in os.listdir(path)
                 if os.path.isfile(path + f)
                 and "_"+str(face1)+"_"+str(face2)+"_" in f
                 and "comparisons" not in f]
    else:
        # Observational comparisons explicitly have 'face' in filename
        files = [path + f for f in os.listdir(path)
                 if os.path.isfile(path + f)
                 and "face_"+str(face1) in f
                 and "comparisons" not in f]
    files.sort()
    print "Files used: %s" % (files)

    if design is not None:
        if isinstance(design, str):
            design = read_csv(design)

        if isinstance(parameters, list):
            design_df = {}
            for param in parameters:
                design_df[param] = Series(design[param])
            design_df = DataFrame(design_df)
        else:
            design_df = design

    for i, f in enumerate(files):
        store = HDFStore(f)
        data_columns = {}
        # Get data from HDF5
        for key in store.keys():
            data = store[key].sort(axis=0).sort(axis=1)
            index = data.index
            mean_data = data.mean(axis=1)
            data_columns[key[1:]] = trunc_float(mean_data, decimal_places)
        store.close()

        # Add on design matrix
        if design is not None:
            for key in design_df:
                # can get nans if the file was made in excel
                design_df = design_df.dropna()
                design_df.index = index
                data_columns[key] = design_df[key]

        if keep_index:
            data_columns = DataFrame(data_columns, index=index)
        else:
            data_columns = DataFrame(data_columns)

        if append_comp:
            data_columns["Fiducial"] = \
                Series(np.asarray([i]*len(index)).T, index=index)
            data_columns["Designs"] = Series(index.T, index=index)

        if i == 0:  # Create dataframe
            df = data_columns
        else:  # Add on to dataframe
            df = concat([df, data_columns])

        print df.index

    if face2 is not None:
        filename = "distances_"+str(face1)+"_"+str(face2)
    else:
        filename = "complete_distances_face_"+str(face1)

    if "Name" in df.keys():
        del df["Name"]

    if output_type == "csv":
        df.to_csv(path+filename+".csv")


def convert_fiducial(filename, output_type="csv", decimal_places=8,
                     append_comp=True, num_fids=5):
{'fraction_non_alphanumeric': 0.09168081494057725, 'fraction_numerical': 0.006791171477079796, 'mean_word_length': 1.9774494556765163, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4807994', 'n_tokens_mistral': 2233, 'n_tokens_neox': 2147, 'n_words': 716}