<commit_before>'''
This module contains utility functions for producing the figures and tables,
primarily compiling the data for them to use.
'''
from os.path import join as _join

def crop_png(filename):
    '''Autocrop excess white margins from PNG file

    Args
    ----
    filename: str
        Path and filename with extension to .png file to crop
    '''
    from PIL import Image

    img = Image.open(filename)
    img.crop(img.getbbox()).save(filename)

    return None


def filt_paths(path_project, cfg_ann):
    import numpy
    import os

    from ..config import paths, fnames
    from .. import utils

    # Get list of paths filtered by parameters in `cfg_ann['data']`
    path_tag = _join(path_project, paths['tag'])
    path_glide = _join(path_project, paths['glide'])
    data_paths = list()
    exp_names = list()
    for p in os.listdir(path_tag):
        path_exp = _join(path_tag, p)
        if os.path.isdir(path_exp):
            # Concatenate data path
            path_glide_data = _join(path_project, path_glide, p)
            path_subdir = utils.get_subdir(path_glide_data, cfg_ann['data'])
            data_paths.append(_join(path_glide_data, path_subdir))
            exp_names.append(p)

    sort_ind = numpy.argsort(data_paths)
    data_paths = numpy.array(data_paths)[sort_ind]
    exp_names = numpy.array(exp_names)[sort_ind]

    return data_paths, exp_names


def compile_exp_data(path_project, field, cfg_ann):
    '''Walk root tag directory and compile derived values to dataframe

    Args
    ----
    field: pandas.DataFrame
        Field experiments with added rho_mod
    paths: OrderedDict
        Dictionary of smartmove project paths
    fnames: OrderedDict
        Dictionary of smartmove project filenames
    cfg_ann: OrderedDict
        Configuration dictionary for ANN analysis

    match data file from data_accelerometer/
    '''
    from collections import OrderedDict
    import numpy
    import os
    import pandas
    from pyotelem.plots import plotutils

    from ..ann import pre
    from ..config import paths, fnames

    # TODO all of this info could be put into cfg_filter.yaml, then just walk
    # through directories based on cfg_ann.yaml in model path

    def get_sgl_path(path_project, path_glide, path_exp, cfg):
        '''Concatenate path to subglide output directory'''
        from .. import utils

        glide_data_path = _join(path_project, path_glide, path_exp)
        subdir_path = utils.get_subdir(glide_data_path, cfg)
        glide_data_path = _join(glide_data_path, subdir_path)

        return glide_data_path

    cols = [ 'id', 'date', 'animal', 'mod_str', 'duration', 'n_dives',
            'n_sgls_asc', 'n_sgls_des', 'sgls_asc_str', 'sgls_des_str',
            'perc_des', 'perc_asc', 'tritium_id', 'density_kgm3', 'rho_mod']

    data = OrderedDict()
    exp_ids = field['exp_id'].values
    for c in cols:
        data[c] = numpy.array([None,]*len(exp_ids))

    # TODO rewrite this to just use `field` columns
    for i in range(len(exp_ids)):
        path = _join(path_project, paths['tag'], exp_ids[i])

        if os.path.isdir(path):

            exp = field.ix[i, 'exp_id']

            data['date'][i] = '{}-{}-{}'.format(exp[:4], exp[4:6], exp[6:8])

            data['animal'][i] = exp.split('_')[3]

            file_tag = _join(path, 'pydata_{}.p'.format(exp))
            tag = pandas.read_pickle(file_tag)

            file_mask = _join(path, 'mask_tag.p')
            masks = pandas.read_pickle(file_mask)

            start = tag['datetimes'][masks['exp']].iloc[0]<commit_msg>moved TODO in visuals/utils.py to issue #6<commit_after>'''
This module contains utility functions for producing the figures and tables,
primarily compiling the data for them to use.
'''
from os.path import join as _join

def crop_png(filename):
    '''Autocrop excess white margins from PNG file

    Args
    ----
    filename: str
        Path and filename with extension to .png file to crop
    '''
    from PIL import Image

    img = Image.open(filename)
    img.crop(img.getbbox()).save(filename)

    return None


def filt_paths(path_project, cfg_ann):
    import numpy
    import os

    from ..config import paths, fnames
    from .. import utils

    # Get list of paths filtered by parameters in `cfg_ann['data']`
    path_tag = _join(path_project, paths['tag'])
    path_glide = _join(path_project, paths['glide'])
    data_paths = list()
    exp_names = list()
    for p in os.listdir(path_tag):
        path_exp = _join(path_tag, p)
        if os.path.isdir(path_exp):
            # Concatenate data path
            path_glide_data = _join(path_project, path_glide, p)
            path_subdir = utils.get_subdir(path_glide_data, cfg_ann['data'])
            data_paths.append(_join(path_glide_data, path_subdir))
            exp_names.append(p)

    sort_ind = numpy.argsort(data_paths)
    data_paths = numpy.array(data_paths)[sort_ind]
    exp_names = numpy.array(exp_names)[sort_ind]

    return data_paths, exp_names


def compile_exp_data(path_project, field, cfg_ann):
    '''Walk root tag directory and compile derived values to dataframe

    Args
    ----
    field: pandas.DataFrame
        Field experiments with added rho_mod
    paths: OrderedDict
        Dictionary of smartmove project paths
    fnames: OrderedDict
        Dictionary of smartmove project filenames
    cfg_ann: OrderedDict
        Configuration dictionary for ANN analysis

    match data file from data_accelerometer/
    '''
    from collections import OrderedDict
    import numpy
    import os
    import pandas
    from pyotelem.plots import plotutils

    from ..ann import pre
    from ..config import paths, fnames

    def get_sgl_path(path_project, path_glide, path_exp, cfg):
        '''Concatenate path to subglide output directory'''
        from .. import utils

        glide_data_path = _join(path_project, path_glide, path_exp)
        subdir_path = utils.get_subdir(glide_data_path, cfg)
        glide_data_path = _join(glide_data_path, subdir_path)

        return glide_data_path

    cols = [ 'id', 'date', 'animal', 'mod_str', 'duration', 'n_dives',
            'n_sgls_asc', 'n_sgls_des', 'sgls_asc_str', 'sgls_des_str',
            'perc_des', 'perc_asc', 'tritium_id', 'density_kgm3', 'rho_mod']

    data = OrderedDict()
    exp_ids = field['exp_id'].values
    for c in cols:
        data[c] = numpy.array([None,]*len(exp_ids))

    for i in range(len(exp_ids)):
        path = _join(path_project, paths['tag'], exp_ids[i])

        if os.path.isdir(path):

            exp = field.ix[i, 'exp_id']

            data['date'][i] = '{}-{}-{}'.format(exp[:4], exp[4:6], exp[6:8])

            data['animal'][i] = exp.split('_')[3]

            file_tag = _join(path, 'pydata_{}.p'.format(exp))
            tag = pandas.read_pickle(file_tag)

            file_mask = _join(path, 'mask_tag.p')
            masks = pandas.read_pickle(file_mask)

            start = tag['datetimes'][masks['exp']].iloc[0]
{'fraction_non_alphanumeric': 0.10707157114342743, 'fraction_numerical': 0.002423723980610208, 'mean_word_length': 2.8586358635863585, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2730469', 'n_tokens_mistral': 2311, 'n_tokens_neox': 2243, 'n_words': 586}