<commit_before>"""

afqbrowser.browser: data munging for AFQ-Browser

"""
import os
import os.path as op
from glob import glob
import json
import shutil
from collections import OrderedDict
import scipy.io as sio
import pandas as pd
import numpy as np
import afqbrowser as afqb

# Shim for Python2/Python3:
try:
    from http.server import SimpleHTTPRequestHandler
    import socketserver
except ImportError:
    from SimpleHTTPServer import SimpleHTTPRequestHandler
    import SocketServer as socketserver


def _create_metadata(subject_ids, meta_fname):
    """Helper function to create a minimal metadata file."""
    meta_df = pd.DataFrame({"subjectID": subject_ids},
                           index=range(len(subject_ids)))
    meta_df.to_csv(meta_fname)


def tracula2nodes(stats_dir, out_path=None, metadata=None):
    """
    Create a nodes table from a TRACULA `stats` directory.

    Read data processed with TRACULA [1]_ and generate an AFQ-browser compliant
    nodes table.

    Parameters
    ----------
    stats_dir : str
        Full path to a directory containing stats results from TRACULA
        processing.

    out_path : str, optional
        Full path to directory where the nodes table will be saved.

    metadata : str, optional
        Full path to a file with user-supplied metadata. This has to be a csv
        file with column headers in the first row, including a column named
        "subjectID". For an example, see the 'data/subjects.csv' that comes
        with the software.

    Returns
    -------

    Notes
    -----
    .. [1] Automated probabilistic reconstruction of white-matter pathways in
           health and disease using an atlas of the underlying anatomy. Yendiki
           A, <NAME>, <NAME>, <NAME>, <NAME>, <NAME>, <NAME>, <NAME>, <NAME>, <NAME>, <NAME>, <NAME> and <NAME>
           (2011). Front. Neuroinform. 5:23. doi: 10.3389/fninf.2011.00023
    """
    ll = glob(op.join(stats_dir, '*.txt'))

    tracks = []
    metrics = []
    for l in ll:
        tt = '.'.join(op.split(l)[-1].split('.')[:2])
        if not (tt.startswith('rh') or tt.startswith('lh')):
            tt = tt.split('.')[0]
        tracks.append(tt)
        metrics.append((op.splitext(op.split(l)[-1])[0]).split('.')[-1])

    tracks = list(set(tracks))
    metrics = [l for l in list(set(metrics)) if l not in ['mean', 'inputs']]
    streamlines = OrderedDict()

    for tt in tracks:
        coords_file = tt + '.avg33_mni_bbr.coords.mean.txt'
        coords = np.loadtxt(op.join(stats_dir, coords_file))
        streamlines[tt] = {'coreFiber': coords.tolist()}
        first_metric = True
        for m in metrics:
            if first_metric:
                fname = op.join(stats_dir, tt + '.avg33_mni_bbr.' + m + '.txt')
                df_nodes = pd.read_csv(fname, delimiter=' ')
                df_nodes = df_nodes.drop(
                    filter(lambda x: x.startswith('Unnamed'),
                           df_nodes.columns),
                    axis=1)
                n_nodes, n_subjects = df_nodes.shape
                re_data = df_nodes.as_matrix().T.reshape(n_nodes * n_subjects)
                re_nodes = np.tile(np.arange(n_nodes), n_subjects)
                re_subs = np.concatenate(
                    [[s for i in range(n_nodes)] for s in df_nodes.columns])
                re_track = np.repeat(tt, n_subjects * n_nodes)
                re_df = pd.DataFrame({'subjectID': re_subs,
                                      'tractID': re_track,
                                      'nodeID': re_nodes,
                                      m: re_data})
                first_metric = False
            else:
                fname = op.join(stats_dir, tt + '.avg33_mni_bbr.' + m + '.txt')
                re_data = df_nodes.as_matrix().T.reshape(n_nodes * n_subjects)
                re_df[m] = re_data

    if out_path is None:
        out_path = '.'

    nodes_fname = op.join(out_path, 'nodes.csv')
    re_df.to_csv(nodes_fname, index=False)

    meta_fname = op.join(out_path, 'subjects.csv')

    if metadata is None:<commit_msg>Stroe and concatenate the dataframes for each tract.<commit_after>"""

afqbrowser.browser: data munging for AFQ-Browser

"""
import os
import os.path as op
from glob import glob
import json
import shutil
from collections import OrderedDict
import scipy.io as sio
import pandas as pd
import numpy as np
import afqbrowser as afqb

# Shim for Python2/Python3:
try:
    from http.server import SimpleHTTPRequestHandler
    import socketserver
except ImportError:
    from SimpleHTTPServer import SimpleHTTPRequestHandler
    import SocketServer as socketserver


def _create_metadata(subject_ids, meta_fname):
    """Helper function to create a minimal metadata file."""
    meta_df = pd.DataFrame({"subjectID": subject_ids},
                           index=range(len(subject_ids)))
    meta_df.to_csv(meta_fname)


def tracula2nodes(stats_dir, out_path=None, metadata=None):
    """
    Create a nodes table from a TRACULA `stats` directory.

    Read data processed with TRACULA [1]_ and generate an AFQ-browser compliant
    nodes table.

    Parameters
    ----------
    stats_dir : str
        Full path to a directory containing stats results from TRACULA
        processing.

    out_path : str, optional
        Full path to directory where the nodes table will be saved.

    metadata : str, optional
        Full path to a file with user-supplied metadata. This has to be a csv
        file with column headers in the first row, including a column named
        "subjectID". For an example, see the 'data/subjects.csv' that comes
        with the software.

    Returns
    -------

    Notes
    -----
    .. [1] Automated probabilistic reconstruction of white-matter pathways in
           health and disease using an atlas of the underlying anatomy. <NAME>, <NAME>, <NAME>, <NAME>, <NAME>, <NAME>, <NAME>, <NAME>, <NAME>, <NAME>, <NAME>, <NAME> and <NAME>
           (2011). Front. Neuroinform. 5:23. doi: 10.3389/fninf.2011.00023
    """
    ll = glob(op.join(stats_dir, '*.txt'))

    tracks = []
    metrics = []
    for l in ll:
        tt = '.'.join(op.split(l)[-1].split('.')[:2])
        if not (tt.startswith('rh') or tt.startswith('lh')):
            tt = tt.split('.')[0]
        tracks.append(tt)
        metrics.append((op.splitext(op.split(l)[-1])[0]).split('.')[-1])

    tracks = list(set(tracks))
    metrics = [l for l in list(set(metrics)) if l not in ['mean', 'inputs']]
    streamlines = OrderedDict()
    dfs = []
    for tt in tracks:
        coords_file = tt + '.avg33_mni_bbr.coords.mean.txt'
        coords = np.loadtxt(op.join(stats_dir, coords_file))
        streamlines[tt] = {'coreFiber': coords.tolist()}
        first_metric = True
        for m in metrics:
            if first_metric:
                fname = op.join(stats_dir, tt + '.avg33_mni_bbr.' + m + '.txt')
                df_nodes = pd.read_csv(fname, delimiter=' ')
                df_nodes = df_nodes.drop(
                    filter(lambda x: x.startswith('Unnamed'),
                           df_nodes.columns),
                    axis=1)
                n_nodes, n_subjects = df_nodes.shape
                re_data = df_nodes.as_matrix().T.reshape(n_nodes * n_subjects)
                re_nodes = np.tile(np.arange(n_nodes), n_subjects)
                re_subs = np.concatenate(
                    [[s for i in range(n_nodes)] for s in df_nodes.columns])
                re_track = np.repeat(tt, n_subjects * n_nodes)
                re_df = pd.DataFrame({'subjectID': re_subs,
                                      'tractID': re_track,
                                      'nodeID': re_nodes,
                                      m: re_data})
                first_metric = False
            else:
                fname = op.join(stats_dir, tt + '.avg33_mni_bbr.' + m + '.txt')
                re_data = df_nodes.as_matrix().T.reshape(n_nodes * n_subjects)
                re_df[m] = re_data
            dfs.append(re_df)

    nodes_df = pd.concat(dfs)
    if out_path is None:
        out_path = '.'

    nodes_fname = op.join(out_path, 'nodes.csv')
    nodes_df.to_csv(nodes_fname, index=False)

    meta_fname = op.join(out_path, 'subjects.csv')

    if metadata is None:
{'fraction_non_alphanumeric': 0.11201063315611406, 'fraction_numerical': 0.009666505558240696, 'mean_word_length': 2.1175141242937854, 'pattern_counts': {'":': 2, '<': 28, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '880019', 'n_tokens_mistral': 2647, 'n_tokens_neox': 2516, 'n_words': 707}