<commit_before>import io
import os
import sys
import binascii
import numpy as np

from .db import surfs
from .volume import mosaic, unmask

def get_roipack(*args, **kwargs):
    import warnings
    warnings.warn('Please use surfs.getOverlay instead', DeprecationWarning)
    return surfs.getOverlay(*args, **kwargs)

def get_mapper(*args, **kwargs):
    from .mapper import get_mapper
    return get_mapper(*args, **kwargs)

def get_ctmpack(subject, types=("inflated",), method="raw", level=0, recache=False, decimated=False):
    ctmform = surfs.getFiles(subject)['ctmcache']
    lvlstr = ("%dd" if decimated else "%d")%level
    ctmfile = ctmform.format(types=','.join(types), method=method, level=lvlstr)
    if os.path.exists(ctmfile) and not recache:
        return ctmfile

    print("Generating new ctm file...")
    from . import brainctm
    ptmap = brainctm.make_pack(ctmfile, subject, types=types, method=method, level=level, decimate=decimated)
    return ctmfile

def get_cortical_mask(subject, xfmname, type='nearest'):
    from .db import surfs
    if type == 'cortical':
        ppts, polys = surfs.getSurf(subject, "pia", merge=True, nudge=False)
        wpts, polys = surfs.getSurf(subject, "wm", merge=True, nudge=False)
        thickness = np.sqrt(((ppts - wpts)**2).sum(1))

        dist, idx = get_vox_dist(subject, xfmname)
        cortex = np.zeros(dist.shape, dtype=bool)
        verts = np.unique(idx)
        for i, vert in enumerate(verts):
            mask = idx == vert
            cortex[mask] = dist[mask] <= thickness[vert]
            if i % 100 == 0:
                print("%0.3f%%"%(i/float(len(verts)) * 100))
        return cortex
    elif type in ('thick', 'thin'):
        dist, idx = get_vox_dist(subject, xfmname)
        return dist < dict(thick=8, thin=2)[type]
    else:
        return get_mapper(subject, xfmname, type=type).mask


def get_vox_dist(subject, xfmname, surface="fiducial"):
    """Get the distance (in mm) from each functional voxel to the closest
    point on the surface.

    Parameters
    ----------
    subject : str
        Name of the subject
    xfmname : str
        Name of the transform
    shape : tuple
        Output shape for the mask

    Returns
    -------
    dist : ndarray
        Distance (in mm) to the closest point on the surface

    argdist : ndarray
        Point index for the closest point
    """
    import nibabel
    from scipy.spatial import cKDTree

    fiducial, polys = surfs.getSurf(subject, surface, merge=True)
    xfm = surfs.getXfm(subject, xfmname)
    z, y, x = xfm.shape
    idx = np.mgrid[:x, :y, :z].reshape(3, -1).T
    mm = xfm.inv(idx)

    tree = cKDTree(fiducial)
    dist, argdist = tree.query(mm)
    dist.shape = (x,y,z)
    argdist.shape = (x,y,z)
    return dist.T, argdist.T

def get_hemi_masks(subject, xfmname, type='nearest'):
    '''Returns a binary mask of the left and right hemisphere
    surface voxels for the given subject.
    '''
    return get_mapper(subject, xfmname, type=type).hemimasks

def add_roi(data, name="new_roi", recache=False, open_inkscape=True, add_path=True, **kwargs):
    import subprocess as sp
    from matplotlib.pylab import imsave
    from .utils import get_roipack
    from . import quickflat
    rois = get_roipack(data.subject)
    try:
        import cStringIO
        fp = cStringIO.StringIO()
    except:
        fp = io.StringIO()
    quickflat.make_png(fp, data, height=1024, recache=recache, with_rois=False, with_labels=False, **kwargs)
    fp.seek(0)
    rois.add_roi(name, binascii.b2a_base64(fp.read()), add_path)
    if open_inkscape:
        return sp.call(["inkscape", '-f', rois.svgfile])

def get_roi_verts(subject, roi=None):
    '''Return vertices for the given ROIs'''
    rois = get_roipack(subject)

    if roi is None:
        roi = rois.names

    roidict = dict()
    if isinstance(roi, str):
        roi = [roi]
<commit_msg>fix small problem in utils<commit_after>import io
import os
import sys
import binascii
import numpy as np

from .db import surfs
from .volume import mosaic, unmask

def get_roipack(*args, **kwargs):
    import warnings
    warnings.warn('Please use surfs.getOverlay instead', DeprecationWarning)
    return surfs.getOverlay(*args, **kwargs)

def get_mapper(*args, **kwargs):
    from .mapper import get_mapper
    return get_mapper(*args, **kwargs)

def get_ctmpack(subject, types=("inflated",), method="raw", level=0, recache=False, decimated=False):
    ctmform = surfs.getFiles(subject)['ctmcache']
    lvlstr = ("%dd" if decimated else "%d")%level
    ctmfile = ctmform.format(types=','.join(types), method=method, level=lvlstr)
    if os.path.exists(ctmfile) and not recache:
        return ctmfile

    print("Generating new ctm file...")
    from . import brainctm
    ptmap = brainctm.make_pack(ctmfile, subject, types=types, method=method, level=level, decimate=decimated)
    return ctmfile

def get_cortical_mask(subject, xfmname, type='nearest'):
    from .db import surfs
    if type == 'cortical':
        ppts, polys = surfs.getSurf(subject, "pia", merge=True, nudge=False)
        wpts, polys = surfs.getSurf(subject, "wm", merge=True, nudge=False)
        thickness = np.sqrt(((ppts - wpts)**2).sum(1))

        dist, idx = get_vox_dist(subject, xfmname)
        cortex = np.zeros(dist.shape, dtype=bool)
        verts = np.unique(idx)
        for i, vert in enumerate(verts):
            mask = idx == vert
            cortex[mask] = dist[mask] <= thickness[vert]
            if i % 100 == 0:
                print("%0.3f%%"%(i/float(len(verts)) * 100))
        return cortex
    elif type in ('thick', 'thin'):
        dist, idx = get_vox_dist(subject, xfmname)
        return dist < dict(thick=8, thin=2)[type]
    else:
        return get_mapper(subject, xfmname, type=type).mask


def get_vox_dist(subject, xfmname, surface="fiducial"):
    """Get the distance (in mm) from each functional voxel to the closest
    point on the surface.

    Parameters
    ----------
    subject : str
        Name of the subject
    xfmname : str
        Name of the transform
    shape : tuple
        Output shape for the mask

    Returns
    -------
    dist : ndarray
        Distance (in mm) to the closest point on the surface

    argdist : ndarray
        Point index for the closest point
    """
    import nibabel
    from scipy.spatial import cKDTree

    fiducial, polys = surfs.getSurf(subject, surface, merge=True)
    xfm = surfs.getXfm(subject, xfmname)
    z, y, x = xfm.shape
    idx = np.mgrid[:x, :y, :z].reshape(3, -1).T
    mm = xfm.inv(idx)

    tree = cKDTree(fiducial)
    dist, argdist = tree.query(mm)
    dist.shape = (x,y,z)
    argdist.shape = (x,y,z)
    return dist.T, argdist.T

def get_hemi_masks(subject, xfmname, type='nearest'):
    '''Returns a binary mask of the left and right hemisphere
    surface voxels for the given subject.
    '''
    return get_mapper(subject, xfmname, type=type).hemimasks

def add_roi(data, name="new_roi", recache=False, open_inkscape=True, add_path=True, **kwargs):
    import subprocess as sp
    from matplotlib.pylab import imsave
    from .utils import get_roipack
    from . import quickflat
    rois = get_roipack(data.subject)
    try:
        import cStringIO
        fp = cStringIO.StringIO()
    except:
        fp = io.StringIO()
    quickflat.make_png(fp, data, height=1024, recache=recache, with_rois=False, with_labels=False, **kwargs)
    fp.seek(0)
    rois.add_roi(name, binascii.b2a_base64(fp.read()), add_path)
    if open_inkscape:
        return sp.call(["inkscape", '-f', rois.svgfile])

def get_roi_verts(subject, roi=None):
    '''Return vertices for the given ROIs'''
    rois = surfs.getOverlay(subject)

    if roi is None:
        roi = rois.names

    roidict = dict()
    if isinstance(roi, str):
        roi = [roi]

{'fraction_non_alphanumeric': 0.12219101123595505, 'fraction_numerical': 0.006128702757916241, 'mean_word_length': 2.9964285714285714, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2705539', 'n_tokens_mistral': 2719, 'n_tokens_neox': 2644, 'n_words': 710}