<commit_before>"""Support for reading DWI data from DICOM files."""

from __future__ import absolute_import, division, print_function
import os.path

import numpy as np
import dicom

import dwi.util


def read_dir(dirname):
    """Read a directory containing DICOM files. See dicomfile.read_files().
    """
    # Sometimes the files reside in an additional 'DICOM' subdirectory.
    path = os.path.join(dirname, 'DICOM')
    if os.path.isdir(path):
        dirname = path
    filenames = os.listdir(dirname)
    pathnames = [os.path.join(dirname, f) for f in filenames]
    return read_files(pathnames)


def read_files(filenames):
    """Read a bunch of files, each containing a single slice with one b-value,
    and construct a 4d image array.

    The slices are sorted simply by their position as it is, assuming it only
    changes in one dimension. In case there are more than one scan of
    a position and a b-value, the files are averaged by mean.

    DICOM files without pixel data are silently skipped.
    """
    d = {}
    for f in filenames:
        read_slice(f, d)
    positions = sorted(d['positions'])
    bvalues = sorted(d['bvalues'])
    echotimes = sorted(d['echotimes'])
    slices = d['slices']
    # If any slices are scanned multiple times, use mean.
    for k, v in slices.iteritems():
        slices[k] = np.mean(v, axis=0)
    image = construct_image(slices, positions, bvalues, echotimes)
    if len(bvalues) == image.shape[-1]:
        parameters = list(bvalues)
    elif len(echotimes) == image.shape[-1]:
        parameters = list(echotimes)
    else:
        raise ValueError('Inconsistent parameters')
    r = dict(image=image, bset=bvalues, echotimes=echotimes,
             parameters=parameters, voxel_spacing=d['voxel_spacing'])
    return r


def read_slice(filename, d):
    """Read a single slice."""
    try:
        df = dicom.read_file(filename)
    except dicom.filereader.InvalidDicomError as e:
        dwi.util.report('Error reading {f}: {e}'.format(f=filename, e=e))
        return
    if 'PixelData' not in df:
        return
    d.setdefault('orientation', df.ImageOrientationPatient)
    if d['orientation'] != df.ImageOrientationPatient:
        raise Exception('Orientation mismatch.')
    d.setdefault('shape', df.pixel_array.shape)
    if d['shape'] != df.pixel_array.shape:
        raise Exception('Shape mismatch.')
    d.setdefault('dtype', df.pixel_array.dtype)
    if d['dtype'] != df.pixel_array.dtype:
        raise Exception('Type mismatch.')
    d.setdefault('voxel_spacing', get_voxel_spacing(df))
    position = tuple(float(x) for x in df.ImagePositionPatient)
    bvalue = get_bvalue(df)
    echotime = df.get('EchoTime')
    pixels = get_pixels(df)
    d.setdefault('positions', set()).add(position)
    d.setdefault('bvalues', set()).add(bvalue)
    d.setdefault('echotimes', set()).add(echotime)
    key = (position, bvalue, echotime)
    slices = d.setdefault('slices', {})  # Indexed by (pos, b, echotime)...
    slices.setdefault(key, []).append(pixels)  # ...are lists of slices.


def construct_image(slices, positions, bvalues, echotimes):
    """Construct uniform image array from slice dictionary."""
    w, h = slices.values()[0].shape
    dtype = slices.values()[0].dtype
    shape = (len(positions), w, h, len(bvalues), len(echotimes))
    image = np.empty(shape, dtype=dtype)
    image.fill(np.nan)
    for key, value in slices.iteritems():
        pos = positions.index(key[0])
        bv = bvalues.index(key[1])
        et = echotimes.index(key[2])
        image[pos, :, :, bv, et] = value
    if image.shape[3] == 1:
        image = image.squeeze(axis=3)
    elif image.shape[4] == 1:
        image = image.squeeze(axis=4)
    assert image.ndim == 4, image.shape
    if np.isnan(np.min(image)):
        raise Exception('Slices missing from shape {:s}.'.format(shape))
    return image<commit_msg>don't allow overlapping slices no moar<commit_after>"""Support for reading DWI data from DICOM files."""

from __future__ import absolute_import, division, print_function
import os.path

import numpy as np
import dicom

import dwi.util


def read_dir(dirname):
    """Read a directory containing DICOM files. See dicomfile.read_files().
    """
    # Sometimes the files reside in an additional 'DICOM' subdirectory.
    path = os.path.join(dirname, 'DICOM')
    if os.path.isdir(path):
        dirname = path
    filenames = os.listdir(dirname)
    pathnames = [os.path.join(dirname, f) for f in filenames]
    return read_files(pathnames)


def read_files(filenames):
    """Read a bunch of files, each containing a single slice with one b-value,
    and construct a 4d image array.

    The slices are sorted simply by their position as it is, assuming it only
    changes in one dimension. In case there are more than one scan of
    a position and a b-value, the files are averaged by mean.

    DICOM files without pixel data are silently skipped.
    """
    d = {}
    for f in filenames:
        read_slice(f, d)
    positions = sorted(d['positions'])
    bvalues = sorted(d['bvalues'])
    echotimes = sorted(d['echotimes'])
    image = construct_image(d['slices'], positions, bvalues, echotimes)
    if len(bvalues) == image.shape[-1]:
        parameters = list(bvalues)
    elif len(echotimes) == image.shape[-1]:
        parameters = list(echotimes)
    else:
        raise ValueError('Inconsistent parameters')
    r = dict(image=image, bset=bvalues, echotimes=echotimes,
             parameters=parameters, voxel_spacing=d['voxel_spacing'])
    return r


def read_slice(filename, d):
    """Read a single slice."""
    try:
        df = dicom.read_file(filename)
    except dicom.filereader.InvalidDicomError as e:
        dwi.util.report('Error reading {f}: {e}'.format(f=filename, e=e))
        return
    if 'PixelData' not in df:
        return
    d.setdefault('orientation', df.ImageOrientationPatient)
    if d['orientation'] != df.ImageOrientationPatient:
        raise Exception('Orientation mismatch.')
    d.setdefault('shape', df.pixel_array.shape)
    if d['shape'] != df.pixel_array.shape:
        raise Exception('Shape mismatch.')
    d.setdefault('dtype', df.pixel_array.dtype)
    if d['dtype'] != df.pixel_array.dtype:
        raise Exception('Type mismatch.')
    d.setdefault('voxel_spacing', get_voxel_spacing(df))
    position = tuple(float(x) for x in df.ImagePositionPatient)
    bvalue = get_bvalue(df)
    echotime = df.get('EchoTime')
    pixels = get_pixels(df)
    d.setdefault('positions', set()).add(position)
    d.setdefault('bvalues', set()).add(bvalue)
    d.setdefault('echotimes', set()).add(echotime)
    key = (position, bvalue, echotime)
    slices = d.setdefault('slices', {})
    if key in slices:
        raise ValueError('Overlapping slices: {}, {}'.format(key, filename))
    slices[key] = pixels


def construct_image(slices, positions, bvalues, echotimes):
    """Construct uniform image array from slice dictionary."""
    w, h = slices.values()[0].shape
    dtype = slices.values()[0].dtype
    shape = (len(positions), w, h, len(bvalues), len(echotimes))
    image = np.empty(shape, dtype=dtype)
    image.fill(np.nan)
    for key, value in slices.iteritems():
        pos = positions.index(key[0])
        bv = bvalues.index(key[1])
        et = echotimes.index(key[2])
        image[pos, :, :, bv, et] = value
    if image.shape[3] == 1:
        image = image.squeeze(axis=3)
    elif image.shape[4] == 1:
        image = image.squeeze(axis=4)
    assert image.ndim == 4, image.shape
    if np.isnan(np.min(image)):
        raise Exception('Slices missing from shape {:s}.'.format(shape))
    return image
{'fraction_non_alphanumeric': 0.12922594961493278, 'fraction_numerical': 0.004046469129356481, 'mean_word_length': 3.273284997211378, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '606271', 'n_tokens_mistral': 2506, 'n_tokens_neox': 2423, 'n_words': 665}