<commit_before>import os, sys
import numpy as np
import struct


def read_image_sequence(data_dir, prefix, num_images, start_index=0, image_format='png', zero_padding=0, crop=None, verbose=False):
    '''Read a series of images into a list of numpy arrays.
    
    :param str data_dir: directory where the image files are located.
    :param str prefix: a string to construct the image file names.
    :param int num_images: the number of images to read.
    :param int start_index: the index to start loading the images (0 by default).
    :param str image_format: can be tif or png (png by default).
    :param int zero_padding: number of zero to use in zero padding (0 by default).
    :param list crop: bounds to crop the images (None by default) 
    :param bool verbose: activate verbose mode (False by default).
    :return: the list of the images read from the disk.
    '''
    # build the numbering pattern
    pat = '0%dd' % zero_padding
    fmt = '{0:s}{1:' + pat + '}.{2:s}'
    image_stack = []
    if image_format == 'tif':
        from pymicro.external.tifffile import TiffFile
    elif image_format == 'png':
        from matplotlib import pyplot as plt
    for i in range(start_index, start_index + num_images):
        image_path = os.path.join(data_dir, fmt.format(prefix, i, image_format))
        if verbose:
            print('loading image %s' % image_path)
        if image_format == 'tif':
            im = TiffFile(image_path).asarray()
        elif image_format == 'png':
            im = (np.mean(plt.imread(image_path)[:, :, :3], axis=2) * 255).astype(np.uint8)
        if crop:
            im = im[crop[2]:crop[3], crop[0]:crop[1]]
        image_stack.append(im.T)
    return image_stack


def unpack_header(header):
    """Unpack an ascii header.

    Form a string with the read binary data and then split it into string
    tokens which are put in a dictionnary.

    :param str header: the ascii header to unpack.
    :return: a dictionnary with the (key, value) fields contained in the header.
    """
    header_values = {}
    for line in header.split('\n'):
        tokens = line.split('=')
        if len(tokens) > 1:
            header_values[tokens[0].strip()] = tokens[1].split(';')[0].strip()
    return header_values


def edf_info(file_name, header_size=None):
    """Read and return informations contained in the header of a .edf file.

    Edf files always start with a header (of variable length) containing
    informations about the file such as acquisition conditions, image
    dimensions... This function reads a certain amount of bytes of a given
    file as ascii data and unpack it.
    If not specified, the header size is determined automatically by
    substracting the data size (read as ascii at the begining of the file)
    to the total file size.

    :param str file_name: the name of the edf file to read.
    :param int header_size: number of bytes to read as a multiple of 512 (None by default).
    :return: a dictionnary containing the file informations.
    """
    try:
        f = open(file_name, 'r', encoding='latin-1')
    except TypeError:
        f = open(file_name, 'r')  # fall back
    if header_size is None:
        # guess the header size by peeking at the first chunk of 512 bytes
        header_values = unpack_header(f.read(512))
        total_file_size = os.path.getsize(file_name)
        payload_size = int(header_values['Size'].split('.')[0])
        header_size = total_file_size - payload_size
        print('determined header size is %d bytes' % header_size)
        f.seek(0)
    header = f.read(header_size)
    f.close()
    return unpack_header(header)


def edf_read(file_name):
    """Read an edf file.

    edf stands for ESRF data file. It has a variable header size which is
    a multiple of 512 bytes and contains the image meta in ASCII format
    (eg. image size, data type, motor positions).

    The ascii header is parsed automatically by `edf_info` to retreive the
    image size and data type. Depending on the information enclosed in the
    header, this function may return a 1d, 2d or 3d array.
    ::

      >>> im = edf_read('radio.edf')
      >>> im.shape
      (2048, 2048)

    :param str file_name: the name of the edf file to read.
    :return: a numpy array containing the data
    """
    header_values = edf_info(file_name)
    f = open(file_name, 'r')
    data_type = esrf_to_numpy_datatype(header_values['DataType'])
    print(header_values['DataType'], data_type)
    # get the payload size
    payload_size = int(header_values['Size'].split('.')[0])
    # get the image size from the ascii header
    dim_1 = int(header_values['Dim_1'].split('.')[0])
    try:
        dim_2 = int(header_values['Dim_2'].split('.')[0])
    except:
        print('Dim_2 not defined in header')
        dim_2 = None
    try:
        dim_3 = int(header_values['Dim_3'].split('.')[0])
    except:
        print('Dim_3 not defined in header')
        dim_3 = None
    # now read binary data
    header_size = os.path.getsize(file_name) - payload_size
    f.seek(header_size)
    payload = np.fromfile(f, dtype=data_type)
    if dim_1 and dim_2 and dim_3:
        data = np.reshape(payload, (dim_3, dim_2, dim_1)).transpose(2, 1, 0)
    elif dim_1 and dim_2:
        data = np.reshape(payload, (dim_2, dim_1)).transpose(1, 0)
    else:
        data = np.reshape(payload, (dim_1))
    f.close()
    # pay attention to byte order
    if header_values['ByteOrder'] == 'HighByteFirst':
        data = data.byteswap()
    return data


def esrf_to_numpy_datatype(data_type):
    return {
        'UnsignedByte': np.uint8,
        'UnsignedShort': np.uint16,
        'UnsignedLong': np.uint32,<commit_msg>add verbose mode to edf_read and edf_info<commit_after>import os, sys
import numpy as np
import struct


def read_image_sequence(data_dir, prefix, num_images, start_index=0, image_format='png', zero_padding=0, crop=None, verbose=False):
    '''Read a series of images into a list of numpy arrays.
    
    :param str data_dir: directory where the image files are located.
    :param str prefix: a string to construct the image file names.
    :param int num_images: the number of images to read.
    :param int start_index: the index to start loading the images (0 by default).
    :param str image_format: can be tif or png (png by default).
    :param int zero_padding: number of zero to use in zero padding (0 by default).
    :param list crop: bounds to crop the images (None by default) 
    :param bool verbose: activate verbose mode (False by default).
    :return: the list of the images read from the disk.
    '''
    # build the numbering pattern
    pat = '0%dd' % zero_padding
    fmt = '{0:s}{1:' + pat + '}.{2:s}'
    image_stack = []
    if image_format == 'tif':
        from pymicro.external.tifffile import TiffFile
    elif image_format == 'png':
        from matplotlib import pyplot as plt
    for i in range(start_index, start_index + num_images):
        image_path = os.path.join(data_dir, fmt.format(prefix, i, image_format))
        if verbose:
            print('loading image %s' % image_path)
        if image_format == 'tif':
            im = TiffFile(image_path).asarray()
        elif image_format == 'png':
            im = (np.mean(plt.imread(image_path)[:, :, :3], axis=2) * 255).astype(np.uint8)
        if crop:
            im = im[crop[2]:crop[3], crop[0]:crop[1]]
        image_stack.append(im.T)
    return image_stack


def unpack_header(header):
    """Unpack an ascii header.

    Form a string with the read binary data and then split it into string
    tokens which are put in a dictionnary.

    :param str header: the ascii header to unpack.
    :return: a dictionnary with the (key, value) fields contained in the header.
    """
    header_values = {}
    for line in header.split('\n'):
        tokens = line.split('=')
        if len(tokens) > 1:
            header_values[tokens[0].strip()] = tokens[1].split(';')[0].strip()
    return header_values


def edf_info(file_name, header_size=None, verbose=False):
    """Read and return informations contained in the header of a .edf file.

    Edf files always start with a header (of variable length) containing
    informations about the file such as acquisition conditions, image
    dimensions... This function reads a certain amount of bytes of a given
    file as ascii data and unpack it.
    If not specified, the header size is determined automatically by
    substracting the data size (read as ascii at the begining of the file)
    to the total file size.

    :param str file_name: the name of the edf file to read.
    :param int header_size: number of bytes to read as a multiple of 512 (None by default).
    :param bool verbose: flag to activate verbose mode.
    :return: a dictionnary containing the file informations.
    """
    try:
        f = open(file_name, 'r', encoding='latin-1')
    except TypeError:
        f = open(file_name, 'r')  # fall back
    if header_size is None:
        # guess the header size by peeking at the first chunk of 512 bytes
        header_values = unpack_header(f.read(512))
        total_file_size = os.path.getsize(file_name)
        payload_size = int(header_values['Size'].split('.')[0])
        header_size = total_file_size - payload_size
        if verbose:
            print('determined header size is %d bytes' % header_size)
        f.seek(0)
    header = f.read(header_size)
    f.close()
    return unpack_header(header)


def edf_read(file_name, verbose=False):
    """Read an edf file.

    edf stands for ESRF data file. It has a variable header size which is
    a multiple of 512 bytes and contains the image meta in ASCII format
    (eg. image size, data type, motor positions).

    The ascii header is parsed automatically by `edf_info` to retreive the
    image size and data type. Depending on the information enclosed in the
    header, this function may return a 1d, 2d or 3d array.
    ::

      >>> im = edf_read('radio.edf')
      >>> im.shape
      (2048, 2048)

    :param str file_name: the name of the edf file to read.
    :param bool verbose: flag to activate verbose mode.
    :return: a numpy array containing the data
    """
    header_values = edf_info(file_name, verbose=verbose)
    f = open(file_name, 'r')
    data_type = esrf_to_numpy_datatype(header_values['DataType'])
    if verbose:
        print(header_values['DataType'], data_type)
    # get the payload size
    payload_size = int(header_values['Size'].split('.')[0])
    # get the image size from the ascii header
    dim_1 = int(header_values['Dim_1'].split('.')[0])
    try:
        dim_2 = int(header_values['Dim_2'].split('.')[0])
    except KeyError:
        if verbose:
            print('Dim_2 not defined in header')
        dim_2 = None
    try:
        dim_3 = int(header_values['Dim_3'].split('.')[0])
    except KeyError:
        if verbose:
            print('Dim_3 not defined in header')
        dim_3 = None
    # now read binary data
    header_size = os.path.getsize(file_name) - payload_size
    f.seek(header_size)
    payload = np.fromfile(f, dtype=data_type)
    if dim_1 and dim_2 and dim_3:
        data = np.reshape(payload, (dim_3, dim_2, dim_1)).transpose(2, 1, 0)
    elif dim_1 and dim_2:
        data = np.reshape(payload, (dim_2, dim_1)).transpose(1, 0)
    else:
        data = np.reshape(payload, (dim_1))
    f.close()
    # pay attention to byte order
    if header_values['ByteOrder'] == 'HighByteFirst':
        data = data.byteswap()
    return data


def esrf_to_numpy_datatype(data_type):
    return {
        'UnsignedByte': np.uint8,
        'UnsignedShort': np.uint16,
        'UnsignedLong': np.uint32,
{'fraction_non_alphanumeric': 0.100606371167478, 'fraction_numerical': 0.014177128704415407, 'mean_word_length': 2.900732844770153, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5739327', 'n_tokens_mistral': 3733, 'n_tokens_neox': 3596, 'n_words': 1282}