<commit_before>import bz2
import os
import zlib
from collections import OrderedDict
from datetime import datetime

from nrrd.errors import NRRDError
from nrrd.formatters import *
from nrrd.reader import _get_field_type

# Older versions of Python had issues when uncompressed data was larger than 4GB (2^32). This should be fixed in latest
# version of Python 2.7 and all versions of Python 3. The fix for this issue is to read the data in smaller chunks. The
# chunk size is set to be small here at 1MB since performance did not vary much based on the chunk size. A smaller chunk
# size has the benefit of using less RAM at once.
_WRITE_CHUNKSIZE = 2 ** 20

_NRRD_FIELD_ORDER = [
    'type',
    'dimension',
    'space dimension',
    'space',
    'sizes',
    'space directions',
    'kinds',
    'endian',
    'encoding',
    'min',
    'max',
    'oldmin',
    'old min',
    'oldmax',
    'old max',
    'content',
    'sample units',
    'spacings',
    'thicknesses',
    'axis mins',
    'axismins',
    'axis maxs',
    'axismaxs',
    'centerings',
    'labels',
    'units',
    'space units',
    'space origin',
    'measurement frame',
    'data file']

_TYPEMAP_NUMPY2NRRD = {
    'i1': 'int8',
    'u1': 'uint8',
    'i2': 'int16',
    'u2': 'uint16',
    'i4': 'int32',
    'u4': 'uint32',
    'i8': 'int64',
    'u8': 'uint64',
    'f4': 'float',
    'f8': 'double',
    'V': 'block'
}

_NUMPY2NRRD_ENDIAN_MAP = {
    '<': 'little',
    'L': 'little',
    '>': 'big',
    'B': 'big'
}


def _format_field_value(value, field_type):
    if field_type == 'int':
        return format_number(value)
    elif field_type == 'double':
        return format_number(value)
    elif field_type == 'string':
        return str(value)
    elif field_type == 'int list':
        return format_number_list(value)
    elif field_type == 'double list':
        return format_number_list(value)
    elif field_type == 'string list':
        # TODO Handle cases where the user wants quotation marks around the items
        return ' '.join(value)
    elif field_type == 'int vector':
        return format_vector(value)
    elif field_type == 'double vector':
        return format_optional_vector(value)
    elif field_type == 'int matrix':
        return format_matrix(value)
    elif field_type == 'double matrix':
        return format_optional_matrix(value)
    else:
        raise NRRDError('Invalid field type given: %s' % field_type)


def write(filename, data, header=None, detached_header=False, relative_data_path=True, custom_field_map=None,
          compression_level=9):
    """Write :class:`numpy.ndarray` to NRRD file

    The :obj:`filename` parameter specifies the absolute or relative filename to write the NRRD file to. If the
    :obj:`filename` extension is .nhdr, then the :obj:`detached_header` parameter is set to true automatically. If the
    :obj:`detached_header` parameter is set to :obj:`True` and the :obj:`filename` ends in .nrrd, then the header file
    will have the same path and base name as the :obj:`filename` but with an extension of .nhdr. In all other cases,
    the header and data are saved in the same file.

    :obj:`header` is an optional parameter containing the fields and values to be added to the NRRD header.

    .. note::
            The following fields are automatically generated based on the :obj:`data` parameter ignoring these values
            in the :obj:`header`: 'type', 'endian', 'dimension', 'sizes'. In addition, the generated fields will be
            added to the given :obj:`header`. Thus, one can check the generated fields by viewing the passed
            :obj:`header`.

    .. note::
            The default encoding field used if not specified in :obj:`header` is 'gzip'.

    See :ref:`user-guide:Writing NRRD files` for more information on writing NRRD files.

    Parameters
    ----------
    filename : :class:`str`
        Filename of the NRRD file
    data : :class:`numpy.ndarray`
        Data to save to the NRRD file
    detached_header : :obj:`bool`, optional
        Whether the header and data should be saved in separate files. Defaults to :obj:`False`
    relative_data_path : :class:`bool`
        Whether the data filename in detached header is saved with a relative path or absolute path.
        This parameter is ignored if there is no detached header. Defaults to :obj:`True`
    custom_field_map : :class:`dict` (:class:`str`, :class:`str`), optional
        Dictionary used for parsing custom field types where the key is the custom field name and the value is a
        string identifying datatype for the custom field.
    compression_level : :class:`int`
        Int specifying compression level, when using a compressed encoding (.gz, .bz2).
        - For zlib (.gz): 1-9 set low to high compression; 0 disables; -1 uses zlib default.
        - For bzip2 (.bz2): 1-9 set low to high compression.

    See Also
    --------
    :meth:`read`, :meth:`read_header`, :meth:`read_data`
    """

    if header is None:
        header = {}

    # Infer a number of fields from the NumPy array and overwrite values in the header dictionary.
    # Get type string identifier from the NumPy datatype
    header['type'] = _TYPEMAP_NUMPY2NRRD[data.dtype.str[1:]]

    # If the datatype contains more than one byte and the encoding is not ASCII, then set the endian header value
    # based on the datatype's endianness. Otherwise, delete the endian field from the header if present
    if data.dtype.itemsize > 1 and header.get('encoding', '').lower() not in ['ascii', 'text', 'txt']:
        header['endian'] = _NUMPY2NRRD_ENDIAN_MAP[data.dtype.str[:1]]<commit_msg>Remove documentation regarding compression_level=0 (#91)<commit_after>import bz2
import os
import zlib
from collections import OrderedDict
from datetime import datetime

from nrrd.errors import NRRDError
from nrrd.formatters import *
from nrrd.reader import _get_field_type

# Older versions of Python had issues when uncompressed data was larger than 4GB (2^32). This should be fixed in latest
# version of Python 2.7 and all versions of Python 3. The fix for this issue is to read the data in smaller chunks. The
# chunk size is set to be small here at 1MB since performance did not vary much based on the chunk size. A smaller chunk
# size has the benefit of using less RAM at once.
_WRITE_CHUNKSIZE = 2 ** 20

_NRRD_FIELD_ORDER = [
    'type',
    'dimension',
    'space dimension',
    'space',
    'sizes',
    'space directions',
    'kinds',
    'endian',
    'encoding',
    'min',
    'max',
    'oldmin',
    'old min',
    'oldmax',
    'old max',
    'content',
    'sample units',
    'spacings',
    'thicknesses',
    'axis mins',
    'axismins',
    'axis maxs',
    'axismaxs',
    'centerings',
    'labels',
    'units',
    'space units',
    'space origin',
    'measurement frame',
    'data file']

_TYPEMAP_NUMPY2NRRD = {
    'i1': 'int8',
    'u1': 'uint8',
    'i2': 'int16',
    'u2': 'uint16',
    'i4': 'int32',
    'u4': 'uint32',
    'i8': 'int64',
    'u8': 'uint64',
    'f4': 'float',
    'f8': 'double',
    'V': 'block'
}

_NUMPY2NRRD_ENDIAN_MAP = {
    '<': 'little',
    'L': 'little',
    '>': 'big',
    'B': 'big'
}


def _format_field_value(value, field_type):
    if field_type == 'int':
        return format_number(value)
    elif field_type == 'double':
        return format_number(value)
    elif field_type == 'string':
        return str(value)
    elif field_type == 'int list':
        return format_number_list(value)
    elif field_type == 'double list':
        return format_number_list(value)
    elif field_type == 'string list':
        # TODO Handle cases where the user wants quotation marks around the items
        return ' '.join(value)
    elif field_type == 'int vector':
        return format_vector(value)
    elif field_type == 'double vector':
        return format_optional_vector(value)
    elif field_type == 'int matrix':
        return format_matrix(value)
    elif field_type == 'double matrix':
        return format_optional_matrix(value)
    else:
        raise NRRDError('Invalid field type given: %s' % field_type)


def write(filename, data, header=None, detached_header=False, relative_data_path=True, custom_field_map=None,
          compression_level=9):
    """Write :class:`numpy.ndarray` to NRRD file

    The :obj:`filename` parameter specifies the absolute or relative filename to write the NRRD file to. If the
    :obj:`filename` extension is .nhdr, then the :obj:`detached_header` parameter is set to true automatically. If the
    :obj:`detached_header` parameter is set to :obj:`True` and the :obj:`filename` ends in .nrrd, then the header file
    will have the same path and base name as the :obj:`filename` but with an extension of .nhdr. In all other cases,
    the header and data are saved in the same file.

    :obj:`header` is an optional parameter containing the fields and values to be added to the NRRD header.

    .. note::
            The following fields are automatically generated based on the :obj:`data` parameter ignoring these values
            in the :obj:`header`: 'type', 'endian', 'dimension', 'sizes'. In addition, the generated fields will be
            added to the given :obj:`header`. Thus, one can check the generated fields by viewing the passed
            :obj:`header`.

    .. note::
            The default encoding field used if not specified in :obj:`header` is 'gzip'.

    See :ref:`user-guide:Writing NRRD files` for more information on writing NRRD files.

    Parameters
    ----------
    filename : :class:`str`
        Filename of the NRRD file
    data : :class:`numpy.ndarray`
        Data to save to the NRRD file
    detached_header : :obj:`bool`, optional
        Whether the header and data should be saved in separate files. Defaults to :obj:`False`
    relative_data_path : :class:`bool`
        Whether the data filename in detached header is saved with a relative path or absolute path.
        This parameter is ignored if there is no detached header. Defaults to :obj:`True`
    custom_field_map : :class:`dict` (:class:`str`, :class:`str`), optional
        Dictionary used for parsing custom field types where the key is the custom field name and the value is a
        string identifying datatype for the custom field.
    compression_level : :class:`int`
        Integer between 1 to 9 specifying the compression level when using a compressed encoding (gzip or bzip). A value
        of :obj:`1` compresses the data the least amount and is the fastest, while a value of :obj:`9` compresses the
        data the most and is the slowest.

    See Also
    --------
    :meth:`read`, :meth:`read_header`, :meth:`read_data`
    """

    if header is None:
        header = {}

    # Infer a number of fields from the NumPy array and overwrite values in the header dictionary.
    # Get type string identifier from the NumPy datatype
    header['type'] = _TYPEMAP_NUMPY2NRRD[data.dtype.str[1:]]

    # If the datatype contains more than one byte and the encoding is not ASCII, then set the endian header value
    # based on the datatype's endianness. Otherwise, delete the endian field from the header if present
    if data.dtype.itemsize > 1 and header.get('encoding', '').lower() not in ['ascii', 'text', 'txt']:
        header['endian'] = _NUMPY2NRRD_ENDIAN_MAP[data.dtype.str[:1]]
{'fraction_non_alphanumeric': 0.10354199544099597, 'fraction_numerical': 0.009118008065930211, 'mean_word_length': 3.066666666666667, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4960275', 'n_tokens_mistral': 3459, 'n_tokens_neox': 3335, 'n_words': 1323}