<commit_before># ParseDM3File reads in a DM3 file and translates it into a dictionary
# this module treats that dictionary as an image-file and extracts the
# appropriate image data as numpy arrays.
# It also tries to create files from numpy arrays that DM can read.
#
# Some notes:
# Only complex64 and complex128 types are converted to structarrays,
# ie they're arrays of structs. Everything else, (including RGB) are
# standard arrays.
# There is a seperate DatatType and PixelDepth stored for images different
# from the tag file datatype. I think these are used more than the tag
# datratypes in describing the data.
from parse_dm3 import *
import numpy as np

structarray_to_np_map = {
    ('d', 'd'): np.complex128,
    ('f', 'f'): np.complex64}

np_to_structarray_map = {v: k for k, v in structarray_to_np_map.iteritems()}

# we want to amp any image type to a single np array type
# but a sinlge np array type could map to more than one dm type.
# For the moment, we won't be strict about, eg, discriminating
# int8 from bool, or even unit32 from RGB. In the future we could
# convert np bool type eg to DM bool and treat y,x,3 int8 images
# as RGB.

# note uint8 here returns the same data type as int8 - could be that the
# only way they're differentiated is via this type, not the raw type
# in the tag file? Might be same for PackedComplex images too
# And 8 is missing!
dm_image_dtypes = {
    1: ("int16", np.int16),
    2: ("float32", np.float32),
    3: ("Complex64", np.complex64),
    6: ("uint8", np.int8),
    7: ("int32", np.int32),
    9: ("int8", np.int8),
    10: ("uint16", np.uint16),
    11: ("uint32", np.uint32),
    12: ("float64", np.float64),
    13: ("Complex128", np.complex128),
    14: ("Bool", np.int8),
    23: ("RGB", np.int32),
    27: ("PackedComplex64", np.float32),
    28: ("PackedComplex128", np.float64)
}


def imagedatadict_to_ndarray(imdict):
    """
    Converts the ImageData dictionary, imdict, to an nd image.
    """
    arr = imdict['Data']
    im = None
    if isinstance(arr, array.array):
        im = np.asarray(arr, dtype=arr.typecode)
    elif isinstance(arr, structarray):
        t = tuple(arr.typecodes)
        im = np.frombuffer(
            arr.raw_data,
            dtype=structarray_to_np_map[t])
    print "Image has dmimagetype", imdict["DataType"], "numpy type is", im.dtype
    assert dm_image_dtypes[imdict["DataType"]][1] == im.dtype
    assert imdict['PixelDepth'] == im.dtype.itemsize
    return im.reshape(imdict['Dimensions'][::-1])


def ndarray_to_imagedatadict(nparr):
    """
    Convert the numpy array nparr into a suitable ImageList entry dictionary.
    Returns a dictionary with the appropriate Data, DataType, PixelDepth
    to be inserted into a dm3 tag dictionary and written to a file.
    """
    ret = {}
    dm_type = (k for k, v in dm_image_dtypes.iteritems() if v[1] == nparr.dtype.type).next()
    ret["DataType"] = dm_type
    ret["PixelDepth"] = nparr.dtype.itemsize
    ret["Dimensions"] = list(nparr.shape[::-1])
    if nparr.dtype.type in np_to_structarray_map:<commit_msg>Commented out print statement<commit_after># ParseDM3File reads in a DM3 file and translates it into a dictionary
# this module treats that dictionary as an image-file and extracts the
# appropriate image data as numpy arrays.
# It also tries to create files from numpy arrays that DM can read.
#
# Some notes:
# Only complex64 and complex128 types are converted to structarrays,
# ie they're arrays of structs. Everything else, (including RGB) are
# standard arrays.
# There is a seperate DatatType and PixelDepth stored for images different
# from the tag file datatype. I think these are used more than the tag
# datratypes in describing the data.
from parse_dm3 import *
import numpy as np

structarray_to_np_map = {
    ('d', 'd'): np.complex128,
    ('f', 'f'): np.complex64}

np_to_structarray_map = {v: k for k, v in structarray_to_np_map.iteritems()}

# we want to amp any image type to a single np array type
# but a sinlge np array type could map to more than one dm type.
# For the moment, we won't be strict about, eg, discriminating
# int8 from bool, or even unit32 from RGB. In the future we could
# convert np bool type eg to DM bool and treat y,x,3 int8 images
# as RGB.

# note uint8 here returns the same data type as int8 - could be that the
# only way they're differentiated is via this type, not the raw type
# in the tag file? Might be same for PackedComplex images too
# And 8 is missing!
dm_image_dtypes = {
    1: ("int16", np.int16),
    2: ("float32", np.float32),
    3: ("Complex64", np.complex64),
    6: ("uint8", np.int8),
    7: ("int32", np.int32),
    9: ("int8", np.int8),
    10: ("uint16", np.uint16),
    11: ("uint32", np.uint32),
    12: ("float64", np.float64),
    13: ("Complex128", np.complex128),
    14: ("Bool", np.int8),
    23: ("RGB", np.int32),
    27: ("PackedComplex64", np.float32),
    28: ("PackedComplex128", np.float64)
}


def imagedatadict_to_ndarray(imdict):
    """
    Converts the ImageData dictionary, imdict, to an nd image.
    """
    arr = imdict['Data']
    im = None
    if isinstance(arr, array.array):
        im = np.asarray(arr, dtype=arr.typecode)
    elif isinstance(arr, structarray):
        t = tuple(arr.typecodes)
        im = np.frombuffer(
            arr.raw_data,
            dtype=structarray_to_np_map[t])
    # print "Image has dmimagetype", imdict["DataType"], "numpy type is", im.dtype
    assert dm_image_dtypes[imdict["DataType"]][1] == im.dtype
    assert imdict['PixelDepth'] == im.dtype.itemsize
    return im.reshape(imdict['Dimensions'][::-1])


def ndarray_to_imagedatadict(nparr):
    """
    Convert the numpy array nparr into a suitable ImageList entry dictionary.
    Returns a dictionary with the appropriate Data, DataType, PixelDepth
    to be inserted into a dm3 tag dictionary and written to a file.
    """
    ret = {}
    dm_type = (k for k, v in dm_image_dtypes.iteritems() if v[1] == nparr.dtype.type).next()
    ret["DataType"] = dm_type
    ret["PixelDepth"] = nparr.dtype.itemsize
    ret["Dimensions"] = list(nparr.shape[::-1])
    if nparr.dtype.type in np_to_structarray_map:
{'fraction_non_alphanumeric': 0.11289277615808228, 'fraction_numerical': 0.031746031746031744, 'mean_word_length': 3.5437821927888153, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4018645', 'n_tokens_mistral': 2108, 'n_tokens_neox': 2003, 'n_words': 742}