<commit_before># IDLSave - a python module to read IDL 'save' files
# Copyright (c) 2010 <NAME>

# Many thanks to <NAME> for publishing the Unofficial Format
# Specification for IDL .sav files, without which this Python module would not
# exist (http://cow.physics.wisc.edu/~craigm/idl/savefmt).

# This code was developed by with permission from ITT Visual Information
# Systems. IDL(r) is a registered trademark of ITT Visual Information Systems,
# Inc. for their Interactive Data Language software.

# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.

import struct
import numpy as np
from numpy.compat import asbytes, asstr
import tempfile
import zlib
import warnings

# Define the different data types that can be found in an IDL save file
DTYPE_DICT = {}
DTYPE_DICT[1] = '>u1'
DTYPE_DICT[2] = '>i2'
DTYPE_DICT[3] = '>i4'
DTYPE_DICT[4] = '>f4'
DTYPE_DICT[5] = '>f8'
DTYPE_DICT[6] = '>c8'
DTYPE_DICT[7] = '|O'
DTYPE_DICT[8] = '|O'
DTYPE_DICT[9] = '>c16'
DTYPE_DICT[12] = '>u2'
DTYPE_DICT[13] = '>u4'
DTYPE_DICT[14] = '>i8'
DTYPE_DICT[15] = '>u8'

# Define the different record types that can be found in an IDL save file
RECTYPE_DICT = {}
RECTYPE_DICT[0] = "START_MARKER"
RECTYPE_DICT[1] = "COMMON_VARIABLE"
RECTYPE_DICT[2] = "VARIABLE"
RECTYPE_DICT[3] = "SYSTEM_VARIABLE"
RECTYPE_DICT[6] = "END_MARKER"
RECTYPE_DICT[10] = "TIMESTAMP"
RECTYPE_DICT[12] = "COMPILED"
RECTYPE_DICT[13] = "IDENTIFICATION"
RECTYPE_DICT[14] = "VERSION"
RECTYPE_DICT[15] = "HEAP_HEADER"
RECTYPE_DICT[16] = "HEAP_DATA"
RECTYPE_DICT[17] = "PROMOTE64"
RECTYPE_DICT[19] = "NOTICE"

# Define a dictionary to contain structure definitions
STRUCT_DICT = {}


def _align_32(f):
    '''Align to the next 32-bit position in a file'''

    pos = f.tell()
    if pos % 4 != 0:
        f.seek(pos + 4 - pos % 4)
    return


def _skip_bytes(f, n):
    '''Skip `n` bytes'''
    f.read(n)
    return


def _read_bytes(f, n):
    '''Read the next `n` bytes'''
    return f.read(n)


def _read_byte(f):
    '''Read a single byte'''
    return np.uint8(struct.unpack('>B', f.read(4)[:1])[0])


def _read_long(f):
    '''Read a signed 32-bit integer'''
    return np.int32(struct.unpack('>l', f.read(4))[0])


def _read_int16(f):
    '''Read a signed 16-bit integer'''
    return np.int16(struct.unpack('>h', f.read(4)[2:4])[0])


def _read_int32(f):
    '''Read a signed 32-bit integer'''
    return np.int32(struct.unpack('>i', f.read(4))[0])


def _read_int64(f):
    '''Read a signed 64-bit integer'''
    return np.int64(struct.unpack('>q', f.read(8))[0])


def _read_uint16(f):
    '''Read an unsigned 16-bit integer'''
    return np.uint16(struct.unpack('>H', f.read(4)[2:4])[0])


def _read_uint32(f):
    '''Read an unsigned 32-bit integer'''
    return np.uint32(struct.unpack('>I', f.read(4))[0])


def _read_uint64(f):
    '''Read an unsigned 64-bit integer'''
    return np.uint64(struct.unpack('>Q', f.read(8))[0])


def _read_float32(f):
    '''Read a 32-bit float'''
    return np.float32(struct.unpack('>f', f.read(4))[0])


def _read_float64(f):
    '''Read a 64-bit float'''
    return np.float64(struct.unpack('>d', f.read(8))[0])


class Pointer(object):
    '''Class used to define pointers'''

    def __init__(self, index):
        self.index = index
        return


def _read_string(f):
    '''Read a string'''
    length = _read_long(f)
    if length > 0:
        chars = _read_bytes(f, length)
        _align_32(f)
        chars = asstr(chars)
    else:
        chars = None
    return chars


def _read_string_data(f):
    '''Read a data string (length is specified twice)'''
    length = _read_long(f)
    if length > 0:
        length = _read_long(f)
        string = _read_bytes(f, length)
        _align_32(f)
    else:
        string = None
    return string


def _read_data(f, dtype):
    '''Read a variable with a specified data type'''
    if dtype==1:
        if _read_int32(f) != 1:
            raise Exception("Error occurred while reading byte variable")
        return _read_byte(f)
    elif dtype==2:
        return _read_int16(f)
    elif dtype==3:
        return _read_int32(f)
    elif dtype==4:
        return _read_float32(f)
    elif dtype==5:
        return _read_float64(f)
    elif dtype==6:
        real = _read_float32(f)
        imag = _read_float32(f)
        return np.complex64(real + imag * 1j)
    elif dtype==7:
        return _read_string_data(f)
    elif dtype==8:
        raise Exception("Should not be here - please report this")
    elif dtype==9:
        real = _read_float64(f)
        imag = _read_float64(f)
        return np.complex128(real + imag * 1j)<commit_msg>If string has zero length, return '' not None<commit_after># IDLSave - a python module to read IDL 'save' files
# Copyright (c) 2010 <NAME>

# Many thanks to <NAME> for publishing the Unofficial Format
# Specification for IDL .sav files, without which this Python module would not
# exist (http://cow.physics.wisc.edu/~craigm/idl/savefmt).

# This code was developed by with permission from ITT Visual Information
# Systems. IDL(r) is a registered trademark of ITT Visual Information Systems,
# Inc. for their Interactive Data Language software.

# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.

import struct
import numpy as np
from numpy.compat import asbytes, asstr
import tempfile
import zlib
import warnings

# Define the different data types that can be found in an IDL save file
DTYPE_DICT = {}
DTYPE_DICT[1] = '>u1'
DTYPE_DICT[2] = '>i2'
DTYPE_DICT[3] = '>i4'
DTYPE_DICT[4] = '>f4'
DTYPE_DICT[5] = '>f8'
DTYPE_DICT[6] = '>c8'
DTYPE_DICT[7] = '|O'
DTYPE_DICT[8] = '|O'
DTYPE_DICT[9] = '>c16'
DTYPE_DICT[12] = '>u2'
DTYPE_DICT[13] = '>u4'
DTYPE_DICT[14] = '>i8'
DTYPE_DICT[15] = '>u8'

# Define the different record types that can be found in an IDL save file
RECTYPE_DICT = {}
RECTYPE_DICT[0] = "START_MARKER"
RECTYPE_DICT[1] = "COMMON_VARIABLE"
RECTYPE_DICT[2] = "VARIABLE"
RECTYPE_DICT[3] = "SYSTEM_VARIABLE"
RECTYPE_DICT[6] = "END_MARKER"
RECTYPE_DICT[10] = "TIMESTAMP"
RECTYPE_DICT[12] = "COMPILED"
RECTYPE_DICT[13] = "IDENTIFICATION"
RECTYPE_DICT[14] = "VERSION"
RECTYPE_DICT[15] = "HEAP_HEADER"
RECTYPE_DICT[16] = "HEAP_DATA"
RECTYPE_DICT[17] = "PROMOTE64"
RECTYPE_DICT[19] = "NOTICE"

# Define a dictionary to contain structure definitions
STRUCT_DICT = {}


def _align_32(f):
    '''Align to the next 32-bit position in a file'''

    pos = f.tell()
    if pos % 4 != 0:
        f.seek(pos + 4 - pos % 4)
    return


def _skip_bytes(f, n):
    '''Skip `n` bytes'''
    f.read(n)
    return


def _read_bytes(f, n):
    '''Read the next `n` bytes'''
    return f.read(n)


def _read_byte(f):
    '''Read a single byte'''
    return np.uint8(struct.unpack('>B', f.read(4)[:1])[0])


def _read_long(f):
    '''Read a signed 32-bit integer'''
    return np.int32(struct.unpack('>l', f.read(4))[0])


def _read_int16(f):
    '''Read a signed 16-bit integer'''
    return np.int16(struct.unpack('>h', f.read(4)[2:4])[0])


def _read_int32(f):
    '''Read a signed 32-bit integer'''
    return np.int32(struct.unpack('>i', f.read(4))[0])


def _read_int64(f):
    '''Read a signed 64-bit integer'''
    return np.int64(struct.unpack('>q', f.read(8))[0])


def _read_uint16(f):
    '''Read an unsigned 16-bit integer'''
    return np.uint16(struct.unpack('>H', f.read(4)[2:4])[0])


def _read_uint32(f):
    '''Read an unsigned 32-bit integer'''
    return np.uint32(struct.unpack('>I', f.read(4))[0])


def _read_uint64(f):
    '''Read an unsigned 64-bit integer'''
    return np.uint64(struct.unpack('>Q', f.read(8))[0])


def _read_float32(f):
    '''Read a 32-bit float'''
    return np.float32(struct.unpack('>f', f.read(4))[0])


def _read_float64(f):
    '''Read a 64-bit float'''
    return np.float64(struct.unpack('>d', f.read(8))[0])


class Pointer(object):
    '''Class used to define pointers'''

    def __init__(self, index):
        self.index = index
        return


def _read_string(f):
    '''Read a string'''
    length = _read_long(f)
    if length > 0:
        chars = _read_bytes(f, length)
        _align_32(f)
        chars = asstr(chars)
    else:
        chars = ''
    return chars


def _read_string_data(f):
    '''Read a data string (length is specified twice)'''
    length = _read_long(f)
    if length > 0:
        length = _read_long(f)
        string = _read_bytes(f, length)
        _align_32(f)
    else:
        string = ''
    return string


def _read_data(f, dtype):
    '''Read a variable with a specified data type'''
    if dtype==1:
        if _read_int32(f) != 1:
            raise Exception("Error occurred while reading byte variable")
        return _read_byte(f)
    elif dtype==2:
        return _read_int16(f)
    elif dtype==3:
        return _read_int32(f)
    elif dtype==4:
        return _read_float32(f)
    elif dtype==5:
        return _read_float64(f)
    elif dtype==6:
        real = _read_float32(f)
        imag = _read_float32(f)
        return np.complex64(real + imag * 1j)
    elif dtype==7:
        return _read_string_data(f)
    elif dtype==8:
        raise Exception("Should not be here - please report this")
    elif dtype==9:
        real = _read_float64(f)
        imag = _read_float64(f)
        return np.complex128(real + imag * 1j)
{'fraction_non_alphanumeric': 0.12709298183113643, 'fraction_numerical': 0.03259707873174207, 'mean_word_length': 3.5851367905267457, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5167830', 'n_tokens_mistral': 4144, 'n_tokens_neox': 3890, 'n_words': 1218}