<commit_before>import os
import struct

tags = {
    'image_width': 0x0100,
    'image_length': 0x0101,
    'bits_per_sample': 0x0102,
    'compression': 0x0103,
    'make': 0x010f,
    'model': 0x0110,
    'strip_offset': 0x0111,
    'orientation': 0x0112,
    'strip_byte_counts': 0x0117,
    'x_resolution': 0x011a,
    'y_resolution': 0x011b,
    'resolution_unit': 0x0128,
    'datetime': 0x0132,
    'exif': 0x8769,
    'gps_data': 0x8825
}


class Cr2(object):

    class Cr2Header(object):

        def __init__(self, header_buffer):
            (endianness,) = struct.unpack_from('H', header_buffer)
            if endianness == 0x4949:
                # Intel
                self.endian_flag = '<'
            elif endianness == 0x4D4D:
                # Motorola
                self.endian_flag = '>'
            else:
                # WTF (use native)?
                self.endian_flag = '@'
            raw_header = struct.unpack_from(self.endian_flag + 'HHLHBBL',
                                            header_buffer)
            self.raw_header = raw_header
            self.tiff_magic_word = raw_header[1]
            self.tiff_offset = raw_header[2]
            self.magic_word = raw_header[3]
            self.major_version = raw_header[4]
            self.minor_version = raw_header[5]
            self.raw_ifd_offset = raw_header[6]

    class IfdEntry(object):

        def __init__(self, raw_idf_entry):
            (self.tag_id, self.tag_type,
             self.num_value, self.value) = raw_idf_entry

    def __init__(self, file_path):
        self.file_path = file_path
        self.fhandle = open(file_path, "rb")
        # Reasonably sure the header is always 16 bytes... not sure.
        self.header = self.Cr2Header(self.fhandle.read(16))
        # Number of entries in IFD0
        # TODO: Factor out into IDF class
        # TODO: Seek and read the correct ammount.
        # self.fhandle.seek(0)
        #buf = self.fhandle.read(1024)
        #(num_entries,) = struct.unpack_from(self.header.endian_flag + 'H', buf)
        #self.num_entries = num_entries

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        self.fhandle.close()

    def get_header(self):
        return self.header

    def get_idf_entry(self, entry_num):
        # TODO: Figure out how much to read... 1024 is not enough.
        buf = self.fhandle.read(1024)
        return self.IdfEntry(struct.unpack_from(
            self.header.endian_flag + 'HHLL', buf,
            self.header.ifd_offset + 2 + entry_num * 12))

    def find_idf_entry(self, name):

        for entry_num in range(0, num_entries):
            idf_entry = self.get_idf_entry(entry_num)

            if idf_entry.tag_id == tags[name]:
                return idf_entry
<commit_msg>Use correct offsets for IDF0<commit_after>import os
import struct

tags = {
    'image_width': 0x0100,
    'image_length': 0x0101,
    'bits_per_sample': 0x0102,
    'compression': 0x0103,
    'make': 0x010f,
    'model': 0x0110,
    'strip_offset': 0x0111,
    'orientation': 0x0112,
    'strip_byte_counts': 0x0117,
    'x_resolution': 0x011a,
    'y_resolution': 0x011b,
    'resolution_unit': 0x0128,
    'datetime': 0x0132,
    'exif': 0x8769,
    'gps_data': 0x8825
}


# Format info: http://lclevy.free.fr/cr2/
class Cr2(object):

    class Cr2Header(object):

        def __init__(self, header_buffer):
            (endianness,) = struct.unpack_from('H', header_buffer)
            if endianness == 0x4949:
                # Intel
                self.endian_flag = '<'
            elif endianness == 0x4D4D:
                # Motorola
                self.endian_flag = '>'
            else:
                # WTF (use native)?
                self.endian_flag = '@'
            raw_header = struct.unpack_from(self.endian_flag + 'HHLHBBL',
                                            header_buffer)
            self.raw_header = raw_header
            self.tiff_magic_word = raw_header[1]
            self.tiff_offset = raw_header[2]
            self.magic_word = raw_header[3]
            self.major_version = raw_header[4]
            self.minor_version = raw_header[5]
            self.raw_ifd_offset = raw_header[6]

    class IfdEntry(object):

        def __init__(self, raw_idf_entry):
            (self.tag_id, self.tag_type,
             self.num_value, self.value) = raw_idf_entry

    def __init__(self, file_path):
        self.file_path = file_path
        self.fhandle = open(file_path, "rb")
        # Reasonably sure the header is always 16 bytes... not sure.
        self.header = self.Cr2Header(self.fhandle.read(16))

        # Number of entries in IFD0
        self.fhandle.seek(16)
        buf = self.fhandle.read(2)
        (num_entries,) = struct.unpack_from(self.header.endian_flag + 'H', buf)
        self.num_entries = num_entries

    def __enter__(self):
        return self

    def __exit__(self, type, value, traceback):
        self.fhandle.close()

    def get_endianness(self):
        return self.header.endianness

    def get_header(self):
        return self.header

    def get_idf_entry(self, entry_num):
        # TODO: Figure out how much to read... 1024 is not enough.
        buf = self.fhandle.read(1024)
        return self.IdfEntry(struct.unpack_from(
            self.header.endian_flag + 'HHLL', buf,
            self.header.ifd_offset + 2 + entry_num * 12))

    def find_idf_entry(self, name):

        for entry_num in range(0, num_entries):
            idf_entry = self.get_idf_entry(entry_num)

            if idf_entry.tag_id == tags[name]:
                return idf_entry

{'fraction_non_alphanumeric': 0.1049371792603079, 'fraction_numerical': 0.03928508228632101, 'mean_word_length': 2.0452586206896552, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7422275', 'n_tokens_mistral': 2047, 'n_tokens_neox': 1850, 'n_words': 376}