<commit_before>"""
Reads data from FPT (memo) files.

FPT files are used to varying lenght text or binary data which is too
large to fit in a DBF field.
"""

from collections import namedtuple
from .struct_parser import StructParser


Header = StructParser(
    'FPTHeader',
    '>LHH504s',
    ['nextblock',
     'reserved1',
     'blocksize',
     'reserved2'])

BlockHeader = StructParser(
    'FPTBlock',
    '>LL',
    ['type',
     'length'])

# Record type
record_types = {
    0x0 : 'picture',
    0x1 : 'memo',
    0x2 : 'object',
}

Record = namedtuple('Record', ['type', 'data'])


class FPT:
    """
    This class implement read access to a FPT files.

    FPT files are used to store varying-length data (strings or
    binary) that is too large to fit in a DBF field.

    
    Documentation of the FPT file format:
    http://www.clicketyclick.dk/databases/xbase/format/fpt.html
    """

    def __init__(self, filename):
        self.filename = filename
        self.file = open(filename, 'rb')
        self.header = Header.read(self.file)

    def __getitem__(self, index):
        """Get a memo from the file.
        
        Returns a Record with attributes.
        Memos are returned as byte strings.
        """

        # Todo: Handle reading block header in middle of a memo?
        # Todo: Handle n > end of file and n < 0 and n inside header
        # Todo: Handle wrong sized memo

        if index == 0:
            return Record(type='memo', data=b'')

        self.file.seek(index * self.header.blocksize)<commit_msg>Memo file now raises IndexError if index <= 0.<commit_after>"""
Reads data from FPT (memo) files.

FPT files are used to varying lenght text or binary data which is too
large to fit in a DBF field.
"""

from collections import namedtuple
from .struct_parser import StructParser


Header = StructParser(
    'FPTHeader',
    '>LHH504s',
    ['nextblock',
     'reserved1',
     'blocksize',
     'reserved2'])

BlockHeader = StructParser(
    'FPTBlock',
    '>LL',
    ['type',
     'length'])

# Record type
record_types = {
    0x0: 'picture',
    0x1: 'memo',
    0x2: 'object',
}

Record = namedtuple('Record', ['type', 'data'])


class FPT:
    """
    This class implement read access to a FPT files.

    FPT files are used to store varying-length data (strings or
    binary) that is too large to fit in a DBF field.

    
    Documentation of the FPT file format:
    http://www.clicketyclick.dk/databases/xbase/format/fpt.html
    """

    def __init__(self, filename):
        self.filename = filename
        self.file = open(filename, 'rb')
        self.header = Header.read(self.file)

    def __getitem__(self, index):
        """Get a memo from the file.
        
        Returns a Record with attributes.
        Memos are returned as byte strings.
        """

        # Todo: Handle reading block header in middle of a memo?
        # Todo: Handle n > end of file and n < 0 and n inside header
        # Todo: Handle wrong sized memo

        if index <= 0:
            raise IndexError('memo file got index {}'.format(index))

        self.file.seek(index * self.header.blocksize)
{'fraction_non_alphanumeric': 0.10634920634920635, 'fraction_numerical': 0.008571428571428572, 'mean_word_length': 2.474090407938258, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5472454', 'n_tokens_mistral': 988, 'n_tokens_neox': 949, 'n_words': 345}