<commit_before>import io
import struct

from fitparse.processors import FitFileDataProcessor
from fitparse.profile import FIELD_TYPE_TIMESTAMP, MESSAGE_TYPES
from fitparse.records import (
    DataMessage, FieldData, FieldDefinition, DevFieldDefinition, DefinitionMessage, MessageHeader,
    BASE_TYPES, BASE_TYPE_BYTE, DevField,
    add_dev_data_id, add_dev_field_description, get_dev_type
)
from fitparse.utils import calc_crc


class FitParseError(Exception):
    pass


class FitFile(object):
    def __init__(self, fileish, check_crc=True, data_processor=None):
        if hasattr(fileish, 'read'):
            self._file = fileish
        elif isinstance(fileish, bytes) and fileish[8:12] == b'.FIT':
            self._file = io.BytesIO(fileish)
        else:
            self._file = open(fileish, 'rb')

        self.check_crc = check_crc

        self._accumulators = {}
        self._bytes_left = -1  # Not valid until after _parse_file_header()
        self._complete = False
        self._compressed_ts_accumulator = 0
        self._crc = 0
        self._local_mesgs = {}
        self._messages = []
        self._processor = data_processor or FitFileDataProcessor()

        # Start off by parsing the file header (makes self._bytes_left valid)
        self._parse_file_header()

    ##########
    # Private low-level utility methods for reading of fit file

    def _read(self, size):
        if size <= 0:
            return ''
        data = self._file.read(size)
        self._crc = calc_crc(data, self._crc)
        self._bytes_left -= len(data)
        return data

    def _read_struct(self, fmt, endian='<', data=None, always_tuple=False):
        fmt_with_endian = "%s%s" % (endian, fmt)
        size = struct.calcsize(fmt_with_endian)
        if data is None:
            data = self._read(size)

        if size != len(data):
            raise FitParseError("Tried to read %d bytes from .FIT file but got %d" % (size, len(data)))

        unpacked = struct.unpack(fmt_with_endian, data)
        # Flatten tuple if it's got only one value
        return unpacked if (len(unpacked) > 1) or always_tuple else unpacked[0]

    def _read_and_assert_crc(self, allow_zero=False):
        # CRC Calculation is little endian from SDK
        crc_expected, crc_actual = self._crc, self._read_struct('H')

        if (crc_actual != crc_expected) and not (allow_zero and (crc_actual == 0)):
            if self.check_crc:
                raise FitParseError('CRC Mismatch [expected = 0x%04X, actual = 0x%04X]' % (
                    crc_expected, crc_actual))

    ##########
    # Private Data Parsing Methods

    def _parse_file_header(self):
        header_data = self._read(12)
        if header_data[8:12] != b'.FIT':
            raise FitParseError("Invalid .FIT File Header")

        # Larger fields are explicitly little endian from SDK
        header_size, protocol_ver_enc, profile_ver_enc, data_size = self._read_struct('2BHI4x', data=header_data)

        # Decode the same way the SDK does
        self.protocol_version = float("%d.%d" % (protocol_ver_enc >> 4, protocol_ver_enc & ((1 << 4) - 1)))
        self.profile_version = float("%d.%d" % (profile_ver_enc / 100, profile_ver_enc % 100))

        # Consume extra header information
        extra_header_size = header_size - 12
        if extra_header_size > 0:
            # Make sure extra field in header is at least 2 bytes to calculate CRC
            if extra_header_size < 2:
                raise FitParseError('Irregular File Header Size')

            # Consume extra two bytes of header and check CRC
            self._read_and_assert_crc(allow_zero=True)

            # Consume any extra bytes, since header size "may be increased in
            # "future to add additional optional information" (from SDK)
            self._read(extra_header_size - 2)

        # After we've consumed the header, set the bytes left to be read
        self._bytes_left = data_size

    def _parse_message(self):
        # When done, calculate the CRC and return None
        if self._bytes_left <= 0:
            if not self._complete:
                self._read_and_assert_crc()

                if hasattr(self._file, 'close'):
                    self._file.close()
                self._complete = True

            return None

        header = self._parse_message_header()

        if header.is_definition:
            message = self._parse_definition_message(header)
        else:
            message = self._parse_data_message(header)
            if message.mesg_type is not None:
                if message.mesg_type.name == 'developer_data_id':
                    add_dev_data_id(message)
                elif message.mesg_type.name == 'field_description':
                    add_dev_field_description(message)

        self._messages.append(message)
        return message

    def _parse_message_header(self):
        header = self._read_struct('B')

        if header & 0x80:  # bit 7: Is this record a compressed timestamp?
            return MessageHeader(<commit_msg>Make an effort to close the file handle<commit_after>import io
import struct

from fitparse.processors import FitFileDataProcessor
from fitparse.profile import FIELD_TYPE_TIMESTAMP, MESSAGE_TYPES
from fitparse.records import (
    DataMessage, FieldData, FieldDefinition, DevFieldDefinition, DefinitionMessage, MessageHeader,
    BASE_TYPES, BASE_TYPE_BYTE, DevField,
    add_dev_data_id, add_dev_field_description, get_dev_type
)
from fitparse.utils import calc_crc


class FitParseError(Exception):
    pass


class FitFile(object):
    def __init__(self, fileish, check_crc=True, data_processor=None):
        if hasattr(fileish, 'read'):
            self._file = fileish
        elif isinstance(fileish, bytes) and fileish[8:12] == b'.FIT':
            self._file = io.BytesIO(fileish)
        else:
            self._file = open(fileish, 'rb')

        self.check_crc = check_crc

        self._accumulators = {}
        self._bytes_left = -1  # Not valid until after _parse_file_header()
        self._complete = False
        self._compressed_ts_accumulator = 0
        self._crc = 0
        self._local_mesgs = {}
        self._messages = []
        self._processor = data_processor or FitFileDataProcessor()

        # Start off by parsing the file header (makes self._bytes_left valid)
        self._parse_file_header()

    def __del__(self):
        self.close()

    def close(self):
        if self._file and hasattr(self._file, "close"):
            self._file.close()
            self._file = None

    def __enter__(self):
        return self

    def __exit__(self, *_):
        self.close()

    ##########
    # Private low-level utility methods for reading of fit file

    def _read(self, size):
        if size <= 0:
            return ''
        data = self._file.read(size)
        self._crc = calc_crc(data, self._crc)
        self._bytes_left -= len(data)
        return data

    def _read_struct(self, fmt, endian='<', data=None, always_tuple=False):
        fmt_with_endian = "%s%s" % (endian, fmt)
        size = struct.calcsize(fmt_with_endian)
        if data is None:
            data = self._read(size)

        if size != len(data):
            raise FitParseError("Tried to read %d bytes from .FIT file but got %d" % (size, len(data)))

        unpacked = struct.unpack(fmt_with_endian, data)
        # Flatten tuple if it's got only one value
        return unpacked if (len(unpacked) > 1) or always_tuple else unpacked[0]

    def _read_and_assert_crc(self, allow_zero=False):
        # CRC Calculation is little endian from SDK
        crc_expected, crc_actual = self._crc, self._read_struct('H')

        if (crc_actual != crc_expected) and not (allow_zero and (crc_actual == 0)):
            if self.check_crc:
                raise FitParseError('CRC Mismatch [expected = 0x%04X, actual = 0x%04X]' % (
                    crc_expected, crc_actual))

    ##########
    # Private Data Parsing Methods

    def _parse_file_header(self):
        header_data = self._read(12)
        if header_data[8:12] != b'.FIT':
            raise FitParseError("Invalid .FIT File Header")

        # Larger fields are explicitly little endian from SDK
        header_size, protocol_ver_enc, profile_ver_enc, data_size = self._read_struct('2BHI4x', data=header_data)

        # Decode the same way the SDK does
        self.protocol_version = float("%d.%d" % (protocol_ver_enc >> 4, protocol_ver_enc & ((1 << 4) - 1)))
        self.profile_version = float("%d.%d" % (profile_ver_enc / 100, profile_ver_enc % 100))

        # Consume extra header information
        extra_header_size = header_size - 12
        if extra_header_size > 0:
            # Make sure extra field in header is at least 2 bytes to calculate CRC
            if extra_header_size < 2:
                raise FitParseError('Irregular File Header Size')

            # Consume extra two bytes of header and check CRC
            self._read_and_assert_crc(allow_zero=True)

            # Consume any extra bytes, since header size "may be increased in
            # "future to add additional optional information" (from SDK)
            self._read(extra_header_size - 2)

        # After we've consumed the header, set the bytes left to be read
        self._bytes_left = data_size

    def _parse_message(self):
        # When done, calculate the CRC and return None
        if self._bytes_left <= 0:
            if not self._complete:
                self._read_and_assert_crc()
                self._complete = True
                self.close()

            return None

        header = self._parse_message_header()

        if header.is_definition:
            message = self._parse_definition_message(header)
        else:
            message = self._parse_data_message(header)
            if message.mesg_type is not None:
                if message.mesg_type.name == 'developer_data_id':
                    add_dev_data_id(message)
                elif message.mesg_type.name == 'field_description':
                    add_dev_field_description(message)

        self._messages.append(message)
        return message

    def _parse_message_header(self):
        header = self._read_struct('B')

        if header & 0x80:  # bit 7: Is this record a compressed timestamp?
            return MessageHeader(
{'fraction_non_alphanumeric': 0.08850752024681836, 'fraction_numerical': 0.008484381025838797, 'mean_word_length': 2.3821323769155525, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2798707', 'n_tokens_mistral': 3167, 'n_tokens_neox': 2965, 'n_words': 825}