<commit_before>from datetime import datetime
import logging
import os

from ..analysis import file_index
from ..messages import MessageType, MessageHeader, Timestamp, message_type_to_class
from ..utils.time_range import TimeRange


class MixedLogReader(object):
    """!
    @brief Generator class for decoding FusionEngine messages contained in a mixed-content binary file.

    For real-time deserialization of an incoming binary data stream, see @ref FusionEngineDecoder.
    """
    logger = logging.getLogger('point_one.fusion_engine.parsers.mixed_log_reader')

    def __init__(self, input_file, warn_on_gaps: bool = False, show_progress: bool = False,
                 generate_index: bool = True, ignore_index: bool = False,
                 time_range: TimeRange = None, message_types: set = None,
                 return_header: bool = True, return_payload: bool = True,
                 return_bytes: bool = False, return_offset: bool = False):
        """!
        @brief Construct a new generator instance.

        Each call to @ref next() will return a tuple containing any/all of the message header, payload, serialized
        `bytes`, and byte offset depending on the values of the `return_*` parameters.

        @param input_file The path to an input file (`.p1log` or mixed-content binary file), or an open file-like
               object.
        @param warn_on_gaps If `True`, print warnings if gaps are detected in the FusionEngine message sequence numbers.
        @param show_progress If `True`, print file read progress to the console periodically.
        @param generate_index If `True`, generate an index file if one does not exist for faster reading in the future.
               See @ref FileIndex for details.
        @param ignore_index If `True`, ignore the existing index file and read from the binary file directly. If
               `generate_index == True`, this will delete the existing file and create a new one.
        @param return_header If `True`, return the decoded @ref MessageHeader for each message.
        @param return_payload If `True`, parse and return the payload for each message as a subclass of @ref
               MessagePayload. Will return `None` if the payload cannot be parsed.
        @param return_bytes If `True`, return a `bytes` object containing the serialized message header and payload.
        @param return_offset If `True`, return the offset into the file (in bytes) at which the message began.
        """
        self.warn_on_gaps = warn_on_gaps

        self.return_header = return_header
        self.return_payload = return_payload
        self.return_bytes = return_bytes
        self.return_offset = return_offset

        self.time_range = time_range
        self.message_types = message_types

        self.valid_count = 0
        self.message_counts = {}
        self.prev_sequence_number = None
        self.total_bytes_read = 0

        self.show_progress = show_progress
        self.last_print_bytes = 0
        self.start_time = datetime.now()

        # Open the file to be read.
        if isinstance(input_file, str):
            self.input_file = open(input_file, 'rb')
        else:
            self.input_file = input_file

        input_path = self.input_file.name
        self.file_size_bytes = os.stat(input_path).st_size

        # Open the companion index file if one exists.
        self.index_path = file_index.FileIndex.get_path(input_path)
        self.next_index_elem = 0
        if ignore_index:
            if os.path.exists(self.index_path):
                self.logger.debug("Ignoring index file @ '%s'." % self.index_path)
            self.index = None<commit_msg>Allow single message type specifier.<commit_after>from datetime import datetime
import logging
import os
from typing import Union

from ..analysis import file_index
from ..messages import MessageType, MessageHeader, Timestamp, message_type_to_class
from ..utils.time_range import TimeRange


class MixedLogReader(object):
    """!
    @brief Generator class for decoding FusionEngine messages contained in a mixed-content binary file.

    For real-time deserialization of an incoming binary data stream, see @ref FusionEngineDecoder.
    """
    logger = logging.getLogger('point_one.fusion_engine.parsers.mixed_log_reader')

    def __init__(self, input_file, warn_on_gaps: bool = False, show_progress: bool = False,
                 generate_index: bool = True, ignore_index: bool = False,
                 time_range: TimeRange = None, message_types: Union[set, MessageType] = None,
                 return_header: bool = True, return_payload: bool = True,
                 return_bytes: bool = False, return_offset: bool = False):
        """!
        @brief Construct a new generator instance.

        Each call to @ref next() will return a tuple containing any/all of the message header, payload, serialized
        `bytes`, and byte offset depending on the values of the `return_*` parameters.

        @param input_file The path to an input file (`.p1log` or mixed-content binary file), or an open file-like
               object.
        @param warn_on_gaps If `True`, print warnings if gaps are detected in the FusionEngine message sequence numbers.
        @param show_progress If `True`, print file read progress to the console periodically.
        @param generate_index If `True`, generate an index file if one does not exist for faster reading in the future.
               See @ref FileIndex for details.
        @param ignore_index If `True`, ignore the existing index file and read from the binary file directly. If
               `generate_index == True`, this will delete the existing file and create a new one.
        @param return_header If `True`, return the decoded @ref MessageHeader for each message.
        @param return_payload If `True`, parse and return the payload for each message as a subclass of @ref
               MessagePayload. Will return `None` if the payload cannot be parsed.
        @param return_bytes If `True`, return a `bytes` object containing the serialized message header and payload.
        @param return_offset If `True`, return the offset into the file (in bytes) at which the message began.
        """
        self.warn_on_gaps = warn_on_gaps

        self.return_header = return_header
        self.return_payload = return_payload
        self.return_bytes = return_bytes
        self.return_offset = return_offset

        self.time_range = time_range
        if isinstance(message_types, MessageType):
            self.message_types = set((message_types,))
        else:
            self.message_types = message_types

        self.valid_count = 0
        self.message_counts = {}
        self.prev_sequence_number = None
        self.total_bytes_read = 0

        self.show_progress = show_progress
        self.last_print_bytes = 0
        self.start_time = datetime.now()

        # Open the file to be read.
        if isinstance(input_file, str):
            self.input_file = open(input_file, 'rb')
        else:
            self.input_file = input_file

        input_path = self.input_file.name
        self.file_size_bytes = os.stat(input_path).st_size

        # Open the companion index file if one exists.
        self.index_path = file_index.FileIndex.get_path(input_path)
        self.next_index_elem = 0
        if ignore_index:
            if os.path.exists(self.index_path):
                self.logger.debug("Ignoring index file @ '%s'." % self.index_path)
            self.index = None
{'fraction_non_alphanumeric': 0.06894722885176346, 'fraction_numerical': 0.0013259082471492973, 'mean_word_length': 2.692119432207538, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2138428', 'n_tokens_mistral': 1978, 'n_tokens_neox': 1921, 'n_words': 736}