<commit_before>from mtools.util.logline import LogLine

class LogFile(object):
    """ wrapper class for log files, either as open file streams of from stdin. 
        Later planned to include logfile from a mongod instance.
    """

    def __init__(self, logfile):
        """ provide logfile as open file stream or stdin. """
        self.logfile = logfile
        self.from_stdin = logfile.name == "<stdin>"
        self._start = None
        self._end = None


    @property
    def start(self):
        """ lazy evaluation of start and end of logfile. Returns None for stdin input currently. """
        if self.from_stdin:
            return None
        if not self._start:
            self._calculate_bounds()
        return self._start

    @property
    def end(self):
        """ lazy evaluation of start and end of logfile. Returns None for stdin input currently. """
        if self.from_stdin:
            return None
        if not self._end:
            self._calculate_bounds()
        return self._end


    def _calculate_bounds(self):
        """ calculate beginning and end of logfile. """

        if self.from_stdin: 
            return None

        # get start datetime 
        for line in self.logfile:
            logline = LogLine(line)
            date = logline.datetime
            if date:
                break
        self._start = date

        # get end datetime (lines are at most 10k, go back 15k at most to make sure)
        self.logfile.seek(0, 2)
        file_size = self.logfile.tell()
        self.logfile.seek(-min(file_size, 15000), 2)

        for line in reversed(self.logfile.readlines()):
            logline = LogLine(line)
            date = logline.datetime
            if date:
                break
        self._end = date

        # if there was a roll-over, subtract 1 year from start time<commit_msg>temporary check-in. the line count may not be working yet.<commit_after>from mtools.util.logline import LogLine

class LogFile(object):
    """ wrapper class for log files, either as open file streams of from stdin. 
        Later planned to include logfile from a mongod instance.
    """

    def __init__(self, logfile):
        """ provide logfile as open file stream or stdin. """
        self.logfile = logfile
        self.from_stdin = logfile.name == "<stdin>"
        self._start = None
        self._end = None


    @property
    def start(self):
        """ lazy evaluation of start and end of logfile. Returns None for stdin input currently. """
        if self.from_stdin:
            return None
        if not self._start:
            self._calculate_bounds()
        return self._start

    @property
    def end(self):
        """ lazy evaluation of start and end of logfile. Returns None for stdin input currently. """
        if self.from_stdin:
            return None
        if not self._end:
            self._calculate_bounds()
        return self._end

    @property
    def num_lines(self):
        """ lazy evaluation of the number of lines. Returns None for stdin input currently. """
        if self.from_stdin:
            return None
        if not self._num_lines:
            self._count_lines()
        return self._num_lines



    def _count_lines(self):
        """ count number of lines (can be expensive). """
        self._num_lines = 0
        for line in self.logfile:
            self._num_lines += 1


    def _calculate_bounds(self):
        """ calculate beginning and end of logfile. """

        if self.from_stdin: 
            return None

        # get start datetime 
        for line in self.logfile:
            logline = LogLine(line)
            date = logline.datetime
            if date:
                break
        self._start = date

        # get end datetime (lines are at most 10k, go back 15k at most to make sure)
        self.logfile.seek(0, 2)
        file_size = self.logfile.tell()
        self.logfile.seek(-min(file_size, 15000), 2)

        for line in reversed(self.logfile.readlines()):
            logline = LogLine(line)
            date = logline.datetime
            if date:
                break
        self._end = date

        # if there was a roll-over, subtract 1 year from start time
{'fraction_non_alphanumeric': 0.079357581483231, 'fraction_numerical': 0.006613131790269249, 'mean_word_length': 1.8271028037383177, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5358493', 'n_tokens_mistral': 1188, 'n_tokens_neox': 1121, 'n_words': 405}