<commit_before>import datetime
import sys


''' Simplistic module for manipulating MCF files (https://www.moviecontentfilter.com/specification)
'''


class McfSegment():
    def __init__(self, start, end, text):
        self.start = McfTiming(start)
        self.end = McfTiming(end)
        self.text = text


class Mcf():
    def __init__(self, start, end):
        self.start = McfTiming(start)
        self.end = McfTiming(end)
        self.segments = []

    @classmethod
    def fromfile(cls, filename):
        with open(filename) as file:
            for line in file:
                if line.startswith('NOTE') or line.startswith('WEBVTT') or len(line.strip()) == 0:
                    continue

                elif line.startswith('START'):
                    mcf_start = line.split()[1]

                elif line.startswith('END'):
                    mcf_end = line.split()[1]

                    mcf = cls(mcf_start, mcf_end)

                elif line[0].isdigit():
                    segment_start_timestamp = line.strip().split()[0]
                    segment_end_timestamp = line.strip().split()[2]

                else:
                    assert line.find('=') != -1

                    mcf.segments.append(
                        McfSegment(
                            segment_start_timestamp,
                            segment_end_timestamp,
                            line.strip()
                        )
                    )

        return mcf

    def write(self, filename):
        with open(filename, 'w') as file:
            file.write('WEBVTT MovieContentFilter 1.0.0\n\n')

            file.write('NOTE\n')
            file.write('START {}\n'.format(self.start))
            file.write('END {}\n'.format(self.end))

            for segment in self.segments:
                file.write('\n{} --> {}\n'.format(segment.start, segment.end))
                file.write('{}\n'.format(segment.text))


class McfTiming():
    def __init__(self, timestamp):
        if isinstance(timestamp, str):
            self.datetime = McfTiming.timestamp_to_timedelta(timestamp)
        elif isinstance(timestamp, datetime.timedelta):
            self.datetime = timestamp
        elif isinstance(timestamp, McfTiming):
            self.datetime = timestamp.datetime
        else:
            sys.exit('ERROR: timing is neither a string nor a datetime.timestamp')

    def __add__(self, other):
        return McfTiming(self.datetime + other.datetime)

    def __mul__(self, other):
        return McfTiming(self.datetime * other)

    def __sub__(self, other):
        return McfTiming(self.datetime - other.datetime)

    def __truediv__(self, other):
        return self.datetime / other.datetime

    @staticmethod
    # Timestamp spec: https://developer.mozilla.org/en-US/docs/Web/API/WebVTT_API#Cue_timings
    def timestamp_to_timedelta(timestamp_string):
        # hh:mm:ss.ttt
        if timestamp_string.count(':') == 2:
            hours, minutes, seconds_milliseconds = timestamp_string.split(':')
        # mm:ss.ttt
        elif timestamp_string.count(':') == 1:
            hours = 0
            minutes, seconds_milliseconds = timestamp_string.split(':')
        else:
            sys.exit('ERROR: invalid timestamp ({})\n'.format(timestamp_string))

        seconds, milliseconds = seconds_milliseconds.split('.')

        return datetime.timedelta(
            hours=int(hours),
            minutes=int(minutes),
            seconds=int(seconds),
            milliseconds=int(milliseconds)
        )

    def __str__(self):
        hours, remainder = divmod(self.datetime.total_seconds(), 3600)
        minutes, seconds = divmod(remainder, 60)
        microseconds = self.datetime.microseconds

        return '{:02d}:{:02d}:{:02d}.{}'.format(
            int(hours),
            int(minutes),
            int(seconds),
            str('{:06d}'.format(self.datetime.microseconds))[0:3]
        )<commit_msg>Reorganize<commit_after>import datetime
import sys


''' Simplistic module for manipulating MCF files (https://www.moviecontentfilter.com/specification)
'''


class Mcf():
    def __init__(self, start, end):
        self.start = McfTiming(start)
        self.end = McfTiming(end)
        self.segments = []

    @classmethod
    def fromfile(cls, filename):
        with open(filename) as file:
            for line in file:
                if line.startswith('NOTE') or line.startswith('WEBVTT') or len(line.strip()) == 0:
                    continue

                elif line.startswith('START'):
                    mcf_start = line.split()[1]

                elif line.startswith('END'):
                    mcf_end = line.split()[1]

                    mcf = cls(mcf_start, mcf_end)

                elif line[0].isdigit():
                    segment_start_timestamp = line.strip().split()[0]
                    segment_end_timestamp = line.strip().split()[2]

                else:
                    assert line.find('=') != -1

                    mcf.segments.append(
                        McfSegment(
                            segment_start_timestamp,
                            segment_end_timestamp,
                            line.strip()
                        )
                    )

        return mcf

    def write(self, filename):
        with open(filename, 'w') as file:
            file.write('WEBVTT MovieContentFilter 1.0.0\n\n')

            file.write('NOTE\n')
            file.write('START {}\n'.format(self.start))
            file.write('END {}\n'.format(self.end))

            for segment in self.segments:
                file.write('\n{} --> {}\n'.format(segment.start, segment.end))
                file.write('{}\n'.format(segment.text))


class McfSegment():
    def __init__(self, start, end, text):
        self.start = McfTiming(start)
        self.end = McfTiming(end)
        self.text = text


class McfTiming():
    def __init__(self, timestamp):
        if isinstance(timestamp, str):
            self.datetime = McfTiming.timestamp_to_timedelta(timestamp)
        elif isinstance(timestamp, datetime.timedelta):
            self.datetime = timestamp
        elif isinstance(timestamp, McfTiming):
            self.datetime = timestamp.datetime
        else:
            sys.exit('ERROR: timing is neither a string nor a datetime.timestamp')

    def __add__(self, other):
        return McfTiming(self.datetime + other.datetime)

    def __mul__(self, other):
        return McfTiming(self.datetime * other)

    def __sub__(self, other):
        return McfTiming(self.datetime - other.datetime)

    def __truediv__(self, other):
        return self.datetime / other.datetime

    # Output as hh:mm:ss.ttt
    def __str__(self):
        hours, remainder = divmod(self.datetime.total_seconds(), 3600)
        minutes, seconds = divmod(remainder, 60)
        microseconds = self.datetime.microseconds

        return '{:02d}:{:02d}:{:02d}.{}'.format(
            int(hours),
            int(minutes),
            int(seconds),
            str('{:06d}'.format(self.datetime.microseconds))[0:3]
        )

    # Timestamp spec: https://developer.mozilla.org/en-US/docs/Web/API/WebVTT_API#Cue_timings
    @staticmethod
    def timestamp_to_timedelta(timestamp_string):
        # hh:mm:ss.ttt
        if timestamp_string.count(':') == 2:
            hours, minutes, seconds_milliseconds = timestamp_string.split(':')
        # mm:ss.ttt
        elif timestamp_string.count(':') == 1:
            hours = 0
            minutes, seconds_milliseconds = timestamp_string.split(':')
        else:
            sys.exit('ERROR: invalid timestamp ({})\n'.format(timestamp_string))

        seconds, milliseconds = seconds_milliseconds.split('.')

        return datetime.timedelta(
            hours=int(hours),
            minutes=int(minutes),
            seconds=int(seconds),
            milliseconds=int(milliseconds)
        )
{'fraction_non_alphanumeric': 0.1202419964708848, 'fraction_numerical': 0.007310310057978321, 'mean_word_length': 1.883357558139535, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2620085', 'n_tokens_mistral': 2335, 'n_tokens_neox': 2195, 'n_words': 435}