<commit_before>import json
from .constants import MAX_GAP


def merge_streams(streams):
    # returns a combination of all streams
    merged_stream = streams[0].copy()
    for stream in streams[1:]:
        merged_stream['files'].extend(stream['files'])
        for height in stream['heights']:
            if height not in merged_stream['heights']:
                merged_stream['heights'].append(height)
        merged_stream['end_time'] = stream['end_time']
        for key in ("total_size", "total_frames", "total_duration", "total_gaps"):
            merged_stream[key] += stream[key]
    return merged_stream


def unify_streams(room1, room2):
    """

    Args:
        room1: Room with fewer streams
        room2: Room with more streams, some streams may be merged

    Returns:
        Nothing, but modifies its arguments
    """
    if room1['total_duration'] >= room2['total_duration']:
        for stream in room1['streams']:
            start_time = stream['start_time']
            end_time = stream['end_time']
            overlapped = [s for s in room2['streams'] if start_time - MAX_GAP < s['start_time'] < end_time + MAX_GAP]
            if len(overlapped) == 1:
                continue
            elif len(overlapped) > 1:
                if overlapped[-1]['end_time'] <= end_time + MAX_GAP:
                    # room2's stream set entirely within the bounds of room1's stream
                    merge_streams(overlapped)
                    for stream in overlapped[1:]:
                        room2['streams'].remove(stream)
                else:
                    # SOMETHING WEIRD IS HAPPENING IN OZ
                    pass
            else:
                # ERROR
                pass

    else:
        # most likely streams are entirely absent
        # this isn't really what this function is for though...
        pass


def compare_rooms(main_room, alt_room):
    result = {}

    SIZE_IEC_MB = 2 ** 20

    # TODO: drastically reduce the tolerances for top priority (< 8 or so) rooms
    def calc_max_time_diff(duration, priority):
        # this should never allow more than a minute of difference
        # 0-30 minutes: 15 seconds
        base_case = 41 - min(priority, 11)
        max_seconds = 15

        for mult in (1, 3, 6, 12):
            if duration > base_case*60*mult and max_seconds < 60:
                max_seconds += 15
            else:
                break

        return max_seconds

        # while factor < 5:
        #     if factor > priority / factor:
        #         break
        #     else:
        #         factor += 1
        # return min(factor + int(duration * factor / (60 * 12)), 30)

    def calc_max_size_diff(duration, priority):
        # TODO: this should allow between 1 and 5 MiB difference depending on the length of the stream
        # it should never allow anything larger than that.
        # 0-30 minutes: 1 MiB    30*1    3**0
        # 30-90 minutes: 2 MiB   30*3    3**1
        # 90-180 minutes: 3 MiB  30*6    3*2
        # 180-360 minutes: 4 MiB 30*12   3*4
        # 360+ minutes: 5 MiB
        base_case = 41 - min(priority, 11)
        max_mib = 1

        for mult in (1, 3, 6, 12):
            if duration > base_case*60*mult and max_mib < 5:
                max_mib+=1
            else:
                break

        return SIZE_IEC_MB*max_mib

        # factor = 1
        # while factor < 5:
        #     if factor > priority / factor:
        #         break
        #     else:
        #         factor += 1

        # return min(SIZE_IEC_MB + int(SIZE_IEC_MB * duration * factor / (12 * 60 ** 2)), SIZE_IEC_MB*3)

    def calc_max_frame_diff(duration, priority):
        return calc_max_time_diff(duration, priority) * 25

    
    # TODO: Compare streams instead of rooms
    main_size = sum((s['total_size'] for s in main_room['streams']))
    alt_size = sum((s['total_size'] for s in alt_room['streams']))
    main_frames = sum((s['total_frames'] for s in main_room['streams']))
    alt_frames = sum((s['total_frames'] for s in alt_room['streams']))
    result = {
        "time_diff": alt_room['total_duration'] - main_room['total_duration'],
        "size_diff": alt_size - main_size,
        "frame_diff": alt_frames - main_frames,<commit_msg>Weight all archives equally<commit_after>import json
from .constants import MAX_GAP


def merge_streams(streams):
    # returns a combination of all streams
    merged_stream = streams[0].copy()
    for stream in streams[1:]:
        merged_stream['files'].extend(stream['files'])
        for height in stream['heights']:
            if height not in merged_stream['heights']:
                merged_stream['heights'].append(height)
        merged_stream['end_time'] = stream['end_time']
        for key in ("total_size", "total_frames", "total_duration", "total_gaps"):
            merged_stream[key] += stream[key]
    return merged_stream


def unify_streams(room1, room2):
    """

    Args:
        room1: Room with fewer streams
        room2: Room with more streams, some streams may be merged

    Returns:
        Nothing, but modifies its arguments
    """
    if room1['total_duration'] >= room2['total_duration']:
        for stream in room1['streams']:
            start_time = stream['start_time']
            end_time = stream['end_time']
            overlapped = [s for s in room2['streams'] if start_time - MAX_GAP < s['start_time'] < end_time + MAX_GAP]
            if len(overlapped) == 1:
                continue
            elif len(overlapped) > 1:
                if overlapped[-1]['end_time'] <= end_time + MAX_GAP:
                    # room2's stream set entirely within the bounds of room1's stream
                    merge_streams(overlapped)
                    for stream in overlapped[1:]:
                        room2['streams'].remove(stream)
                else:
                    # SOMETHING WEIRD IS HAPPENING IN OZ
                    pass
            else:
                # ERROR
                pass

    else:
        # most likely streams are entirely absent
        # this isn't really what this function is for though...
        pass


def compare_rooms(main_room, alt_room):
    result = {}

    SIZE_IEC_MB = 2 ** 20

    # changing these all to 0 is probably going to bias it in favour of whichever is checked first
    # TODO: drastically reduce the tolerances for top priority (< 8 or so) rooms
    def calc_max_time_diff(duration, priority):
        # this should never allow more than a minute of difference
        # 0-30 minutes: 15 seconds
        base_case = 41 - min(priority, 11)
        max_seconds = 15

        for mult in (1, 3, 6, 12):
            if duration > base_case*60*mult and max_seconds < 60:
                max_seconds += 15
            else:
                break

        return 0 # max_seconds

        # while factor < 5:
        #     if factor > priority / factor:
        #         break
        #     else:
        #         factor += 1
        # return min(factor + int(duration * factor / (60 * 12)), 30)

    def calc_max_size_diff(duration, priority):
        # TODO: this should allow between 1 and 5 MiB difference depending on the length of the stream
        # it should never allow anything larger than that.
        # 0-30 minutes: 1 MiB    30*1    3**0
        # 30-90 minutes: 2 MiB   30*3    3**1
        # 90-180 minutes: 3 MiB  30*6    3*2
        # 180-360 minutes: 4 MiB 30*12   3*4
        # 360+ minutes: 5 MiB
        base_case = 41 - min(priority, 11)
        max_mib = 1

        for mult in (1, 3, 6, 12):
            if duration > base_case*60*mult and max_mib < 5:
                max_mib+=1
            else:
                break

        return 0 # SIZE_IEC_MB*max_mib

        # factor = 1
        # while factor < 5:
        #     if factor > priority / factor:
        #         break
        #     else:
        #         factor += 1

        # return min(SIZE_IEC_MB + int(SIZE_IEC_MB * duration * factor / (12 * 60 ** 2)), SIZE_IEC_MB*3)

    def calc_max_frame_diff(duration, priority):
        return calc_max_time_diff(duration, priority) * 25

    
    # TODO: Compare streams instead of rooms
    main_size = sum((s['total_size'] for s in main_room['streams']))
    alt_size = sum((s['total_size'] for s in alt_room['streams']))
    main_frames = sum((s['total_frames'] for s in main_room['streams']))
    alt_frames = sum((s['total_frames'] for s in alt_room['streams']))
    result = {
        "time_diff": alt_room['total_duration'] - main_room['total_duration'],
        "size_diff": alt_size - main_size,
        "frame_diff": alt_frames - main_frames,
{'fraction_non_alphanumeric': 0.09728166570271833, 'fraction_numerical': 0.029265471370734528, 'mean_word_length': 1.9338310145911095, 'pattern_counts': {'":': 6, '<': 19, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3073167', 'n_tokens_mistral': 2733, 'n_tokens_neox': 2476, 'n_words': 772}