<commit_before>import collections
import operator

import mido

from . import complain


class Note:
    def __init__(self, length=None, pitch=None, volume=None, starttime=None):
        self.starttime = starttime
        self.length = length
        self.volume = volume
        self.pitch = pitch

    def __hash__(self):
        return hash((self.length, self.pitch))

    def __eq__(self, other):
        return self.length == other.length and self.pitch == other.pitch

    def __len__(self):
        return len(self.data)


class CachedNote:
    def __init__(self, length, rendered):
        self.used = 1
        self.length = length
        self.data = rendered

    def __len__(self):
        return len(self.data)


class MIDIParser:
    def __init__(self, filename, wav, transpose=0, speed=1):  # TODO: convert the rest of the function to the new notes
        if speed <= 0:
            return ValueError("The speed must be a positive number.")
        results = collections.defaultdict(list)
        notes = collections.defaultdict(list)
        self.notecount = 0
        self.maxvolume = 0
        self.maxpitch = 0
        volume = 0
        
        if speed != 1:
            note_endings = collections.defaultdict(list)

        try:
            with (mido.MidiFile(filename, "r") if isinstance(filename, str) else filename) as mid:
                time = 0
                for message in mid:
                    time += message.time
                    if "channel" in vars(message) and message.channel == 10:
                        continue  # channel 10 is reserved for percussion
                    if message.type == "note_on":
                        note = Note()
                        note.starttime = int(round(time * wav.framerate / speed))
                        note.volume = 127 if message.velocity == 0 else message.velocity
                        note.pitch = self.note_to_freq(message.note + transpose)
                        notes[message.note].append(note)
                        
                        if speed != 1:
                            note_endings[]
                        volume += note.volume
                        self.maxvolume = max(volume, self.maxvolume)
                    elif message.type == "note_off":
                        note = notes[message.note].pop(0)
                        if len(notes[message.note]) == 0:
                            del notes[message.note]

                        try:
                            results[note.starttime].append(note)
                        except IndexError:
                            print("Warning: There was a note end event at {} seconds with no matching begin event".format(time))

                        self.notecount += 1
                        volume -= note.volume
                        self.maxpitch = max(self.maxpitch, note.pitch)
                        note.length = int(time * wav.framerate / speed) - note.starttime

                if len(notes) != 0:
                    print("Warning: The MIDI ended with notes still playing, assuming they end when the MIDI does")
                    for ntime, nlist in notes.items():
                        for note in nlist:
                            note.length = int(time * wav.framerate / speed) - note.starttime
                            self.notecount += 1

                if self.notecount == 0:
                    raise complain.ComplainToUser("This MIDI file doesn't have any notes in it!")

                self.notes = sorted(results.items(), key=operator.itemgetter(0))
                self.length = max(max(note.starttime + note.length for note in nlist) for _, nlist in self.notes)
        except IOError:
            raise complain.ComplainToUser("Error opening MIDI file '{}'.".format(filename))
        except IndexError:
            raise complain.ComplainToUser("This MIDI file is broken. Try opening it in MidiEditor (https://meme.institute/midieditor) and saving it back out again.")

    def note_to_freq(self, notenum):
        # see https://en.wikipedia.org/wiki/MIDI_Tuning_Standard<commit_msg>Use a separate set for keeping track of the stretched notes<commit_after>import collections
import operator

import mido

from . import complain


class Note:
    def __init__(self, length=None, pitch=None, volume=None, starttime=None):
        self.starttime = starttime
        self.length = length
        self.volume = volume
        self.pitch = pitch

    def __hash__(self):
        return hash((self.length, self.pitch))

    def __eq__(self, other):
        return self.length == other.length and self.pitch == other.pitch

    def __len__(self):
        return len(self.data)


class CachedNote:
    def __init__(self, length, rendered):
        self.used = 1
        self.length = length
        self.data = rendered

    def __len__(self):
        return len(self.data)


class MIDIParser:
    def __init__(self, filename, wav, transpose=0, speed=1):  # TODO: convert the rest of the function to the new notes
        if speed <= 0:
            return ValueError("The speed must be a positive number.")
        results = collections.defaultdict(list)
        notes = collections.defaultdict(list)
        self.notecount = 0
        self.maxvolume = 0
        self.maxpitch = 0
        volume = 0
        
        if speed != 1:
            unended_notes = set()  # filled with (endtime, note.volume)

        try:
            with (mido.MidiFile(filename, "r") if isinstance(filename, str) else filename) as mid:
                time = 0
                for message in mid:
                    time += message.time
                    if "channel" in vars(message) and message.channel == 10:
                        continue  # channel 10 is reserved for percussion
                    if message.type == "note_on":
                        note = Note()
                        note.starttime = int(round(time * wav.framerate / speed))
                        note.volume = 127 if message.velocity == 0 else message.velocity
                        note.pitch = self.note_to_freq(message.note + transpose)
                        notes[message.note].append(note)
                        
                        if speed != 1:
                            removed = set()
                            for unended_note in unended_notes:
                                if unended_note[0] < time:
                                    volume -= unended_note[1]
                                    removed.add(unended_note)
                            unended_notes -= removed
                        volume += note.volume
                        self.maxvolume = max(volume, self.maxvolume)
                    elif message.type == "note_off":
                        note = notes[message.note].pop(0)
                        if len(notes[message.note]) == 0:
                            del notes[message.note]

                        try:
                            results[note.starttime].append(note)
                        except IndexError:
                            print("Warning: There was a note end event at {} seconds with no matching begin event".format(time))

                        
                        endtime = int(time * wav.framerate / speed)
                        if speed == 1:
                            volume -= note.volume
                        else:
                            unended_notes.add((endtime, note.volume))
                        note.length = endtime - note.starttime
                        
                        self.notecount += 1
                        self.maxpitch = max(self.maxpitch, note.pitch)
                        

                if len(notes) != 0:
                    print("Warning: The MIDI ended with notes still playing, assuming they end when the MIDI does")
                    for ntime, nlist in notes.items():
                        for note in nlist:
                            note.length = int(time * wav.framerate / speed) - note.starttime
                            self.notecount += 1

                if self.notecount == 0:
                    raise complain.ComplainToUser("This MIDI file doesn't have any notes in it!")

                self.notes = sorted(results.items(), key=operator.itemgetter(0))
                self.length = max(max(note.starttime + note.length for note in nlist) for _, nlist in self.notes)
        except IOError:
            raise complain.ComplainToUser("Error opening MIDI file '{}'.".format(filename))
        except IndexError:
            raise complain.ComplainToUser("This MIDI file is broken. Try opening it in MidiEditor (https://meme.institute/midieditor) and saving it back out again.")

    def note_to_freq(self, notenum):
        # see https://en.wikipedia.org/wiki/MIDI_Tuning_Standard
{'fraction_non_alphanumeric': 0.0844354018311292, 'fraction_numerical': 0.006216796654233073, 'mean_word_length': 1.4401544401544402, 'pattern_counts': {'":': 4, '<': 6, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2713255', 'n_tokens_mistral': 2323, 'n_tokens_neox': 2139, 'n_words': 638}