<commit_before>import copy
import math

MIDI_DEFAULT_BPM = 120
MIDI_MAX_CHANNELS = 16

MIDI_TO_PICO8_PITCH_SUBTRAHEND = 36

PICO8_MAX_PITCH = 63
PICO8_MIN_NOTE_DURATION = 1

# According to https://eev.ee/blog/2016/05/30/extracting-music-from-the-pico-8/
PICO8_MS_PER_TICK = (183 / 22050) * 1000

class Note:
    def __init__(self, event=None):
        # MIDI properties
        self.midiDuration = 0
        self.midiPitch = None
        self.midiChannel = None
        self.midiVelocity = None

        # PICO-8 tracker properties
        self.pitch = None
        self.volume = 0
        self.effect = None

        if event != None:
            self.midiPitch = event.pitch
            self.midiChannel = event.channel
            self.midiVelocity = event.velocity
            self.pitch = event.pitch - MIDI_TO_PICO8_PITCH_SUBTRAHEND
            self.volume = math.floor((event.velocity / 127) * 7)

class TranslatorSettings:
    def __init__(self):
        self.quantization = True
        self.ticksPerNoteOverride = None
        self.staccato = False
        self.legato = False
        self.fixOctaves = True
        self.noteDurationOverride = None

class Translator:
    def __init__(self, midiFile, settings: TranslatorSettings=None): 
        self.midiFile = midiFile

        if settings != None:
            self.settings = settings
        else:
            self.settings = TranslatorSettings()

        if self.settings.ticksPerNoteOverride != None:
            print('setting ticks per note to override setting of ' +
                  str(self.settings.ticksPerNoteOverride))
        self.baseTicks = self.settings.ticksPerNoteOverride

    #def find_notes_any_channel(self, track):
    #    notes = []
    #    activeNote = None
    #    deltaTime = 0

    #    for e, event in enumerate(track.events):
    #        if event.type == 'DeltaTime':
    #            deltaTime += event.time
    #        if event.type == 'NOTE_ON' or event.type == 'NOTE_OFF':
    #            if activeNote != None:
    #                activeNote.midiDuration = deltaTime
    #                if deltaTime == 40:
    #                    print(event)
    #            deltaTime = 0

    #            activeNote = Note(event)
    #            notes.append(activeNote)

    #            if event.type == 'NOTE_OFF':
    #                activeNote.volume = 0

    #    return notes

    def find_notes(self, track, channel):
        notes = []
        activeNotes = []
        deltaTime = 0
        firstNoteHasBeenAdded = False

        events = [
                event for event in track.events
                if event.channel == channel or event.type == 'DeltaTime']

        for e, event in enumerate(events):
            if event.type == 'DeltaTime':
                deltaTime += event.time
            if event.type == 'NOTE_ON' or event.type == 'NOTE_OFF':
                existingNote = next(
                    (note for note in activeNotes
                     if note.midiPitch == event.pitch and
                     note.midiChannel == event.channel),
                    None)

                if existingNote != None:
                    existingNote.midiDuration = deltaTime
                    activeNotes.remove(existingNote)
                elif deltaTime > 0:
                    note = Note()
                    note.midiDuration = deltaTime
                    notes.append(note)

                if event.type == 'NOTE_ON' and event.velocity > 0:
                    note = Note(event)
                    activeNotes.append(note)
                    notes.append(note)

                deltaTime = 0

        return notes

    def analyze(self):
        # Get a list of unique note lengths and the number of occurences of
        # each length
        uniqueLengths = {}
        noteCount = 0
        for track in self.midiFile.tracks:
            occupiedChannels = self.find_occupied_channels(track)
            for channel in occupiedChannels:
                notes = self.find_notes(track, channel)
                
                for note in notes:
                    noteCount += 1
                    length = note.midiDuration
                    if length > 0:
                        if not length in uniqueLengths:
                            uniqueLengths[length] = 0
                        uniqueLengths[length] += 1

        # DEBUG
        import operator
        print('unique lengths:')
        sortedUniqueLengths = sorted(
            uniqueLengths.items(),
            key=operator.itemgetter(1),
            reverse=True)
        print(sortedUniqueLengths)

        mostFrequentLength = None
        highestCount = 0
        for length, count in uniqueLengths.items():
            if count > highestCount:

                highestCount = count
                mostFrequentLength = length

        print('note count: ' + str(noteCount))
        print('most frequent length: ' + str(mostFrequentLength))

        # Take each length and divide the other lengths by it, counting how
        # many other lengths it divides evenly into
        candidateBaseLengths = {}
        for length, occurences in uniqueLengths.items():
            for otherLength in uniqueLengths.keys():
                if otherLength != length:
                    if otherLength % length == 0:
                        if not length in candidateBaseLengths:
                            candidateBaseLengths[length] = 0
                        candidateBaseLengths[length] += 1

        # DEBUG
        print('candidate base lengths:')
        sortedCandidateBaseLengths = sorted(
            candidateBaseLengths.items(),
            key=operator.itemgetter(1),
            reverse=True)
        for length, score in sortedCandidateBaseLengths:
            print(length, score)

        # Find the best of the candidate base-lengths, where "best" is the one
        # with the most even divisions into other lengths. If there is a tie,
        # prefer the shortest candidate base-length.
        bestBaseLength = None
        highestNumberOfEvenDivisions = 0
        for length, evenDivisionCount in candidateBaseLengths.items():
            # If this length is an "outlier", i.e the length occurs on less
            # than 5% of all notes
            if (evenDivisionCount / noteCount) < 0.05:
                continue

            if evenDivisionCount > highestNumberOfEvenDivisions:
                highestNumberOfEvenDivisions = evenDivisionCount
                bestBaseLength = length
            elif evenDivisionCount == highestNumberOfEvenDivisions:
                if length < bestBaseLength:
                    bestBaseLength = length

        if self.baseTicks == None:
            print('setting MIDI base ticks per note to ' + str(bestBaseLength))
            self.baseTicks = bestBaseLength<commit_msg>Implement a base-length algorithm that seems a little better<commit_after>import copy
import math
import statistics

MIDI_DEFAULT_BPM = 120
MIDI_MAX_CHANNELS = 16

MIDI_TO_PICO8_PITCH_SUBTRAHEND = 36

PICO8_MAX_PITCH = 63
PICO8_MIN_NOTE_DURATION = 1

# According to https://eev.ee/blog/2016/05/30/extracting-music-from-the-pico-8/
PICO8_MS_PER_TICK = (183 / 22050) * 1000

class Note:
    def __init__(self, event=None):
        # MIDI properties
        self.midiDuration = 0
        self.midiPitch = None
        self.midiChannel = None
        self.midiVelocity = None

        # PICO-8 tracker properties
        self.pitch = None
        self.volume = 0
        self.effect = None

        if event != None:
            self.midiPitch = event.pitch
            self.midiChannel = event.channel
            self.midiVelocity = event.velocity
            self.pitch = event.pitch - MIDI_TO_PICO8_PITCH_SUBTRAHEND
            self.volume = math.floor((event.velocity / 127) * 7)

class TranslatorSettings:
    def __init__(self):
        self.quantization = True
        self.ticksPerNoteOverride = None
        self.staccato = False
        self.legato = False
        self.fixOctaves = True
        self.noteDurationOverride = None

class Translator:
    def __init__(self, midiFile, settings: TranslatorSettings=None): 
        self.midiFile = midiFile

        if settings != None:
            self.settings = settings
        else:
            self.settings = TranslatorSettings()

        if self.settings.ticksPerNoteOverride != None:
            print('setting ticks per note to override setting of ' +
                  str(self.settings.ticksPerNoteOverride))
        self.baseTicks = self.settings.ticksPerNoteOverride

    #def find_notes_any_channel(self, track):
    #    notes = []
    #    activeNote = None
    #    deltaTime = 0

    #    for e, event in enumerate(track.events):
    #        if event.type == 'DeltaTime':
    #            deltaTime += event.time
    #        if event.type == 'NOTE_ON' or event.type == 'NOTE_OFF':
    #            if activeNote != None:
    #                activeNote.midiDuration = deltaTime
    #                if deltaTime == 40:
    #                    print(event)
    #            deltaTime = 0

    #            activeNote = Note(event)
    #            notes.append(activeNote)

    #            if event.type == 'NOTE_OFF':
    #                activeNote.volume = 0

    #    return notes

    def find_notes(self, track, channel):
        notes = []
        activeNotes = []
        deltaTime = 0
        firstNoteHasBeenAdded = False

        events = [
                event for event in track.events
                if event.channel == channel or event.type == 'DeltaTime']

        for e, event in enumerate(events):
            if event.type == 'DeltaTime':
                deltaTime += event.time
            if event.type == 'NOTE_ON' or event.type == 'NOTE_OFF':
                existingNote = next(
                    (note for note in activeNotes
                     if note.midiPitch == event.pitch and
                     note.midiChannel == event.channel),
                    None)

                if existingNote != None:
                    existingNote.midiDuration = deltaTime
                    activeNotes.remove(existingNote)
                elif deltaTime > 0:
                    note = Note()
                    note.midiDuration = deltaTime
                    notes.append(note)

                if event.type == 'NOTE_ON' and event.velocity > 0:
                    note = Note(event)
                    activeNotes.append(note)
                    notes.append(note)

                deltaTime = 0

        return notes

    def analyze(self):
        # Get a list of unique note lengths and the number of occurrences of
        # each length
        uniqueLengths = {}
        noteCount = 0
        for track in self.midiFile.tracks:
            occupiedChannels = self.find_occupied_channels(track)
            for channel in occupiedChannels:
                notes = self.find_notes(track, channel)
                
                for note in notes:
                    noteCount += 1
                    length = note.midiDuration
                    if length > 0:
                        if not length in uniqueLengths:
                            uniqueLengths[length] = 0
                        uniqueLengths[length] += 1

        # DEBUG
        import operator
        print('unique lengths:')
        sortedUniqueLengths = sorted(
            uniqueLengths.items(),
            key=operator.itemgetter(1),
            reverse=True)
        print(sortedUniqueLengths)

        mostFrequentLength = None
        highestCount = 0
        for length, count in uniqueLengths.items():
            if count > highestCount:

                highestCount = count
                mostFrequentLength = length

        print('note count: ' + str(noteCount))
        print('most frequent length: ' + str(mostFrequentLength))

        # Find the average number of occurrences for a unique length
        averageOccurences = statistics.mean(uniqueLengths.values())
        print('mean occurrences: ' + str(averageOccurences))
        print('median occurrences: ' +
              str(statistics.median(uniqueLengths.values())))

        # Remove lengths from uniqueLengths that have less than the average
        # number of occurrences
        newUniqueLengths = {}
        for length, occurrences in uniqueLengths.items():
            if occurrences >= averageOccurences:
                newUniqueLengths[length] = occurrences
        uniqueLengths = newUniqueLengths

        # Take each length and divide the other lengths by it, counting how
        # many other lengths it divides evenly into
        candidateBaseLengths = {}
        for length, occurrences in uniqueLengths.items():
            for otherLength in uniqueLengths.keys():
                if otherLength % length == 0:
                    if not length in candidateBaseLengths:
                        candidateBaseLengths[length] = 0
                    candidateBaseLengths[length] += 1

        # DEBUG
        print('candidate base lengths:')
        sortedCandidateBaseLengths = sorted(
            candidateBaseLengths.items(),
            key=operator.itemgetter(1),
            reverse=True)
        for length, score in sortedCandidateBaseLengths:
            print(length, score)


        # Find the best of the candidate base-lengths, where "best" is the one
        # with the most even divisions into other lengths. If there is a tie,
        # prefer the shortest candidate base-length.
        bestBaseLength = None
        highestNumberOfEvenDivisions = 0
        for length, evenDivisionCount in candidateBaseLengths.items():
            if evenDivisionCount > highestNumberOfEvenDivisions:
                highestNumberOfEvenDivisions = evenDivisionCount
                bestBaseLength = length
            elif evenDivisionCount == highestNumberOfEvenDivisions:
                if length < bestBaseLength:
                    bestBaseLength = length

        if self.baseTicks == None:
            print('setting MIDI base ticks per note to ' + str(bestBaseLength))
            self.baseTicks = bestBaseLength
{'fraction_non_alphanumeric': 0.0731052036199095, 'fraction_numerical': 0.009332579185520362, 'mean_word_length': 1.6734076734076735, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 14, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1326758', 'n_tokens_mistral': 3901, 'n_tokens_neox': 3548, 'n_words': 968}