<jupyter_start><jupyter_text>Set Up  Standard library imports<jupyter_code>import copy
import datetime
import math
import os
import pdb
import random
import time<jupyter_output><empty_output><jupyter_text>Third party imports<jupyter_code>import pydot
import numpy as np
import pandas as pd
import tensorflow as tf
import tensorflow_addons as tfa<jupyter_output><empty_output><jupyter_text>Local imports<jupyter_code>import modules.batch as batch
import modules.midi_related as midi
import modules.preprocessing as prep
import modules.subclasses as sub<jupyter_output><empty_output><jupyter_text>Extensions and autoreload<jupyter_code>%load_ext autoreload
%load_ext tensorboard
%autoreload 2<jupyter_output><empty_output><jupyter_text>Dis-/enable GPU<jupyter_code>disable_gpu = False
debugging = False

if disable_gpu:
    # Hide GPU from visible devices
    tf.config.set_visible_devices([], 'GPU')
    if debugging:
        # To find out which devices your operations and tensors are assigned to
        tf.debugging.set_log_device_placement(True)

        # Create some tensors and perform an operation
        a = tf.constant([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]])
        b = tf.constant([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])
        c = tf.matmul(a, b)

        print(c)
        tf.debugging.set_log_device_placement(False)
physical_devices = tf.config.list_physical_devices('GPU')<jupyter_output><empty_output><jupyter_text>Setting relative directories<jupyter_code>Working_Directory = os.getcwd()
Project_Directory = os.path.abspath(os.path.join(Working_Directory,'..'))
Music_In_Directory = Project_Directory + "/data/" 
Output_Directory = Project_Directory + "/outputs/"
Model_Directory = Output_Directory + "models/"
Checkpoint_Directory = Model_Directory + "ckpt/"
Numpy_Directory = Model_Directory + "arrays/"
Music_Out_Directory = Output_Directory + "midi/"
Music_Out_Training_Directory =  Music_Out_Directory + "train/"
Music_Out_Genereating_Directory = Music_Out_Directory + "generated/"

Midi_Directories = [
    "albeniz", 
    "beeth",
    "borodin",
    "brahms",
    "burgm",
    "chopin", 
    #"chopin_midi",
    "debussy", 
    "granados", 
    "grieg", 
    "haydn", 
    "liszt", 
    "mendelssohn", 
    "mozart", 
    "muss", 
    "schubert", 
    "schumann", 
    "tschai"
]<jupyter_output><empty_output><jupyter_text>Data preperation  Load pieces (i.e. import midi files)<jupyter_code># First checkt that importing single midi (i.e. Chopin Op 28 No.4) works
chop2804 = midi.midiToNoteStateMatrix(Music_In_Directory + "chopin_midi/chop7201.mid") 
min_time_steps = 128 # only files with at least this many 48th note steps are saved
lowerBound = 21
upperBound = 109
# Import all Midi data

all_pieces = {}
chopin_only_pieces = {}
piano_midi_only_pieces = {}

start_time_loading = time.time()
time_loading_old = start_time_loading

# Gather the pieces from the specified directory
for f in range(len(Midi_Directories)):
    Training_Midi_Folder = Music_In_Directory + Midi_Directories[f]
    if Midi_Directories[f] == 'chopin_midi':
        chopin_only_pieces = {**chopin_only_pieces, **midi.loadPieces(Training_Midi_Folder,
                                                                      min_time_steps,
                                                                      lowerBound, 
                                                                      upperBound,
                                                                      verbose=False,
                                                                      verbose_name=True)}
    else: 
        piano_midi_only_pieces = {**piano_midi_only_pieces, **midi.loadPieces(Training_Midi_Folder,
                                                                              min_time_steps,
                                                                              lowerBound, 
                                                                              upperBound,
                                                                              verbose=False,
                                                                              verbose_name=True)}
    time_loading_new = time.time()
    duration = time_loading_new - time_loading_old
    time_loading_old = time_loading_new
    print('Loading directory ' + Midi_Directories[f] + ' took ' + str(round(duration, 3)) + 's' )

all_pieces = {**chopin_only_pieces, **piano_midi_only_pieces}
end_time_loading = time.time()

print('Number of total pieces = ', len(all_pieces))    
print('Loading all pieces took ' +  str(round(end_time_loading - start_time_loading, 3)) + 's')<jupyter_output>Loaded alb_esp1
Loaded alb_esp2
Loaded alb_esp3
Loaded alb_esp4
Loaded alb_esp5
Loaded alb_se1
Loaded alb_se2
Loaded alb_se3
Loaded alb_se4
Loaded alb_se5
Loaded alb_se6
Loaded alb_se7
Loaded alb_se8
Loading directory albeniz took 11.876s
Loaded appass_1
Loaded appass_2
Loaded appass_3
Loaded beethoven_hammerklavier_1
Loaded beethoven_hammerklavier_2
Loaded beethoven_hammerklavier_3
Loaded beethoven_hammerklavier_4
Loaded beethoven_les_adieux_1
Loaded beethoven_les_adieux_2
Loaded beethoven_les_adieux_3
Loaded beethoven_opus10_1
Loaded beethoven_opus10_2
Loaded beethoven_opus10_3
Loaded beethoven_opus22_1
Loaded beethoven_opus22_2
Loaded beethoven_opus22_3
Loaded beethoven_opus22_4
Loaded beethoven_opus90_1
Loaded beethoven_opus90_2
Loaded elise
Loaded mond_1
Loaded mond_2
Loaded mond_3
Loaded pathetique_1
Loaded pathetique_2
Loaded pathetique_3
Loaded waldstein_1
Loaded waldstein_2
Loaded waldstein_3
Loading directory beeth took 68.894s
Loaded bor_ps1
Loaded bor_ps2
Loaded bor_ps3
Loa[...]<jupyter_text>Chopin pieces by genre<jupyter_code>print('Number of Chopin pieces loaded = ', len(chopin_only_pieces))
#Only include pieces which were not recoreded (i.e. which MIDI files are exact) and are in 2/4, 3/4 or 4/4
time_signatures = []
verbose = False #True
chopin_pieces_exact = chopin_only_pieces.copy()
keys = list(chopin_pieces_exact.keys())
for k in keys:
        piece = chopin_pieces_exact[str(k)]
        time_signature = max([b[0][3] for b in  piece])
        if not (str(k) in midi.EXACT_FILES):
            chopin_pieces_exact.pop(k)
        else:
            time_signatures.append(time_signature)
        if verbose:
            print("Piece: {}".format(k) + "  Time signature: {}".format(time_signature))
time_signatures = np.array(time_signatures)
unique, counts = np.unique(time_signatures, return_counts=True)
dict(zip(unique, counts))
# Create table of piece names for midi files loaded
df_chopin_opus = pd.read_csv(Project_Directory + '/data_info/chopin_title_opus.csv')
op_numbers = [int(str(k)[4:6]) for k in chopin_pieces_exact.keys()]
op_numbers_unique = set(op_numbers)
df_chopin_opus_midi = df_chopin_opus.loc[df_chopin_opus['Op'].isin(op_numbers_unique)]
# Define genres
genres = [
    'Ballade', 
    'Etudes', 
    'promptu', 
    'Mazurkas', 
    'Nocturne', 
    'Preludes',  
    'Sonata', 
    'Waltz',
    'Polonaise',
    'Scherzo',
    'Marche funèbre',
    'Ecossaises'
]
def agg_op(x):
    pdb.set_trace()
    list(x)
    
def aggregate(data):
    d = {}
    Op_list = data['Op'].tolist()
    Midi_list = data['MIDI files'].tolist()
    d['Op'] = [ str(a) + ' (' + str(b) + ')' for a, b in zip(Op_list, Midi_list)]
    d['MIDI files'] = sum(data['MIDI files'])
    return pd.Series(d)

df_pieces_summary = pd.concat([midi.get_piece_summary_df(i, df_chopin_opus_midi, op_numbers) for i in genres])
df_genre_summary = df_pieces_summary[['MIDI files', 'genre', 'Op']] \
                    .groupby(['genre']) \
                    .apply(aggregate)
df_genre_summary.rename(index={'Ballade':'Ballades','promptu':'Impromptus', 'Sonata' : 'Sonatas', 'Nocturne' : 'Nocturnes', 'Waltz' :'Waltzes'}, inplace=True)
df_genre_summary = df_genre_summary.sort_index()
df_genre_summary.style.set_properties(subset=['Op'], **{'width-min': '300px'})
# Update all_pieces to only use pieces from defined genres 
op_train = [item for sublist in df_genre_summary['Op'] for item in sublist]
genere_keys = [k for k in chopin_only_pieces.keys() if int(str(k)[4:6]) in op_train]
genere_pieces = {k:v for k, v in chopin_only_pieces.items() if k in genere_keys}
print('Number of pieces by Chopin left for selected generes = ', len(genere_pieces))
# Define pieces for each genre
ballades = midi.get_subset_all_pieces_for_genre('Ballades', chopin_only_pieces, df_genre_summary)
etudes = midi.get_subset_all_pieces_for_genre('Etudes', chopin_only_pieces, df_genre_summary)
impromptus = midi.get_subset_all_pieces_for_genre('Impromptus', chopin_only_pieces, df_genre_summary)
mazurkas = midi.get_subset_all_pieces_for_genre('Mazurkas', chopin_only_pieces, df_genre_summary)
nocturnes = midi.get_subset_all_pieces_for_genre('Nocturnes', chopin_only_pieces, df_genre_summary)
preludes = midi.get_subset_all_pieces_for_genre('Preludes', chopin_only_pieces, df_genre_summary)
sonatas = midi.get_subset_all_pieces_for_genre('Sonatas', chopin_only_pieces, df_genre_summary)
waltzes = midi.get_subset_all_pieces_for_genre('Waltzes', chopin_only_pieces, df_genre_summary)<jupyter_output><empty_output><jupyter_text>Filter 'incorrect' Chopin pieces by genre<jupyter_code>#Check time signature occurences
time_signatures = []
verbose = False #True
keys = list(chopin_only_pieces.keys())
for k in keys:
    piece = chopin_only_pieces[str(k)]
    time_signature = max([b[0][3] for b in  piece])
    if str(k) in midi.EXACT_FILES:
        time_signatures.append(time_signature)
    if verbose:
        print("Piece: {}".format(k) + "  Time signature: {}".format(time_signature))
time_signatures = np.array(time_signatures)
unique, counts = np.unique(time_signatures, return_counts=True)
dict(zip(unique, counts))
print('Number of synthesized Chopin pieces = ', sum(counts))
import matplotlib.pyplot as plt
time_signatures_dict = dict()
time_signatures_dict["other"] =  0
keys = [
    "3/8",
    "4/8\n 2/4",
    "6/8\n 3/4",
    "8/8\n 4/4",
    "12/8\n 6/4",
    "other"
]

for ts in zip(unique, counts):
    if ts[0] == 6:
        # append the new number to the existing array at this slot
        time_signatures_dict[keys[0]] = ts[1]
    elif ts[0] == 8:
        # append the new number to the existing array at this slot
        time_signatures_dict[keys[1]] = ts[1]
    elif ts[0] == 12:
        # append the new number to the existing array at this slot
        time_signatures_dict[keys[2]] = ts[1]
    elif ts[0] == 16:
        # append the new number to the existing array at this slot
        time_signatures_dict[keys[3]] = ts[1]
    elif ts[0] == 24:
        # append the new number to the existing array at this slot
        time_signatures_dict[keys[4]] = ts[1]
    else:
        # create a new array in this slot
        time_signatures_dict[keys[5]] += ts[1]
        
desired_order_list = [1, 2, 3, 4, 5, 6]

time_signatures_dict = {keys[k-1]: time_signatures_dict[keys[k-1]] for k in desired_order_list}
import matplotlib.pyplot as plt
from matplotlib.pyplot import figure

x_tmp = time_signatures_dict.keys()
y_tmp = time_signatures_dict.values()

fig, ax = plt.subplots()
fig.set_facecolor('white')
width = 0.75 # the width of the bars 
ax.bar(x_tmp, y_tmp, width, color="black", fill=False)
plt.title('F. Chopin dataset')
plt.xlabel('Time signature')
plt.ylabel('Number of MIDI files')   
plt.ylim([0, 45])
for i, v in enumerate(y_tmp):
    ax.text(i -0.2, v + 1 , str(v), color='black')
    
fig = plt.gcf()
fig.set_size_inches(7, 4)
plt.rcParams.update({'font.size': 22})
plt.savefig(os.path.join('Chopin_dataset.png'), dpi=300, format='png', bbox_inches='tight', transparent= False)
#Only include pieces which were not recoreded (i.e. which MIDI files are exact) and are in 2/4, 3/4 or 4/4
time_signatures = []
verbose = False #True
chopin_pieces_filtered = chopin_only_pieces.copy()
keys = list(chopin_pieces_filtered.keys())
for k in keys:
        piece = chopin_pieces_filtered[str(k)]
        time_signature = max([b[0][3] for b in  piece])
        if not ((time_signature in [16]) and str(k) in midi.EXACT_FILES) :
            chopin_pieces_filtered.pop(k)
        else:
            time_signatures.append(time_signature)
        if verbose:
            print("Piece: {}".format(k) + "  Time signature: {}".format(time_signature))
time_signatures = np.array(time_signatures)
unique, counts = np.unique(time_signatures, return_counts=True)
dict(zip(unique, counts))
print('Number of Chopin pieces left after filtering = ', len(chopin_pieces_filtered))<jupyter_output>Number of Chopin pieces left after filtering =  31<jupyter_text>Filter Piano Midi pieces by 3/4 or 4/4 measure<jupyter_code>print('Number of Piano Midi pieces loaded = ', len(piano_midi_only_pieces))
#Check time signature occurences
time_signatures = []
verbose = False #True
keys = list(piano_midi_only_pieces.keys())
for k in keys:
    piece = piano_midi_only_pieces[str(k)]
    time_signature = max([b[0][3] for b in  piece])
    time_signatures.append(time_signature)
    if verbose:
        print("Piece: {}".format(k) + "  Time signature: {}".format(time_signature))
time_signatures = np.array(time_signatures)
unique, counts = np.unique(time_signatures, return_counts=True)
dict(zip(unique, counts))
import matplotlib.pyplot as plt
time_signatures_dict = dict()
time_signatures_dict["other"] =  0
keys = [
    "3/8",
    "4/8\n 2/4",
    "6/8\n 3/4",
    "8/8\n 4/4",
    "12/8\n 6/4",
    "other"
]

for ts in zip(unique, counts):
    if ts[0] == 6:
        # append the new number to the existing array at this slot
        time_signatures_dict[keys[0]] = ts[1]
    elif ts[0] == 8:
        # append the new number to the existing array at this slot
        time_signatures_dict[keys[1]] = ts[1]
    elif ts[0] == 12:
        # append the new number to the existing array at this slot
        time_signatures_dict[keys[2]] = ts[1]
    elif ts[0] == 16:
        # append the new number to the existing array at this slot
        time_signatures_dict[keys[3]] = ts[1]
    elif ts[0] == 24:
        # append the new number to the existing array at this slot
        time_signatures_dict[keys[4]] = ts[1]
    else:
        # create a new array in this slot
        time_signatures_dict[keys[5]] += ts[1]
        
desired_order_list = [1, 2, 3, 4, 5, 6]

time_signatures_dict = {keys[k-1]: time_signatures_dict[keys[k-1]] for k in desired_order_list}
import matplotlib.pyplot as plt

x_tmp = time_signatures_dict.keys()
y_tmp = time_signatures_dict.values()

fig, ax = plt.subplots()
fig.set_facecolor('white')
width = 0.75 # the width of the bars 
ax.bar(x_tmp, y_tmp, width, color="black", fill=False)
plt.title('Piano-midi.de')
plt.xlabel('Time signature')
plt.ylabel('Number of MIDI files')   
plt.ylim([0, 105])
for i, v in enumerate(y_tmp):
    ax.text(i - 0.2 , v + 2 , str(v), color='black')

fig = plt.gcf()
fig.set_size_inches(7, 4)
plt.rcParams.update({'font.size': 22})
plt.savefig(os.path.join('Piano_midi.png'), dpi=300, format='png', bbox_inches='tight', transparent= False)
#order training pieces by time signature
piano_midi_pieces_filtered = piano_midi_only_pieces.copy()
piano_midi_pieces_remaining = piano_midi_only_pieces.copy()
keys = list(piano_midi_only_pieces.keys())
verbose = False

for k in keys:
    piece = piano_midi_only_pieces[str(k)]
    time_signature = int(max([b[0][3] for b in  piece]))
    if not (time_signature in [16]):
        piano_midi_pieces_filtered.pop(k)
    else:
        piano_midi_pieces_remaining.pop(k)
    if verbose:
        print("Piece: {}".format(k) + "  Time signature: {}".format(time_signature))
print('Number of Piano Midi pieces left after filtering = ', len(piano_midi_pieces_filtered))<jupyter_output>Number of Piano Midi pieces left after filtering =  89<jupyter_text>Train/Validation pieces split<jupyter_code># Define pieces relevant for training and validation
pieces_tmp = piano_midi_pieces_filtered.copy()
#pieces_tmp = chopin_pieces_filtered.copy()
# Free up memory
def free_memory():
    all_pieces.clear()
    chopin_only_pieces.clear()
    genere_pieces.clear()
    piano_midi_only_pieces.clear()
    piano_midi_pieces_filtered.clear()
    piano_midi_pieces_remaining.clear()
    pieces_tmp.clear()
# Either select one validation and one training piece or 
# set aside a random set of pieces for validation purposes
n_subset = 30
#selection = 'single_piece' 
#selection = 'subset'
selection = 'all'

random.seed(1337)
if selection == 'single_piece':
    validation_pieces = copy.deepcopy({'chop2803' : all_pieces['chop2803']})
    training_pieces   = copy.deepcopy({'chop2804' : all_pieces['chop2804']})
    free_memory()
elif selection == 'subset' or selection == 'all':
    if selection == 'subset':
        training_pieces = copy.deepcopy({k: pieces_tmp[k] for k in random.sample(list(pieces_tmp.keys()), n_subset)})
        free_memory()
    elif selection == 'all':
        training_pieces = pieces_tmp.copy()
    num_validation_pieces = len(training_pieces) // 10

    validation_pieces={}
    for v in range(num_validation_pieces):
        index = random.choice(list(training_pieces.keys()))
        validation_pieces[index] = training_pieces.pop(index)
print('Number of training   pieces = ', len(training_pieces))    
print('Number of validation pieces = ', len(validation_pieces))<jupyter_output>Number of training   pieces =  81
Number of validation pieces =  8<jupyter_text>Check that features (X) and lables (y) generation work<jupyter_code># Generate sample Note State Matrix for dimension measurement and numerical checking purposes
y = batch.getPieceBatch(training_pieces, batch_size = 2, num_time_steps = 96) 
X = prep.inputKernel(y, Midi_low = 21, Midi_high = 108)


print('Dimensions y: (sample_size, num_notes, num_timesteps, play_articulate_velocity) = ', y.shape)
print('Dimensions X: (sample_size, num_notes, num_timesteps, feature_dim             ) = ', X.shape)
# Feature vector for the 1st batch the 31rd note and the 51st 48th-note timestep (cast to int)
prep.noteRNNInputSummary(X[0,46,3,:])
# Compare to all notes played at timestep 51
[(i,j) for i,j in enumerate(list(tf.cast(y[0,:,0,:], dtype=tf.int32).numpy())) if j[0]== 1]<jupyter_output><empty_output><jupyter_text>Test generating midi files from y_train<jupyter_code># Backtransform the 128 48th notes from the first random sampled training_piece to MIDI
test_midi = tf.cast(tf.transpose(y, perm=[0,2,1,3])[0,:,:,:], dtype=tf.int32).numpy()
midi.noteStateMatrixToMidi(test_midi, Music_Out_Directory + "random_test")<jupyter_output><empty_output><jupyter_text>Check number of notes played, articulated and velocity per timestep<jupyter_code>test = tf.transpose(y, perm=[0,2,1,3])[0,:,:,:].numpy()
notes_per_timestep = [(i,tf.reduce_sum(test[i,:,0]).numpy()) for i in range(test.shape[0])]
articulate_per_timestep = [(i,tf.reduce_sum(test[i,:,1]).numpy()) for i in range(test.shape[0])]
mean_tempo_per_timestep = [(i,tf.reduce_mean(test[i,np.array(test[i,:,0], dtype=bool),2]).numpy()) for i in range(test.shape[0])]<jupyter_output><empty_output><jupyter_text>Model architecture  Parameters<jupyter_code>Midi_low = lowerBound
Midi_high = upperBound - 1
num_notes = Midi_high + 1 - Midi_low # X.shape[1] = Midi_high + 1 - Midi_low 
num_timesteps = -1 # keep the num_timesteps variable, in training set to X.shape[2]
input_size = 108 # X.shape[3]
drop_out_rate = 0.5

num_t_units = [128, 128] # [256, 128]
num_n_units = [64,  64] # [128,  64]
dense_units = 3    # (play,articulate,velocity)

TIME_BATCH_SHAPE = (num_timesteps, input_size)
NOTE_BATCH_SHAPE = (num_notes, num_t_units[1])<jupyter_output><empty_output><jupyter_text>Inputs<jupyter_code>inputs = tf.keras.Input(shape=(num_notes, None, input_size), name="inputs")
inputs_shape = sub.GetShape()(inputs)<jupyter_output><empty_output><jupyter_text>Timewise LSTM<jupyter_code># Reshaping
x = sub.BatchReshape(TIME_BATCH_SHAPE, False)(inputs)

# Timewise LSTMs
x = tf.keras.layers.LSTM(num_t_units[0], return_sequences=True, dropout= drop_out_rate)(x)
x = tf.keras.layers.LSTM(num_t_units[1], return_sequences=True, dropout= drop_out_rate)(x)<jupyter_output><empty_output><jupyter_text>Notewise LSTM<jupyter_code># Reshaping
x = sub.BatchReshape([num_notes, num_timesteps , num_t_units[1]], True)(x)
x = tf.keras.layers.Permute((2,1,3))(x)
x = sub.BatchReshape(NOTE_BATCH_SHAPE, False)(x)

# Notewise LSTMs
x = tf.keras.layers.LSTM(num_n_units[0], return_sequences=True, dropout= drop_out_rate)(x)
x = tf.keras.layers.LSTM(num_n_units[1], return_sequences=True, dropout= drop_out_rate)(x)<jupyter_output><empty_output><jupyter_text>Simulate conditional probabilty using dense layers<jupyter_code>x = sub.SliceNotesTensor()(x)

x_tmp = tf.keras.layers.Dense(units=dense_units, activation=None)(x[0])
note = sub.SampleNote()(x_tmp)

x_list = [x_tmp]
note_list = [sub.SliceNoteVelocityTensor()(note)]

for n in range(1,len(x)):
    x_tmp = tf.keras.layers.Concatenate(axis=-1)([x[n], note])
    x_tmp = tf.keras.layers.Dense(units=dense_units, activation=None)(x_tmp)
    note = sub.SampleNote()(x_tmp)
    x_list.append(x_tmp)
    note_list.append(sub.SliceNoteVelocityTensor()(note))<jupyter_output><empty_output><jupyter_text>Outputs<jupyter_code># Output 1    
x_list = sub.ExpandDims()(x_list, axis=1)
x = tf.keras.layers.Concatenate(axis=1)(x_list)
x = sub.BatchReshape([num_timesteps, num_notes , dense_units], True)(x, inputs_shape)
x_1, x_2 = sub.SliceNotesVelocityTensor()(x)
output_1 = tf.keras.layers.Permute((2,1,3), name='play_articulate_prob')(x_1)

#Output 2
x_2 = sub.BackTransformVelocity()(x_2)
output_2 = tf.keras.layers.Permute((2,1,3), name='velocity')(x_2)

# Output 3
note_list = sub.ExpandDims()(note_list, axis=1)
note = tf.keras.layers.Concatenate(axis=1)(note_list)
note = sub.BatchReshape([num_timesteps, num_notes , 2], True)(note, inputs_shape)
output_3 = tf.keras.layers.Permute((2,1,3), name='play_articulate_sampled')(note)<jupyter_output><empty_output><jupyter_text>Full LSTM<jupyter_code>lstm = tf.keras.Model(inputs, [output_1, output_2, output_3], name="full")
lstm.summary()
plot = False
if plot:
    plot_res = tf.keras.utils.plot_model(lstm, to_file="model.svg")<jupyter_output><empty_output><jupyter_text>Model training<jupyter_code># Reload alread trained model 
reload = True
if reload:
    Date = "20210920"
    name = "alpha_0.01_beta_1.0_epoch_64_model"
    load_path = Checkpoint_Directory + Date + "/" + name
    lstm = tf.keras.models.load_model(load_path)
# loss
losses = (sub.CustomSigmoidFocalCrossEntropy(from_logits = True, 
                                           gamma = 0, 
                                           alpha = 0),
          sub.MeanSquaredErrorVelocity())

# metric
metrics = (sub.CustomBinaryAccuracy(threshold=0.5), 
           tfa.metrics.MultiLabelConfusionMatrix(num_classes=2),
           sub.root_mean_squared_error_velocity_metric)

# optimizer
optimizer = tf.keras.optimizers.Adadelta(learning_rate=1, epsilon=1e-04)
### The tf.function 
@tf.function(experimental_relax_shapes=True)
def train_on_batch(X, y, alpha, beta):
    
    with tf.GradientTape() as tape:
        
        y_pred_p_a, y_pred_velocity, sampled_p_a = lstm(X, training=True)
        
        loss_p_a        = losses[0](y, y_pred_p_a)
        loss_velocity   = losses[1](y, y_pred_velocity)
        loss_total      = beta * loss_p_a + alpha * tf.math.sqrt(loss_velocity) / 127 
                
        metric_p_a      = metrics[0](y, y_pred_p_a)
        confs           = metrics[1](tf.reshape(y[:,:,:,0:2],  [-1,2]), 
                                    tf.reshape(sampled_p_a, [-1,2]))
        p_conf, a_conf  = confs[0], confs[1]
        metric_velocity = metrics[2](y, y_pred_velocity)
        

    grads = tape.gradient(loss_total, lstm.trainable_weights)
    optimizer.apply_gradients(zip(grads, lstm.trainable_weights))

    return (loss_p_a, 
            tf.math.sqrt(loss_velocity), 
            tf.reduce_mean(metric_p_a), 
            tf.reduce_mean(metric_velocity), 
            [p_conf, a_conf])

@tf.function(experimental_relax_shapes=True)
def validate_on_batch(X, y, alpha):
    
    y_pred_p_a, y_pred_velocity, sampled_p_a = lstm(X, training=False)
    
    loss_p_a       = losses[0](y, y_pred_p_a)
    loss_velocity  = losses[1](y, y_pred_velocity)
    
    metric_p_a     = metrics[0](y, y_pred_p_a)
    confs          = metrics[1](tf.reshape(y[:,:,:,0:2],  [-1,2]), 
                                tf.reshape(sampled_p_a, [-1,2]))
    p_conf, a_conf = confs[0], confs[1]    
    metric_velocity = metrics[2](y, y_pred_velocity)
    
    return (loss_p_a, 
            tf.math.sqrt(loss_velocity),
            tf.reduce_mean(metric_p_a), 
            tf.reduce_mean(metric_velocity), 
            [p_conf, a_conf])
current_time_str = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")

epochs = 64 
batch_size = 2
epoch_save_list = [1, 2, 4, 8, 16, 32, 64] #Checkpoint_Directory
alpha = tf.constant(1/100)
beta = tf.constant(1.)

n_train_batches = prep.getNumberOfBatches(training_pieces, batch_size)
n_val_batches = prep.getNumberOfBatches(validation_pieces, batch_size)

# Values for loss, metric and confusion matrix
train_loss_p_a_array   = np.full((epochs, n_train_batches), 10.0)
train_loss_vel_array   = np.full((epochs, n_train_batches), 10.0)
train_metric_p_a_array = np.full((epochs, n_train_batches), 10.0)
train_metric_vel_array = np.full((epochs, n_train_batches), 10.0)
val_loss_p_a_array     = np.full((epochs, n_val_batches), 10.0)
val_loss_vel_array     = np.full((epochs, n_val_batches), 10.0)
val_metric_p_a_array   = np.full((epochs, n_val_batches), 10.0)
val_metric_vel_array   = np.full((epochs, n_val_batches), 10.0)
train_p_conf_array     = np.full((epochs, 2, 2), 10.0)
train_a_conf_array     = np.full((epochs, 2, 2), 10.0)
val_p_conf_array       = np.full((epochs, 2, 2), 10.0)
val_a_conf_array       = np.full((epochs, 2, 2), 10.0)
(n_train_batches, n_val_batches)
### CUSTOM TRAINING LOOP

# Timing
start_time = time.time()
time_old = start_time

for epoch in range(epochs):
    print('\rStart of Epoch [%d/%d]'% (epoch + 1, epochs))
    print('\n')

    #ensure that every epoch the same training data is used
    n = 0
    for k in training_pieces.keys():
        start_old = 0
        piece = training_pieces[str(k)]
        sixteenth_index = [b[0][3] for b in  piece]
        num_time_steps =  max(sixteenth_index)*3*2 + 1
        while start_old < (len(piece)- num_time_steps):
            print('Training batch: %d/%d' % (n + 1, n_train_batches), end='\r')
            train_dataset, start_old = prep.createDataSet2(piece, 
                                                           num_time_steps=num_time_steps, 
                                                           batch_size=batch_size, 
                                                           start_old=start_old)
            for _, (X_train, y_train) in enumerate(train_dataset):
                l_1, l_2, m_1, m_2, confusion_mat = train_on_batch(X_train, 
                                                                   y_train, 
                                                                   alpha,
                                                                   beta)
                train_loss_p_a_array[epoch, n]    = l_1
                train_loss_vel_array[epoch, n]    = l_2
                train_metric_p_a_array[epoch, n]  = m_1
                train_metric_vel_array[epoch, n]  = m_2
            n += 1
    print('')

    # storing the confusion matrix of validation set for predicting play/ articulate
    train_p_conf_array[epoch, : , :] = confusion_mat[0]
    train_a_conf_array[epoch, : , :] = confusion_mat[1]

    # reset metrics
    metrics[0].reset_states()
    metrics[1].reset_states()

    n = 0
    for k in validation_pieces.keys():
        start_old = 0
        piece = validation_pieces[str(k)]
        sixteenth_index = [b[0][3] for b in  piece]
        num_time_steps =  max(sixteenth_index)*3*2 + 1
        while start_old < (len(piece)- num_time_steps):
            print('Validation batch: %d/%d' % (n + 1, n_val_batches), end='\r')
            val_dataset, start_old = prep.createDataSet2(piece, 
                                                         num_time_steps=num_time_steps, 
                                                         batch_size=batch_size, 
                                                         start_old=start_old)
            for _, (X_val, y_val) in enumerate(val_dataset):
                l_1, l_2, m_1, m_2, confusion_mat = validate_on_batch(X_val, y_val, alpha)
                val_loss_p_a_array[epoch, n]      = l_1
                val_loss_vel_array[epoch, n]      = l_2
                val_metric_p_a_array[epoch, n]    = m_1 
                val_metric_vel_array[epoch, n]    = m_2 
            n += 1

    print('')
    print('Seed:' + str(random.randrange(0,1000000,1)))

    # storing the confusion matrix of validation set for predicting play/ articulate
    val_p_conf_array[epoch, : , :] = confusion_mat[0]
    val_a_conf_array[epoch, : , :] = confusion_mat[1]

    # reset metrics
    metrics[0].reset_states()
    metrics[1].reset_states()

    print('Training   Loss p_a: '     + str(np.mean(train_loss_p_a_array[epoch,:])))
    print('Validation Loss p_a: '     + str(np.mean(val_loss_p_a_array[epoch,:])))
    print('Training   Loss vel: '     + str(np.mean(train_loss_vel_array[epoch,:])))
    print('Validation Loss vel: '     + str(np.mean(val_loss_vel_array[epoch,:])))
    print('Training   Accuracy p_a: ' + str(np.mean(train_metric_p_a_array[epoch,:])))
    print('Validation Accuracy p_a: ' + str(np.mean(val_metric_p_a_array[epoch,:])))
    print('Training   Accuracy vel: ' + str(np.mean(train_metric_vel_array[epoch,:])))
    print('Validation Accuracy vel: ' + str(np.mean(val_metric_vel_array[epoch,:])))
    print('Training Confusion Matrix Play:\n')
    print(train_p_conf_array[epoch, : , :])
    print('')
    print('Training Confusion Matrix Articulate:\n')
    print(train_a_conf_array[epoch, : , :])
    print('')
    print('Validation Confusion Matrix Play:\n')
    print(val_p_conf_array[epoch, : , :])
    print('')
    print('Validation Confusion Matrix Articulate:\n')
    print(val_a_conf_array[epoch, : , :])
    print('')


    time_new = time.time()
    duration = time_new - time_old
    time_old = time_new
    print('Time: ' +  str(round(duration, 3)) + 's')


    if (epoch + 1) in epoch_save_list:
        
        # epoch = (epoch + 1) + 16

        # save model weights
        iteration_name = 'alpha_' + str(alpha.numpy()) + '_beta_' + str(beta.numpy()) + '_epoch_' + str(epoch + 1) +'_'
        save_path = Checkpoint_Directory + current_time_str[:-7] + '/' +  iteration_name + 'model'
        lstm.save(save_path)

        # save audios for train from y_pred_train, y_train
        y_pred_note_train, y_pred_velocity_train, _ = lstm(X_train, training=False)
        y_pred_train = tf.concat([y_pred_note_train, y_pred_velocity_train], axis=-1)
        midi.generate_audio(y_pred_train, 
                            Music_Out_Training_Directory + current_time_str[:-7] + '/', 
                            iteration_name + 'train_pred', 
                            sample=True)
        midi.generate_audio(y_train, 
                            Music_Out_Training_Directory + current_time_str[:-7] + '/', 
                            iteration_name + 'train_true', 
                            sample=False)

        # save audios for val from y_pred
        y_pred_note_val, y_pred_velocity_val, _ = lstm(X_val, training=False)
        y_pred_val = tf.concat([y_pred_note_val, y_pred_velocity_val], axis=-1)
        midi.generate_audio(y_pred_val,    
                            Music_Out_Training_Directory + current_time_str[:-7] + '/', 
                            iteration_name + 'val_pred', 
                            sample=True)
        midi.generate_audio(y_val,    
                            Music_Out_Training_Directory + current_time_str[:-7] + '/', 
                            iteration_name + 'val_true', 
                            sample=False)

        # save the arrays 
        save_path = Numpy_Directory + current_time_str[:-7] + '/' 
        try:
            os.mkdir(save_path)    
        except:
            print('destination folder exists')
        np.savez(save_path + iteration_name + 'array', 
                 train_loss_p_a_array, 
                 train_loss_vel_array,
                 train_metric_p_a_array,
                 train_metric_vel_array,
                 val_loss_p_a_array, 
                 val_loss_vel_array,
                 val_metric_p_a_array,
                 val_metric_vel_array,
                 train_p_conf_array,
                 train_a_conf_array,
                 val_p_conf_array,
                 val_a_conf_array,
                 X_train.numpy(),
                 y_train.numpy(),
                 y_pred_train.numpy(),
                 X_val.numpy(),
                 y_val.numpy(),
                 y_pred_val.numpy()
                )
    print('\n\n')
    tf.keras.backend.clear_session()

total_time = time.time() - start_time 
print('Total time: ' + str(datetime.timedelta(seconds=total_time)))<jupyter_output>Start of Epoch [1/64]


Training batch: 889/889
Validation batch: 56/56
Seed:864418
Training   Loss p_a: 0.028782486661805596
Validation Loss p_a: 0.027655624235714122
Training   Loss vel: 16.60463765179749
Validation Loss vel: 12.833436046327863
Training   Accuracy p_a: 0.9957133154767168
Validation Accuracy p_a: 0.9961178079247475
Training   Accuracy vel: 16.60463765179749
Validation Accuracy vel: 12.833436046327863
Training Confusion Matrix Play:

[[14449938.   105020.]
 [  104375.   361211.]]

Training Confusion Matrix Articulate:

[[14858947.    72894.]
 [   71873.    16830.]]

Validation Confusion Matrix Play:

[[911369.   5384.]
 [  4483.  24940.]]

Validation Confusion Matrix Articulate:

[[937505.   3480.]
 [  4037.   1154.]]

Time: 171.148s<jupyter_text>Evaluate Model<jupyter_code>trials = 10000
batch_size = 2
alpha = tf.constant(1/100)
beta = tf.constant(1.)
train_loss_p_a_array2 = np.full(trials, 10.0)
train_log_likelihood  = np.full(trials, 10.0)
val_loss_p_a_array2   = np.full(trials, 10.0)
val_log_likelihood    = np.full(trials, 10.0)
### CUSTOM EVALUATION LOOP

# Timing
start_time = time.time()
time_old = start_time

for trial in range(trials):
    print('Start of Trail [%d/%d]'% (trial + 1, trials), end='\r')

    #ensure that every trial random training data is used
    k = random.choice(list(training_pieces.keys()))
    piece = training_pieces[str(k)]
    sixteenth_index = [b[0][3] for b in  piece]
    num_time_steps =  max(sixteenth_index)*3*2 + 1
    piece = {k: training_pieces[str(k)]}
    train_dataset = prep.createDataSet(piece,
                                       sample_size=batch_size,
                                       num_timesteps=num_time_steps,
                                       batch_size=batch_size)
    for _, (X_train, y_train) in enumerate(train_dataset):
        l_1, _, _, _, _ = validate_on_batch(X_train,y_train,alpha)
        train_loss_p_a_array2[trial] = l_1
        train_log_likelihood[trial]  = -num_notes*l_1

    #ensure that every trial random validation data is used
    k = random.choice(list(validation_pieces.keys()))
    piece = validation_pieces[str(k)]
    sixteenth_index = [b[0][3] for b in  piece]
    num_time_steps =  max(sixteenth_index)*3*2 + 1
    piece = {k: validation_pieces[str(k)]}
    val_dataset= prep.createDataSet(piece, 
                                    sample_size=batch_size,
                                    num_timesteps=num_time_steps, 
                                    batch_size=batch_size)
    for _, (X_val, y_val) in enumerate(val_dataset):
        l_1, _, _, _, _ = validate_on_batch(X_val, y_val, alpha)
        val_loss_p_a_array2[trial] = l_1
        val_log_likelihood[trial]  = -num_notes*l_1

    time_new = time.time()
    duration = time_new - time_old
    time_old = time_new
    tf.keras.backend.clear_session()

total_time = time.time() - start_time 
print('Total time: ' + str(datetime.timedelta(seconds=total_time)))
np.around(np.max(train_log_likelihood),2), np.around(np.median(train_log_likelihood),2)
np.around(np.max(val_log_likelihood),2), np.around(np.median(val_log_likelihood),2)
dec = 2
for i in range(10):
    print('\n\n')
    print("Max Train:    ", np.around(np.max(train_log_likelihood[(i*1000):((i+1)*1000)]),decimals=dec))
    print("Max val:      ", np.around(np.max(val_log_likelihood[(i*1000):((i+1)*1000)]),decimals=dec))
    print("Median Train: ", np.around(np.median(train_log_likelihood[(i*1000):((i+1)*1000)]),decimals=dec))
    print("Median val:   ", np.around(np.median(val_log_likelihood[(i*1000):((i+1)*1000)]),decimals=dec))<jupyter_output>Max Train:     -0.45
Max val:       -0.74
Median Train:  -1.8
Median val:    -1.68



Max Train:     -0.45
Max val:       -0.38
Median Train:  -1.82
Median val:    -1.72



Max Train:     -0.45
Max val:       -0.72
Median Train:  -1.76
Median val:    -1.67



Max Train:     -0.45
Max val:       -0.72
Median Train:  -1.78
Median val:    -1.71



Max Train:     -0.45
Max val:       -0.81
Median Train:  -1.82
Median val:    -1.68



Max Train:     -0.45
Max val:       -0.71
Median Train:  -1.84
Median val:    -1.7



Max Train:     -0.45
Max val:       -0.65
Median Train:  -1.78
Median val:    -1.7



Max Train:     -0.45
Max val:       -0.66
Median Train:  -1.85
Median val:    -1.73



Max Train:     -0.45
Max val:       -0.65
Median Train:  -1.81
Median val:    -1.7



Max Train:     -0.45
Max val:       -0.78
Median Train:  -1.77
Median val:    -1.69
{'fraction_non_alphanumeric': 0.11775304716481187, 'fraction_numerical': 0.02882882882882883, 'mean_word_length': 3.1633756205184778, 'pattern_counts': {'":': 0, '<': 89, '<?xml version=': 0, '>': 87, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 59, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '23', 'dir': 'jupyter-structured-clean-dedup', 'id': '544349', 'license': 'MIT', 'n_tokens_mistral': 14098, 'n_tokens_neox': 13147, 'n_words': 2644, 'path': 'notebooks/modelling.ipynb', 'repo_name': 'MirkoNJ/generating-music-tf-2', 'size': '37740'}