<jupyter_start><jupyter_text>**Please update the start path and destination path**<jupyter_code>start ='/content/drive/MyDrive/MUS' # Divide all 9 directories of the MAPS Dataset into train/test/val and provide its path (format eg. test/.wav)
dest = '/content/drive/MyDrive/cqt_test_train' # Destination Path for storing the .npy files
RangeMIDInotes=[21,108]
sr=16000.
bins_per_octave=36
n_octave=7
val_rate=1./7
pretty_midi.pretty_midi.MAX_TICK = 1e10
n_bins= n_octave * bins_per_octave
hop_length = 512
win_width = 32
kernel_size=7
overlap=True
def midi2mat(midi_path_train, length, cqt_len, sr, RangeMIDInotes=RangeMIDInotes):
    midi_data = pretty_midi.PrettyMIDI(midi_path_train)
    pianoRoll = midi_data.instruments[0].get_piano_roll(fs=cqt_len * sr/length)
    Ground_truth_mat = (pianoRoll[RangeMIDInotes[0]:RangeMIDInotes[1] + 1, :cqt_len] > 0)
    return Ground_truth_mat
fil = [direc for direc in os.listdir(start)]
for direc in fil:
    j=0
    k=0
    startpath= os.path.join(start,direc)
    destpath = os.path.join(dest,direc)
    if not os.path.exists(destpath):
        os.makedirs(destpath)
    print(direc)
    files = [f for f in os.listdir(startpath)]
    for f in files:
        fpath=startpath
        f1=f
        if 1:
            ffile=os.path.join(fpath,f1)
            file_name,file_extensions=os.path.splitext(f1)
            if file_extensions == '.txt':
                continue
            if file_extensions==".mid":
                ffile=os.path.join(fpath,file_name+'.wav')
            x,sr = librosa.load(ffile,sr=sr)

            # perform stft to obtain 2D array of complex values(frequencies)
            cqt_file = librosa.cqt(x,sr=sr,fmin=librosa.note_to_hz('A0'),hop_length = hop_length,n_bins=n_bins,bins_per_octave=bins_per_octave,scale=False)
            # taking their absolute values
            cqt_abs = np.abs(cqt_file) 
            # converting the frequency data on a logarthmic scale for better visualisation
            cqt = np.transpose(librosa.amplitude_to_db(cqt_abs))

            midi_file = os.path.join(fpath,f1)

            if file_extensions==".wav":
                midi_file = os.path.join(fpath,file_name+'.mid')

            Ground_truth_mat=midi2mat(midi_file, len(x), cqt.shape[0], sr, RangeMIDInotes=RangeMIDInotes)
            midi_train = np.transpose(Ground_truth_mat)
            
            #midi length<stft length, cut stft
            if midi_train.shape[0]<cqt.shape[0]:
                cqt=cqt[:midi_train.shape[0],:]
        
            if file_extensions == ".wav" :
                ofolder = 'wav'
                subname = 'CQT'
                no=j
            elif file_extensions == ".mid" :
                ofolder = 'mid'
                subname = 'label'
                no=k

            opath = os.path.join(destpath,f,ofolder,file_name)+subname+'.npy'
            temp_path = os.path.join(destpath,f,ofolder)

            if not os.path.exists(temp_path):
                os.makedirs(temp_path)

            if file_extensions == ".wav":
                np.save(opath,cqt)
            elif file_extensions == ".mid":
                np.save(opath,midi_train)

            # print('Preprocessed ',f1)   

            # cut the tensor along the first axis by the win_width with a single frame hop
            matrix = np.array(np.load(opath))
            l=matrix.shape[0]
            cut_matrix=[]
            nb_win=int(l/win_width)   #integer division=floor

            if not overlap:
                for i in range(nb_win):
                    cut_matrix.append(matrix[i*win_width:(i+1)*win_width,:])
            else:
                w=matrix.shape[1]
                matrix_1=np.concatenate([np.zeros([int(kernel_size/2),w]),matrix,np.zeros([int(kernel_size/2),w])],axis=0)  #padding
                cut_matrix = []
                for i in range(nb_win):
                    cut_matrix.append(matrix_1[i * win_width:(i + 1) * win_width+kernel_size-1,:])

            cut_matrix = np.asarray(cut_matrix)
            os.remove(opath)
            # print("Removed ",f1)            
            if file_extensions == ".wav":
                if j == 0:
                    X = cut_matrix
                    #print(cut_matrix.shape)
                else:
                    X = np.concatenate((X,cut_matrix),axis=0)
                    #print(cut_matrix.shape)
                j=j+1
                
            elif file_extensions == ".mid":
                if k == 0:
                    Y = cut_matrix
                    #print(cut_matrix.shape)
                else:
                    Y = np.concatenate((Y,cut_matrix),axis=0)
                    #print(cut_matrix.shape)
                k=k+1
            print('Joined ',f1,"no ",no)
            # print('--------------')
            os.rmdir(temp_path)
        os.rmdir(os.path.join(destpath,f))
    X = np.expand_dims(X,axis=-2)
    Y = np.expand_dims(Y,axis=-2) 
    opath1= os.path.join(destpath,"X_final_CQT_")+direc+'.npy'
    opath2= os.path.join(destpath,"Y_final_CQT_")+direc+'.npy' 
    np.save(opath1,X)
    np.save(opath2,Y)
    # print('Saved X_train final')     
    # print('Saved Y_train final')
    # print('X_train_Shape -',X.shape)     
    # print('Y_train_Shape -',Y.shape)<jupyter_output>test_source
Joined  MAPS_MUS-alb_se2_ENSTDkCl.mid no  0
Joined  MAPS_MUS-alb_se2_ENSTDkCl.wav no  0
Joined  MAPS_MUS-bk_xmas1_ENSTDkCl.mid no  1
Joined  MAPS_MUS-bk_xmas1_ENSTDkCl.wav no  1
Joined  MAPS_MUS-bk_xmas4_ENSTDkCl.mid no  2
Joined  MAPS_MUS-bk_xmas4_ENSTDkCl.wav no  2
Joined  MAPS_MUS-bk_xmas5_ENSTDkCl.mid no  3
Joined  MAPS_MUS-bk_xmas5_ENSTDkCl.wav no  3
Joined  MAPS_MUS-bor_ps6_ENSTDkCl.mid no  4
Joined  MAPS_MUS-bor_ps6_ENSTDkCl.wav no  4
Joined  MAPS_MUS-chpn-e01_ENSTDkCl.mid no  5
Joined  MAPS_MUS-chpn-e01_ENSTDkCl.wav no  5
Joined  MAPS_MUS-chpn-p19_ENSTDkCl.mid no  6
Joined  MAPS_MUS-chpn-p19_ENSTDkCl.wav no  6
Joined  MAPS_MUS-deb_clai_ENSTDkCl.mid no  7
Joined  MAPS_MUS-deb_clai_ENSTDkCl.wav no  7
Joined  MAPS_MUS-deb_menu_ENSTDkCl.mid no  8
Joined  MAPS_MUS-deb_menu_ENSTDkCl.wav no  8
Joined  MAPS_MUS-grieg_butterfly_ENSTDkCl.mid no  9
Joined  MAPS_MUS-grieg_butterfly_ENSTDkCl.wav no  9
Joined  MAPS_MUS-liz_et6_ENSTDkCl.mid no  10
Joined  MAPS_MUS-liz_et6_ENSTDkCl[...]
{'fraction_non_alphanumeric': 0.1084740363172985, 'fraction_numerical': 0.017840076457470534, 'mean_word_length': 2.2216521292970755, 'pattern_counts': {'":': 6, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '65270', 'license': 'MIT', 'n_tokens_mistral': 2488, 'n_tokens_neox': 2333, 'n_words': 389, 'path': 'MODEL -2 (CNN)/CQT_CNN_PreprocessingCode_Model2.ipynb', 'repo_name': 'rrustagi20/Deep-learning---Automatic-Music-Transcription', 'size': '6278'}