<jupyter_start><jupyter_text>Version 2 of DDR Spectrogram  We Will Train the Steps from Spectrograms of MP3 Files<jupyter_code>import librosa
import librosa.display
import numpy as np
import matplotlib.pyplot as plt
from pathlib import Path
import pandas as pd
import re
#Get the list of all of the step files
step_files = list(Path("C:/Users/brent/Desktop/StepMania 5").rglob("*.[dD][wW][iI]"))

#Get the list of all of the step files
song_files = list(Path("C:/Users/brent/Desktop/StepMania 5").rglob("*.[mM][pP][3]"))
#There are not equal amounts of mp3s and matching step files
print(len(song_files), len(step_files))<jupyter_output><empty_output><jupyter_text>We modify the code below to capture the gap and the bpm<jupyter_code>def process_song(path, title):
    #Open File
    text_file = open(path, "r")
    lines = text_file.readlines()
    text_file.close()
    
    #Combine all text into single line
    song = "".join(lines)
    
    #Remove newline characters
    song = re.sub('\n', '', song)
    
    #Split on semicolon and then add the semicolons back into the respective lines
    song = song.split(';')
    song = [line+';' for line in song][:-1]
    
    #Remove lines that start with 2 // (some files had this for some reason)
    song = [line for line in song if (line.find('//') == -1)]
    
    #Create a dataframe of the song
    df = pd.DataFrame()
    df[title] = song
    return df

def pull_all_step_patterns(song, row, path):
    song = song[row].str.split(":", n = 3, expand = True)
    
    #Get BPM
    bpm = song[song[0] == "#BPM"].iloc[0,1]
    #Remove ;
    bpm = float(bpm[:-1])
    
    #Get Gap
    gap = song[song[0] == "#GAP"].iloc[0,1]
    #Remove ;
    gap = float(gap[:-1])
    
    song = song[song[0].isin(["#SINGLE","#SOLO"])]
    
    song['4'] = bpm
    song['5'] = gap
    song['6'] = path
    
    return song

    
def join_all_step_patterns(step_files):
    songs = pd.DataFrame()
    for row, path in enumerate(step_files):
        df = process_song(path, row)
        df = pull_all_step_patterns(df, row, path)
        
        #songs = pd.merge(songs, df, left_index=True, right_index=True, how="outer")
        songs = pd.concat([songs,df])
    
    return songs
songs = join_all_step_patterns(step_files)<jupyter_output><empty_output><jupyter_text>Iterate through step patterns and Specify location of mp3<jupyter_code>songs = songs.reset_index()
songs.drop(['index'], axis=1, inplace=True)
#Check that all step files have matching mp3 files
#Create a dataframe that has both the step file path as well as the mp3 path
for index, row in songs.iterrows():
    #Get location of each dwi file and then determine location of mp3 file
    steppath = songs.iloc[index, 6]
    songpath = Path(steppath.parent.as_posix() + '/' + steppath.stem + '.mp3')
    
    if songpath in song_files:
        songs.loc[index, '7'] = songpath
    else:
        songs.loc[index, '7'] is None<jupyter_output><empty_output><jupyter_text>Now we remove the step files that don't have interpreted mp3's<jupyter_code>songs = songs[~songs['7'].isnull()]
songs = songs.reset_index()
songs.drop(['index'], axis=1, inplace=True)<jupyter_output><empty_output><jupyter_text>Create Spectrograms for Matching Songs<jupyter_code>def mp3_to_spectrogram(path, gap):
    #offset should be duration of gap specifided in .DWI FIle
    #Gap is in milliseconds while offset is in seconds

    #Choose the file to create the graphics
    filename = path

    #y, sr = librosa.load(filename,offset=30, duration=12.0)
    y, sr = librosa.load(filename, offset=gap/1000)
    onset_frames = librosa.onset.onset_detect(y=y, sr=sr)
    librosa.frames_to_time(onset_frames, sr=sr)

    o_env = librosa.onset.onset_strength(y, sr=sr)
    times = librosa.frames_to_time(np.arange(len(o_env)), sr=sr)
    onset_frames = librosa.onset.onset_detect(onset_envelope=o_env, sr=sr)
    D = np.abs(librosa.stft(y))

    #Create the spectrogram
    plt.clf()
    plt.axis('off')
    librosa.display.specshow(librosa.amplitude_to_db(D, ref=np.max),x_axis='time', y_axis='log')
    plt.savefig(Path('image')/(filename.stem+'_1.png'))

    #Create a beat offset graph
    plt.clf()
    plt.axis('off')
    plt.plot(times, o_env)
    plt.savefig(Path('image')/(filename.stem+'_2.png'))<jupyter_output><empty_output><jupyter_text>This next block creates about 1500 spectrograms and takes a long time.  On the order of hours<jupyter_code>#Iterate through each row of the dataframe and create both versions of the spectrogram.
for index, row in songs.iterrows():
    songpath = songs.iloc[index, 7]
    gap = songs.iloc[index, 5]
    
    imagepath1 = Path('image' + '/' + songpath.stem + '_1.png')
    imagepath2 = Path('image' + '/' + songpath.stem + '_2.png')

    songs.loc[index, '8'] = imagepath1
    songs.loc[index, '9'] = imagepath2
    
    #Generate the actual spectrograms
    #Toggled Off because it has been previously run
    #mp3_to_spectrogram(songpath, gap)<jupyter_output><empty_output><jupyter_text>Create the final file that we will use for our model<jupyter_code>songs.columns = ['mode','label','difficulty','text','bpm','gap','step_path','mp3_path','spectrogram_path','spectrogram_path2']
#songs = songs[[1,3]]
#songs.columns = ['label','text']

#Split the song into characters with semicolons
songs['text'] = songs['text'].apply(lambda x: ";".join(x))

#Remove the trailing semicolon as we can add it back in when we are done predicting songs
songs['text'] = songs['text'].apply(lambda x: x[:-2])
songs.to_csv("spectrogram.csv", index=False)<jupyter_output><empty_output><jupyter_text>We Will Create our Data Bunch from the Spectrogram Images<jupyter_code>import pandas as pd
songs = pd.read_csv('spectrogram.csv', index_col=False)
from fastai.vision import *<jupyter_output><empty_output><jupyter_text>Use multi category label through use of ';' as a delimiter<jupyter_code>#Create our databunch
#We choose not to use any transforms as the images
#were created in a uniform fashion from mp3s
data = (ImageList.from_df(df=songs, path='.', cols = ['spectrogram_path'])
        .split_by_rand_pct(valid_pct=0.2)
        .label_from_df(cols=['text'],label_delim=';')
        .transform(([],[]))
        .databunch(bs=4))
learn = cnn_learner(data, models.resnet152, metrics=[mean_absolute_error])
learn.lr_find()
learn.recorder.plot(skip_end=15)
learn.fit_one_cycle(100,1e-3)
learn.save('spectrogram_multicategory')
learn = cnn_learner(data, models.resnet152, metrics=[mean_absolute_error])
learn.load('spectrogram_multicategory')
pass
data.x[1000]
data.y[100]
 data.train_ds.y.items[0]
data.train_ds.y.__class__<jupyter_output><empty_output><jupyter_text>Using Regular Category Labels<jupyter_code>#Create our databunch
#We choose not to use any transforms as the images
#were created in a uniform fashion from mp3s
data = (ImageList.from_df(df=songs, path='.', cols = ['spectrogram_path'])
        .split_by_rand_pct(valid_pct=0)
        .label_from_df(cols=['text'])
        .transform(([],[]))
        .databunch(bs=8))
learn = cnn_learner(data, models.resnet152, metrics=[accuracy])
learn.lr_find()
learn.recorder.plot(skip_end=15)
learn.fit_one_cycle(20,1e-4)
learn.save('spectrogram2')
learn.fit_one_cycle(20,1e-4)
learn.save('spectrogram2')
learn.fit_one_cycle(20,1e-4)
learn.save('spectrogram2')
learn.fit_one_cycle(20,1e-4)
learn.fit_one_cycle(20,1e-4)
learn.save('spectrogram2')
learn.unfreeze()
learn.fit_one_cycle(100,1e-5)
learn.save('spectrogram2')<jupyter_output><empty_output><jupyter_text>We Now Have Our Trained Model and Area Ready for Prediction<jupyter_code>learn.load('spectrogram2')
pass
data.x[1000]
learn.predict(data.x[1000])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1343128331816409, 'fraction_numerical': 0.017943050318554154, 'mean_word_length': 4.553790613718411, 'pattern_counts': {'":': 1, '<': 45, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '295088', 'license': 'MIT', 'n_tokens_mistral': 2861, 'n_tokens_neox': 2754, 'n_words': 649, 'path': 'captioning/DDR_Spectrogram-v2.ipynb', 'repo_name': 'brentbiseda/DDR_Deep_Learning', 'size': '7691'}