<jupyter_start><jupyter_text>User Testing Planning/Coding Notebook Objectives:- Plan user testing- Code scripts to generate test maps:    - Randomly pull maps/music from 30 songs with maps for all difficulty levels from Beat Saver (tried all >70% rating but not enough were rated)    - Strip obstacles/lighting events from human made maps and set background environment all the same    - Map all songs with each model type    - Automatically categorize songs/beat structure?- For each user/volunteer:    - Randomly assign Random, HMM, Segmented HMM, Rate-Seg HMM, or Human-made    - Record email address, user-ID, and assigned map information (song, model/human, difficulty) in pd DataFrame    - Send assignment to user<jupyter_code>import pandas as pd
import numpy as np
import json
import requests
import zipfile
from io import BytesIO
import pickle
import src.beatmapsynth as bms
import os
import time
with open("../../data/metadata.pkl", 'rb') as file:
    metadata = pickle.load(file)
potential_songs = []

for x in metadata:
    if all(value == True for value in x['metadata']['difficulties'].values()):
        potential_songs.append(x)
len(potential_songs)
test_songs = np.random.choice(potential_songs, 30, replace = False)<jupyter_output><empty_output><jupyter_text>For each test song in the list:1. Download with key and unzip2. For each difficulty level:    - Edit/strip human map    - Map with each model    - Save zip folder<jupyter_code>keys = []
for x in test_songs:
    keys.append(x['key'])
def open_map_file(difficulty):
    """This function opens the map file listed in the info.dat file for the specificed difficulty level."""
    with open('./temp/info.dat', 'rb') as i:
        info = json.load(i)
    for x in info['_difficultyBeatmapSets']:
        if x['_beatmapCharacteristicName'] == 'Standard':
            for y in x['_difficultyBeatmaps']:
                if y['_difficulty'].casefold() == difficulty.casefold():
                    file_name = y['_beatmapFilename']
                    with open(f"./temp/{file_name}", 'rb') as f:
                        map_file = json.load(f)
                
    return map_file
def download_song_and_map(key):
    """Downloads the zipped folder of song and mapping data from the beatsaber api. Extracts files to a 'temp' folder 
    in the local directory."""
    response = requests.get(f"https://beatsaver.com/api/download/key/{key}")
    if response.status_code == 200:
        content_as_file = BytesIO(response.content)
        zip_file = ZipFile(content_as_file)
        for x in zip_file.filelist:
            print(zip_file.extract(x.filename, path = 'temp'))
        return response.status_code
    else:
        return print(f"API call failed at {key} with error code {response.status_code}")
download_song_and_map(keys[0])<jupyter_output>API call failed at 501f with error code 403<jupyter_text>Requests isn't working. Used Bash in terminal and wget to download zip folders instead.<jupyter_code>with open("../../data/raw/keys.txt", 'w') as w:
    w.write(str(keys))<jupyter_output><empty_output><jupyter_text>`for VARIABLE in 501f 56ce 3342 7548 3aab 74a9 192 57e0 6ea9 6e58 b9c 41f2 634f 5257 6020 54ad 3680 710c 5bdc 2660 1fea 7bd9 5cf4 6b64 43e1 4d44 4d1b 22b6 6a21 5ad9; do  wget https://beatsaver.com/api/download/key/$VARIABLE; done` Needed a few extra keys, three keys didn't work anymore.<jupyter_code>extra = np.random.choice(potential_songs, 3, replace = False)
extra_keys = []
for x in extra:
    extra_keys.append(x['key'])
for x in extra_keys:
    keys.append(x)
extra_keys<jupyter_output><empty_output><jupyter_text>`for VARIABLE in 1ff6 4f00 2f1d; do  wget https://beatsaver.com/api/download/key/$VARIABLE; done`<jupyter_code>two_more = np.random.choice(potential_songs, 2, replace = False)
two_more_keys = []
for x in two_more:
    two_more_keys.append(x['key'])
two_more_keys<jupyter_output><empty_output><jupyter_text>`for VARIABLE in 6eb6 32c7; do  wget https://beatsaver.com/api/download/key/$VARIABLE; done`<jupyter_code>one_more = np.random.choice(potential_songs, 1, replace = False)
one_more_key = []
for x in one_more:
    one_more_key.append(x['key'])

one_more_key
files = os.listdir("../../data/raw/")<jupyter_output><empty_output><jupyter_text>For file naming purposes:- m1 = Human Maps- m2 = Random Maps- m3 = HMM Maps- m4 = Segmented HMM Maps- m5 = Rate-Seg HMM Maps<jupyter_code>def strip_human_maps(files):
    """Function to strip human made maps of extra features in 'obstacles' and 'events' and save a new copy."""
    difficulties = ['Easy', 'Normal', 'Hard', 'Expert', 'ExpertPlus']
    for file in files:
        if file.endswith('.zip'):
            with open(f"../../data/raw/{file}", 'rb') as zipfolder:
                zip_file = zipfile.ZipFile(zipfolder)
                #Extract zipped files into temp directory
                for x in zip_file.filelist:
                    zip_file.extract(x.filename, path = 'temp')
            #Open info and save information
            with open('./temp/info.dat', 'rb') as i:
                info = json.load(i)
            song_name = info['_songName']
            bpm = info['_beatsPerMinute']
            song_file = info['_songFilename']
            #For each difficulty, strip extra features, write new info and map file, then zip together files
            for difficulty in difficulties:
                try:
                    map_file = open_map_file(difficulty)
                    map_file['_events'] = []
                    map_file['_obstacles'] = []
                    map_file['_bookmarks'] = []
                    bms.write_info(song_name, bpm, difficulty)
                    with open(f"{difficulty}.dat", 'w') as f:
                        json.dump(map_file, f)
                    write_files = ['info.dat', f"{difficulty}.dat", "cover.jpg", f"./temp/{song_file}"]
                    #Zip files together
                    with zipfile.ZipFile(f"../../data/processed/{song_name}_{difficulty}_m1.zip", 'w') as custom:
                        for wfile in write_files:
                            custom.write(wfile)
                    #Delete files
                    os.remove("info.dat")
                    os.remove(f"{difficulty}.dat")
                    print(f"Finished writing {song_name}, {difficulty}.")
                except Exception as err:
                    print(f"{file}: \n {err}")
            #Delete files in temp directory
            filelist = [ f for f in os.listdir('temp')]
            for f in filelist:
                os.remove(os.path.join('temp', f))
            os.rmdir('temp')
strip_human_maps(files)
def model_based_maps(files):
    """Function that writes a zipped map file with each available model in beatmapsynth for each difficulty level."""
    difficulties = ['Easy', 'Normal', 'Hard', 'Expert', 'ExpertPlus']
    models = ['random', 'HMM', 'segmented_HMM', 'rate_modulated_segmented_HMM']
    for file in files:
        if file.endswith('.zip'):
            with open(f"../../data/raw/{file}", 'rb') as zipfolder:
                zip_file = zipfile.ZipFile(zipfolder)
                #Extract zipped files into temp directory
                for x in zip_file.filelist:
                    zip_file.extract(x.filename, path = 'temp')
            #Open info and save information
            with open('./temp/info.dat', 'rb') as i:
                info = json.load(i)
            song_name = info['_songName']
            song_file = info['_songFilename']
            os.rename(f"./temp/{song_file}", f"./song/song.egg")
            for difficulty in difficulties:
                for model in models:
                    abb = ''
                    if model == 'random':
                        abb = 'm2'
                    elif model == 'HMM':
                        abb = 'm3'
                    elif model == 'segmented_HMM':
                        abb = 'm4'
                    elif model == 'rate_modulated_segmented_HMM':
                        abb = 'm5'
                    start_time = time.time()
                    while (time.time() - start_time) < 300:
                        try:
                            bms.beat_map_synthesizer(f"./song/song.egg", f"{song_name}_{difficulty}_{abb}", difficulty, model)
                            break
                        except Exception as err:
                            print(f"{file}: \n {err}")
                    else:
                        print(f"BeatMapSynth failed mapping {song_name} in {file} on {difficulty} with {model} model")
                        continue        
            #Delete song
            os.remove(f"./song/song.egg")
            #Delete temp directory
            filelist = [ f for f in os.listdir('temp')]
            for f in filelist:
                os.remove(os.path.join('temp', f))
            os.rmdir('temp')
model_based_maps(files[4:])
start_time = time.time()

while (time.time() - start_time) < 5:
    pass
else:
    print("5 seconds has elapsed")<jupyter_output>5 seconds has elapsed
{'fraction_non_alphanumeric': 0.11177446102819237, 'fraction_numerical': 0.014483139856274184, 'mean_word_length': 2.2704266088214027, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 28, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '618751', 'license': 'MIT', 'n_tokens_mistral': 2871, 'n_tokens_neox': 2653, 'n_words': 758, 'path': 'notebooks/exploratory/User Testing.ipynb', 'repo_name': 'wvsharber/BMS_testing', 'size': '9045'}