app.py

import time
import os
import pickle
import sys
sys.path.append('/')
from src.music.pipeline.music_pipeline import encode_music
from src.music2cocktailrep.pipeline.music2cocktailrep import music2cocktailrep, setup_translation_models, debug_translation
from src.cocktails.pipeline.cocktailrep2recipe import cocktailrep2recipe
from src.debugger import Debugger
from datetime import datetime
from shutil import copy
import streamlit as st
from src.music.config import AUDIO_PATH, MIDI_PATH
from pretty_midi import PrettyMIDI
import numpy as np
import pydub
from PIL import Image


st.set_page_config(
    page_title="TastyPiano",
    page_icon="🎹",
)

st.title('TastyPiano')

synestesia_path = 'data/synesthesia/'
debugger = Debugger(verbose=False)

def centered_module(func, text, args=None):
    _, col2, _ = st.columns([1, 2, 1])
    with col2:
        return func(text)

def centered_button(func, text, args=None):
    _, _, _, col3, _, _, _ = st.columns([1, 1, 1, 1, 1, 1, 1])
    with col3:
        return func(text)

def setup_streamlite():
    setup_translation_models()
    image = Image.open('./data/pianocktail.jpg')
    st.image(image, caption='Pianocktail by Compagnie la Rumeur')

    st.subheader("Ready to taste music?")
    st.markdown("TastyPiano generates music--taste synesthetic experiences by letting you turn any piano song into a cocktail. It is inspired by the"
                " *Pianocktail*, a literary invention from the French novelist, jazz musician, singer and engineer Boris Vian. I see TastyPiano as a digital member of the "
                "Pianocktail species, along with [other](https://www.youtube.com/watch?v=5bdX5i0nAWw) "
                "[wonderful](https://www.youtube.com/watch?v=pzsDOH-xtrs&list=PLC196AA37A2D1C066&index=3) [machines](https://www.youtube.com/watch?v=y0RJg7I2x34).")
    st.markdown("But TastyPiano is different from existing pianocktails. While existing version merely map played notes to drops of corresponding ingredients, "
                "TastyPiano listens to the song, analyzes the music, hallucinates matching tastes and flavors, before it starts generating cocktail recipes "
                "until it finds the one matching its gustatory hallucination. Check my [blog post](https://ccolas.github.io/project/pianocktail) for more information."
                " Hear and taste it for yourself!")
    st.subheader("How to use it?")
    st.markdown("Provide a piano solo input and click on the **Taste it!** button.\n"
                "\nYou can input either: \n* a YouTube url **or**\n* an audio file (.mp3) **or**\n* a midi file (.mid)\n"
                "All these should be **only piano**, no other instrument.\n"
                "Note that audio sources are cropped to the first 40s because the process of "
                "converting it to midi is rather slow (1sec/sec). Midi inputs are taken whole. Please report any bug / buggy url in the community tab.")

    st.subheader("Prepare")
    col1, col2, col3 = st.columns(3)
    generate_audio_from_midi = False
    with col1:
        st.markdown('**YouTube url**')
        url = st.text_area('Type it below', 'https://www.youtube.com/watch?v=UGK70IkP830', height=160)
    with col2:
        st.markdown('**Audio file**')
        audio = st.file_uploader("Upload it here (.mp3)", type=['.mp3'])
    with col3:
        st.markdown('**Midi file**')
        midi = st.file_uploader("Upload it here (.mid)", type=['.mid'])
        generate_audio_from_midi = st.checkbox('Generate audio? Untick if the song is too long (>10min)', value=True)

    def run(unit):
        setup_and_run(unit=unit, url=url, midi=midi, audio=audio, generate_audio_from_midi=generate_audio_from_midi, extra_code=None)
    # url = "https://youtu.be/VUCI-1vIbUo"
    # unit = 'mL'
    # run(unit)
    st.markdown('##')
    unit = st.radio('Pick the units (before pressing "Taste it!", default mL)', ['mL', 'oz'], index=0)
    button = centered_button(st.button, 'Taste it!')
    if button:
        run(unit)


def pianocktail(unit='mL', record=False, url=None, midi=None, audio=None, processed=None, crop=40, verbose=False, debug=False, level=0):
    assert url is not None or midi is not None or audio is not None or processed is not None
    if verbose: print('------\nNew synesthetic exploration!')
    init_time = time.time()
    try:
        with st.spinner("Listening to the song (~1min).."):
            music_ai_rep, music_handcoded_rep, all_paths, error = encode_music(record=record, url=url, audio_path=audio, midi_path=midi, nb_aug=0, noise_injection=False,
                                                                               augmentation=False, processed_path=processed, crop=crop, apply_filtering=False, verbose=verbose,
                                                                               level=level+2)
        if music_ai_rep is not None:
            with st.spinner(text="Thinking about corresponding flavors.."):
                cocktail_rep, affective_cluster_id, affect = music2cocktailrep(music_ai_rep, music_handcoded_rep, verbose=verbose, level=level+2)
            with st.spinner("Trying recipes (15s).."):
                cocktail_recipes, scores = cocktailrep2recipe(cocktail_rep, unit=unit, target_affective_cluster=affective_cluster_id, verbose=verbose, full_verbose=verbose, \
                                                                                                                                                              level=level+2)

            cocktail_recipe = cocktail_recipes[0]
            recipe_score = scores[0]
            if debug:
                music_reconstruction = debug_translation(music_ai_rep)
                debugger.extract_info(all_paths, affective_cluster_id, affect, cocktail_rep, music_reconstruction, recipe_score, verbose=verbose, level=level+2)
                debug_info = debugger.debug_dict
            else:
                debug_info = None
            if verbose:
                print(cocktail_recipe.replace('Recipe', ' ' * (level + 2) + 'Generated recipe:').replace('None ()', ''))
                debugger.print_debug(level=level+2)
                print(f'\nEnd of synesthetic exploration ({int(time.time() - init_time)} secs).\n------')
            st.success('Recipe found!')
        else:
            st.error('Error in listening. Is the url valid? the audio an .mp3? the midi a .mid?')
            cocktail_recipe = None
            debug_info = None
    except:
        st.error('Error: ' + error)
        cocktail_recipe = None
        debug_info = None
    return cocktail_recipe, debug_info

def setup_and_run(unit='mL', url=None, midi=None, audio=None, generate_audio_from_midi=False, verbose=True, debug=True, extra_code=None):
    if url is None and midi is None and audio is None:
        st.error('Please enter a piano input.')
        assert False
    st.subheader('Synesthesia')
    now = datetime.now()
    folder_name = f'{now.year}-{now.month}-{now.day}_{now.hour}:{now.minute}:{now.second}'
    folder_path = synestesia_path + folder_name
    if extra_code is not None:
        folder_path += '_' + extra_code
    if os.path.exists(folder_path):
        folder_path += '_2'
    folder_path += '/'
    os.makedirs(folder_path, exist_ok=True)
    if midi is not None:
        st.write(f'  \tReading from midi file: {midi.name}')
        midi_path = MIDI_PATH + 'from_url_midi/' + midi.name[:-4] + '_midi.mid'
        audio_path = AUDIO_PATH + 'from_url/' + midi.name.replace('.mid', '.mp3')
        with open(midi_path, "wb") as f:
            f.write(midi.getbuffer())
        midi = midi_path
        if generate_audio_from_midi:
            midi_data = PrettyMIDI(midi_path)
            audio_data = midi_data.fluidsynth(fs=44100)
            y = np.int16(audio_data * 2 ** 15)
            song = pydub.AudioSegment(y.tobytes(), frame_rate=44100, sample_width=2, channels=1)
            song.export(audio_path, format="mp3", bitrate="320k")
            # st.write(audio_data)
            st.audio(audio_path, format='audio/mp3')
        url = None
    elif audio is not None:
        st.write(f'  \tReading from audio file: {audio.name}')
        audio_path = AUDIO_PATH + 'from_url/' + audio.name
        with open(audio_path, "wb") as f:
            f.write(audio.getbuffer())
        audio = audio_path
        audio_file = open(audio, 'rb')
        audio_bytes = audio_file.read()
        st.audio(audio_bytes, format='audio/mp3')
        url = None
    else:
        st.write(f'  \tReading from YouTube url: {url}')
        st.video(url)

    _, col2, _ = st.columns([1, 1, 1])
    with col2:
        st.markdown('##')
        recipe, debug = pianocktail(unit=unit, url=url, midi=midi, audio=audio, verbose=verbose, debug=debug)
    with open(folder_path + 'debug.pk', 'wb') as f:
        pickle.dump(debug, f)
    with open(folder_path + 'recipe.txt', 'w') as f:
        f.write(recipe)
    paths = debug['all_paths']
    if paths['url'] is not None:
        with open(folder_path + 'url.txt', 'w') as f:
            f.write(paths['url'])
    for k in ['audio_path', 'midi_path']:
        origin = paths[k]
        if origin is not None:
            copy(origin, folder_path + origin.split('/')[-1])

    st.subheader('Recipe')
    recipe = recipe.replace('   Enjoy!', '  \nEnjoy!').replace('\n', '  \n')
    st.text(recipe)

    st.markdown('**About this synesthesia**')
    closest_songs = [debug['nn_music'][i][:-26].split('structured_')[1].replace('_', ' ') for i in range(3)]
    str_songs = 'These are the closest song I know: '
    str_songs += ' '.join([f'  \n* {closest_songs[i]}' for i in range(3)])
    st.markdown(str_songs + '.')
    str_cocktails = 'These are existing cocktails that are close to the taste of this song:'
    str_cocktails += ' '.join([f'  \n* {cocktail_name}: {cocktail_url}' for cocktail_name, cocktail_url in zip(debug['nearest_cocktail_names'][:3],
                                                                                                               debug['nearest_cocktail_urls'][:3])])
    st.markdown(str_cocktails + '.')

if __name__ == '__main__':
    setup_streamlite()
    # urls = ["https://www.youtube.com/watch?v=PLFVGwGQcB0",
    #         "https://www.youtube.com/watch?v=VQmuAr93OlI",
    #         "https://www.youtube.com/watch?v=Nv2GgV34qIg&list=PLO9E3V4rGLD8_iWrCioJRWZXJJE3Fzu_J&index=4",
    #         "https://www.youtube.com/watch?v=qAEIjWYdoYc&list=PLO9E3V4rGLD8_iWrCioJRWZXJJE3Fzu_J&index=1",
    #         "https://www.youtube.com/watch?v=M73x3O7dhmg&list=PLO9E3V4rGLD8_iWrCioJRWZXJJE3Fzu_J&index=5"]
    # setup_translation_models()
    # setup_and_run(url=urls[0], verbose=True, debug=True)
    # recipes = []
    # for url in urls:
    #     recipe = pianocktail(url=url, verbose=True, debug=True)[0]
    #     recipes.append(recipe)
    # stop = 1