Upload 13 files

.gitattributes

@@ -0,0 +1,3 @@
+examples/mona_diner.png filter=lfs diff=lfs merge=lfs -text
+examples/santa.png filter=lfs diff=lfs merge=lfs -text
+examples/winter_hiking.png filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,435 @@
+import gradio as gr
+import os
+import spaces
+import json
+import re
+import random
+import numpy as np
+from gradio_client import Client
+hf_token = os.environ.get("HF_TOKEN")
+
+MAX_SEED = np.iinfo(np.int32).max
+
+def check_api(model_name):
+    if model_name == "MAGNet":
+        try :
+            client = Client("https://fffiloni-magnet.hf.space/")
+            return "api ready"
+        except : 
+            return "api not ready yet"
+    elif model_name == "AudioLDM-2":
+        try :
+            client = Client("https://haoheliu-audioldm2-text2audio-text2music.hf.space/")
+            return "api ready"
+        except : 
+            return "api not ready yet"
+    elif model_name == "Riffusion":
+        try :
+            client = Client("https://fffiloni-spectrogram-to-music.hf.space/")
+            return "api ready"
+        except : 
+            return "api not ready yet"
+    elif model_name == "Mustango":
+        try :
+            client = Client("https://declare-lab-mustango.hf.space/")
+            return "api ready"
+        except : 
+            return "api not ready yet"
+    elif model_name == "MusicGen":
+        try :
+            client = Client("https://facebook-musicgen.hf.space/")
+            return "api ready"
+        except : 
+            return "api not ready yet"
+    elif model_name == "Stable Audio Open":
+        try:
+            client = Client("fffiloni/Stable-Audio-Open-A10", hf_token=hf_token)
+            return "api ready"
+        except:
+            return "api not ready yet"
+    
+        
+from moviepy.editor import VideoFileClip
+from moviepy.audio.AudioClip import AudioClip
+
+def extract_audio(video_in):
+    input_video = video_in
+    output_audio = 'audio.wav'
+    
+    # Open the video file and extract the audio
+    video_clip = VideoFileClip(input_video)
+    audio_clip = video_clip.audio
+    
+    # Save the audio as a .wav file
+    audio_clip.write_audiofile(output_audio, fps=44100)  # Use 44100 Hz as the sample rate for .wav files  
+    print("Audio extraction complete.")
+
+    return 'audio.wav'
+
+
+
+def get_caption(image_in):
+    kosmos2_client = Client("https://ydshieh-kosmos-2.hf.space/")
+    kosmos2_result = kosmos2_client.predict(
+        image_in,	# str (filepath or URL to image) in 'Test Image' Image component
+        "Detailed",	# str in 'Description Type' Radio component
+        fn_index=4
+    )
+
+    print(f"KOSMOS2 RETURNS: {kosmos2_result}")
+
+    with open(kosmos2_result[1], 'r') as f:
+        data = json.load(f)
+    
+    reconstructed_sentence = []
+    for sublist in data:
+        reconstructed_sentence.append(sublist[0])
+
+    full_sentence = ' '.join(reconstructed_sentence)
+    #print(full_sentence)
+
+    # Find the pattern matching the expected format ("Describe this image in detail:" followed by optional space and then the rest)...
+    pattern = r'^Describe this image in detail:\s*(.*)$'
+    # Apply the regex pattern to extract the description text.
+    match = re.search(pattern, full_sentence)
+    if match:
+        description = match.group(1)
+        print(description)
+    else:
+        print("Unable to locate valid description.")
+
+    # Find the last occurrence of "."
+    #last_period_index = full_sentence.rfind('.')
+
+    # Truncate the string up to the last period
+    #truncated_caption = full_sentence[:last_period_index + 1]
+
+    # print(truncated_caption)
+    #print(f"\n—\nIMAGE CAPTION: {truncated_caption}")
+    
+    return description
+
+def get_caption_from_MD(image_in):
+    client = Client("https://vikhyatk-moondream1.hf.space/")
+    result = client.predict(
+		image_in,	# filepath  in 'image' Image component
+		"Describe precisely the image.",	# str  in 'Question' Textbox component
+		api_name="/answer_question"
+    )
+    print(result)
+    return result
+
+def get_magnet(prompt):
+
+    client = Client("https://fffiloni-magnet.hf.space/")
+    result = client.predict(
+        "facebook/magnet-small-10secs",	# Literal['facebook/magnet-small-10secs', 'facebook/magnet-medium-10secs', 'facebook/magnet-small-30secs', 'facebook/magnet-medium-30secs', 'facebook/audio-magnet-small', 'facebook/audio-magnet-medium']  in 'Model' Radio component
+        "",	# str  in 'Model Path (custom models)' Textbox component
+        prompt,	# str  in 'Input Text' Textbox component
+        3,	# float  in 'Temperature' Number component
+        0.9,	# float  in 'Top-p' Number component
+        10,	# float  in 'Max CFG coefficient' Number component
+        1,	# float  in 'Min CFG coefficient' Number component
+        20,	# float  in 'Decoding Steps (stage 1)' Number component
+        10,	# float  in 'Decoding Steps (stage 2)' Number component
+        10,	# float  in 'Decoding Steps (stage 3)' Number component
+        10,	# float  in 'Decoding Steps (stage 4)' Number component
+        "prod-stride1 (new!)",	# Literal['max-nonoverlap', 'prod-stride1 (new!)']  in 'Span Scoring' Radio component
+        api_name="/predict_full"
+    )
+    print(result)
+    return result[1]
+
+def get_audioldm(prompt):
+    client = Client("https://haoheliu-audioldm2-text2audio-text2music.hf.space/")
+    seed = random.randint(0, MAX_SEED)
+    result = client.predict(
+        prompt,	# str in 'Input text' Textbox component
+        "Low quality.",	# str in 'Negative prompt' Textbox component
+        10,	# int | float (numeric value between 5 and 15) in 'Duration (seconds)' Slider component
+        6.5,	# int | float (numeric value between 0 and 7) in 'Guidance scale' Slider component
+        seed,	# int | float in 'Seed' Number component
+        3,	# int | float (numeric value between 1 and 5) in 'Number waveforms to generate' Slider component
+        fn_index=1
+    )
+    print(result)
+    audio_result = extract_audio(result)
+    return audio_result
+
+def get_riffusion(prompt):
+    client = Client("https://fffiloni-spectrogram-to-music.hf.space/")
+    result = client.predict(
+		prompt,	# str  in 'Musical prompt' Textbox component
+		"",	# str  in 'Negative prompt' Textbox component
+		None,	# filepath  in 'parameter_4' Audio component
+		10,	# float (numeric value between 5 and 10) in 'Duration in seconds' Slider component
+		api_name="/predict"
+    )
+    print(result)
+    return result[1]
+
+def get_mustango(prompt):
+    client = Client("https://declare-lab-mustango.hf.space/")
+    result = client.predict(
+		prompt,	# str  in 'Prompt' Textbox component
+		200,	# float (numeric value between 100 and 200) in 'Steps' Slider component
+		6,	# float (numeric value between 1 and 10) in 'Guidance Scale' Slider component
+							api_name="/predict"
+    )
+    print(result)
+    return result
+
+def get_musicgen(prompt):
+    client = Client("https://facebook-musicgen.hf.space/")
+    result = client.predict(
+        prompt,	# str  in 'Describe your music' Textbox component
+        None,	# str (filepath or URL to file) in 'File' Audio component
+        fn_index=0
+    )
+    print(result)
+    return result[1]
+
+def get_stable_audio_open(prompt):
+    client = Client("fffiloni/Stable-Audio-Open-A10", hf_token=hf_token)
+    result = client.predict(
+		prompt=prompt,
+		seconds_total=10,
+		steps=100,
+		cfg_scale=7,
+		api_name="/predict"
+    )
+    print(result)
+    return result
+
+import re
+import torch
+from transformers import pipeline
+
+zephyr_model = "HuggingFaceH4/zephyr-7b-beta"
+mixtral_model = "mistralai/Mixtral-8x7B-Instruct-v0.1"
+
+pipe = pipeline("text-generation", model=zephyr_model, torch_dtype=torch.bfloat16, device_map="auto")
+
+standard_sys = f"""
+You are a musician AI whose job is to help users create their own music which its genre will reflect the character or scene from an image described by users.
+In particular, you need to respond succintly with few musical words, in a friendly tone, write a musical prompt for a music generation model.
+
+For example, if a user says, "a picture of a man in a black suit and tie riding a black dragon", provide immediately a musical prompt corresponding to the image description. 
+Immediately STOP after that. It should be EXACTLY in this format:
+"A grand orchestral arrangement with thunderous percussion, epic brass fanfares, and soaring strings, creating a cinematic atmosphere fit for a heroic battle"
+"""
+
+mustango_sys = f"""
+You are a musician AI whose job is to help users create their own music which its genre will reflect the character or scene from an image described by users.
+In particular, you need to respond succintly with few musical words, in a friendly tone, write a musical prompt for a music generation model, you MUST include chords progression.
+
+For example, if a user says, "a painting of three old women having tea party", provide immediately a musical prompt corresponding to the image description. 
+Immediately STOP after that. It should be EXACTLY in this format:
+"The song is an instrumental. The song is in medium tempo with a classical guitar playing a lilting melody in accompaniment style. The song is emotional and romantic. The song is a romantic instrumental song. The chord sequence is Gm, F6, Ebm. The time signature is 4/4. This song is in Adagio. The key of this song is G minor."
+"""
+
+@spaces.GPU(enable_queue=True)
+def get_musical_prompt(user_prompt, chosen_model):
+
+    """
+    if chosen_model == "Mustango" :
+        agent_maker_sys = standard_sys
+    else :
+        agent_maker_sys = standard_sys
+    """
+    agent_maker_sys = standard_sys
+    
+    instruction = f"""
+<|system|>
+{agent_maker_sys}</s>
+<|user|>
+"""
+    
+    prompt = f"{instruction.strip()}\n{user_prompt}</s>"    
+    outputs = pipe(prompt, max_new_tokens=256, do_sample=True, temperature=0.7, top_k=50, top_p=0.95)
+    pattern = r'\<\|system\|\>(.*?)\<\|assistant\|\>'
+    cleaned_text = re.sub(pattern, '', outputs[0]["generated_text"], flags=re.DOTALL)
+    
+    print(f"SUGGESTED Musical prompt: {cleaned_text}")
+    return cleaned_text.lstrip("\n")
+
+def infer(image_in, chosen_model, api_status):
+    if image_in == None :
+        raise gr.Error("Please provide an image input")
+
+    if chosen_model == [] :
+        raise gr.Error("Please pick a model")
+
+    if api_status == "api not ready yet" :
+        raise gr.Error("This model is not ready yet, you can pick another one instead :)")
+    
+    gr.Info("Getting image caption with Kosmos-2...")
+    user_prompt = get_caption(image_in)
+    #user_prompt = get_caption_from_MD(image_in)
+    
+    gr.Info("Building a musical prompt according to the image caption ...")
+    musical_prompt = get_musical_prompt(user_prompt, chosen_model)
+
+    if chosen_model == "MAGNet" :
+        gr.Info("Now calling MAGNet for music...")
+        music_o = get_magnet(musical_prompt)
+    elif chosen_model == "AudioLDM-2" :
+        gr.Info("Now calling AudioLDM-2 for music...")
+        music_o = get_audioldm(musical_prompt)
+    elif chosen_model == "Riffusion" :
+        gr.Info("Now calling Riffusion for music...")
+        music_o = get_riffusion(musical_prompt)
+    elif chosen_model == "Mustango" :
+        gr.Info("Now calling Mustango for music...")
+        music_o = get_mustango(musical_prompt)
+    elif chosen_model == "MusicGen" :
+        gr.Info("Now calling MusicGen for music...")
+        music_o = get_musicgen(musical_prompt)
+    elif chosen_model == "Stable Audio Open" :
+        gr.Info("Now calling Stable Audio Open for music...")
+        music_o = get_stable_audio_open(musical_prompt)
+    
+    return gr.update(value=musical_prompt, interactive=True), gr.update(visible=True), music_o
+
+def retry(chosen_model, caption):
+    musical_prompt = caption
+
+    if chosen_model == "MAGNet" :
+        gr.Info("Now calling MAGNet for music...")
+        music_o = get_magnet(musical_prompt)
+    elif chosen_model == "AudioLDM-2" :
+        gr.Info("Now calling AudioLDM-2 for music...")
+        music_o = get_audioldm(musical_prompt)
+    elif chosen_model == "Riffusion" :
+        gr.Info("Now calling Riffusion for music...")
+        music_o = get_riffusion(musical_prompt)
+    elif chosen_model == "Mustango" :
+        gr.Info("Now calling Mustango for music...")
+        music_o = get_mustango(musical_prompt)
+    elif chosen_model == "MusicGen" :
+        gr.Info("Now calling MusicGen for music...")
+        music_o = get_musicgen(musical_prompt)
+    elif chosen_model == "Stable Audio Open" :
+        gr.Info("Now calling Stable Audio Open for music...")
+        music_o = get_stable_audio_open(musical_prompt)
+
+    return music_o
+
+demo_title = "圖片轉換成音樂系統"
+description = "將上傳的影片或範例影片轉換為音樂"
+
+css = """
+#col-container {
+    margin: 0 auto;
+    max-width: 980px;
+    text-align: left;
+}
+#inspi-prompt textarea {
+    font-size: 20px;
+    line-height: 24px;
+    font-weight: 600;
+}
+
+"""
+
+with gr.Blocks(css=css) as demo:
+    
+    with gr.Column(elem_id="col-container"):
+    
+        gr.HTML(f"""
+        <h2 style="text-align: center;">{demo_title}</h2>
+        <p style="text-align: center;">{description}</p>
+        """)
+        
+        with gr.Row():
+            
+            with gr.Column():
+                image_in = gr.Image(
+                    label = "上傳圖片檔案",
+                    type = "filepath",
+                    elem_id = "image-in"
+                )
+                
+                with gr.Row():
+                    
+                    chosen_model = gr.Dropdown(
+                        label = "選擇音樂模型",
+                        choices = [
+                            "MAGNet",
+                            "AudioLDM-2",
+                            "Riffusion",
+                            "Mustango",
+                            "MusicGen",
+                            "Stable Audio Open"
+                        ],
+                        value = None,
+                        filterable = False
+                    )
+                    
+                    check_status = gr.Textbox(
+                        label="API是否可用",
+                        interactive=False
+                    )
+                
+                submit_btn = gr.Button("將圖片轉換成音樂")
+
+                gr.Examples(
+                    examples = [
+                        ["examples/ocean_poet.jpeg"],
+                        ["examples/jasper_horace.jpeg"],
+                        ["examples/summer.jpeg"],
+                        ["examples/mona_diner.png"],
+                        ["examples/monalisa.png"],
+                        ["examples/santa.png"],
+                        ["examples/winter_hiking.png"],
+                        ["examples/teatime.jpeg"],
+                        ["examples/news_experts.jpeg"]
+                    ],
+                    fn = infer,
+                    inputs = [image_in, chosen_model],
+                    examples_per_page = 4
+                )
+            
+            with gr.Column():
+            
+                caption = gr.Textbox(
+                    label = "圖片首先產生的文字",
+                    interactive = False,
+                    elem_id = "inspi-prompt"
+                )
+                
+                retry_btn = gr.Button("更改文字重新產生", visible=False)
+                
+                result = gr.Audio(
+                    label = "音樂"
+                )
+        
+
+    chosen_model.change(
+        fn = check_api,
+        inputs = chosen_model,
+        outputs = check_status,
+        queue = False
+    )
+
+    retry_btn.click(
+        fn = retry,
+        inputs = [chosen_model, caption],
+        outputs = [result]
+    )
+    
+    submit_btn.click(
+        fn = infer,
+        inputs = [
+            image_in,
+            chosen_model,
+            check_status
+        ],
+        outputs =[
+            caption,
+            retry_btn,
+            result
+        ]
+    )
+
+demo.queue(max_size=16).launch(show_api=False, show_error=True, share=True)

requirements.txt

@@ -0,0 +1,5 @@
+torch
+transformers
+accelerate
+moviepy
+spaces