redboldcode/spectrogram-to-music
0
1import gradio as gr2import torch3 4from PIL import Image5import numpy as np6from spectro import wav_bytes_from_spectrogram_image7 8from diffusers import StableDiffusionPipeline9from diffusers import StableDiffusionImg2ImgPipeline10 11from share_btn import community_icon_html, loading_icon_html, share_js12 13device = "cuda"14MODEL_ID = "riffusion/riffusion-model-v1"15pipe = StableDiffusionPipeline.from_pretrained(MODEL_ID, torch_dtype=torch.float16)16pipe = pipe.to(device)17pipe2 = StableDiffusionImg2ImgPipeline.from_pretrained(MODEL_ID, torch_dtype=torch.float16)18pipe2 = pipe2.to(device)19 20spectro_from_wav = gr.Interface.load("spaces/fffiloni/audio-to-spectrogram")21 22def predict(prompt, negative_prompt, audio_input, duration):23 if audio_input == None :24 return classic(prompt, negative_prompt, duration)25 else :26 return style_transfer(prompt, negative_prompt, audio_input)27 28def classic(prompt, negative_prompt, duration):29 if duration == 5:30 width_duration=51231 else :32 width_duration = 512 + ((int(duration)-5) * 128)33 spec = pipe(prompt, negative_prompt=negative_prompt, height=512, width=width_duration).images[0]34 print(spec)35 wav = wav_bytes_from_spectrogram_image(spec)36 with open("output.wav", "wb") as f:37 f.write(wav[0].getbuffer())38 return spec, 'output.wav', gr.update(visible=True), gr.update(visible=True), gr.update(visible=True)39 40def style_transfer(prompt, negative_prompt, audio_input):41 spec = spectro_from_wav(audio_input)42 print(spec)43 # Open the image44 im = Image.open(spec)45 46 47 # Open the image48 im = image_from_spectrogram(im, 1)49 50 51 new_spectro = pipe2(prompt=prompt, image=im, strength=0.5, guidance_scale=7).images52 wav = wav_bytes_from_spectrogram_image(new_spectro[0])53 with open("output.wav", "wb") as f:54 f.write(wav[0].getbuffer())55 return new_spectro[0], 'output.wav', gr.update(visible=True), gr.update(visible=True), gr.update(visible=True)56 57def image_from_spectrogram(58 spectrogram: np.ndarray, max_volume: float = 50, power_for_image: float = 0.2559) -> Image.Image:60 """61 Compute a spectrogram image from a spectrogram magnitude array.62 """63 # Apply the power curve64 data = np.power(spectrogram, power_for_image)65 66 # Rescale to 0-25567 data = data * 255 / max_volume68 69 # Invert70 data = 255 - data71 72 # Convert to a PIL image73 image = Image.fromarray(data.astype(np.uint8))74 75 # Flip Y76 image = image.transpose(Image.FLIP_TOP_BOTTOM)77 78 # Convert to RGB79 image = image.convert("RGB")80 81 return image82 83title = """84 <div style="text-align: center; max-width: 500px; margin: 0 auto;">85 <div86 style="87 display: inline-flex;88 align-items: center;89 gap: 0.8rem;90 font-size: 1.75rem;91 margin-bottom: 10px;92 line-height: 1em;93 "94 >95 <h1 style="font-weight: 600; margin-bottom: 7px;">96 Riffusion real-time music generation97 </h1>98 </div>99 <p style="margin-bottom: 10px;font-size: 94%;font-weight: 100;line-height: 1.5em;">100 Describe a musical prompt, generate music by getting a spectrogram image & sound.101 </p>102 </div>103"""104 105article = """106 <p style="text-align: center;font-size: 94%;margin-bottom: 20px;">107 Do you need faster results ? You can skip the queue by duplicating this space: 108 <span style="display: flex;align-items: center;justify-content: center;height: 30px;">109 <a style="margin-right: 10px;" href="https://huggingface.co/fffiloni/spectrogram-to-music?duplicate=true"><img src="https://img.shields.io/badge/-Duplicate%20Space-blue?labelColor=white&style=flat&logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAAXNSR0IArs4c6QAAAP5JREFUOE+lk7FqAkEURY+ltunEgFXS2sZGIbXfEPdLlnxJyDdYB62sbbUKpLbVNhyYFzbrrA74YJlh9r079973psed0cvUD4A+4HoCjsA85X0Dfn/RBLBgBDxnQPfAEJgBY+A9gALA4tcbamSzS4xq4FOQAJgCDwV2CPKV8tZAJcAjMMkUe1vX+U+SMhfAJEHasQIWmXNN3abzDwHUrgcRGmYcgKe0bxrblHEB4E/pndMazNpSZGcsZdBlYJcEL9Afo75molJyM2FxmPgmgPqlWNLGfwZGG6UiyEvLzHYDmoPkDDiNm9JR9uboiONcBXrpY1qmgs21x1QwyZcpvxt9NS09PlsPAAAAAElFTkSuQmCC&logoWidth=14" alt="Duplicate Space"></a> 110 <a href="https://colab.research.google.com/drive/1FhH3HlN8Ps_Pr9OR6Qcfbfz7utDvICl0?usp=sharing" target="_blank"><img src="https://colab.research.google.com/assets/colab-badge.svg" /></a>111 </span>112 </p>113 114 <p style="font-size: 0.8em;line-height: 1.2em;border: 1px solid #374151;border-radius: 8px;padding: 20px;">115 About the model: Riffusion is a latent text-to-image diffusion model capable of generating spectrogram images given any text input. These spectrograms can be converted into audio clips.116 <br />—117 <br />The Riffusion model was created by fine-tuning the Stable-Diffusion-v1-5 checkpoint.118 <br />—119 <br />The model is intended for research purposes only. Possible research areas and tasks include 120 generation of artworks, audio, and use in creative processes, applications in educational or creative tools, research on generative models.121 122 </p>123 124 <div class="footer">125 <p>126 <a href="https://huggingface.co/riffusion/riffusion-model-v1" target="_blank">Riffusion model</a> by Seth Forsgren and Hayk Martiros - 127 Demo by 🤗 <a href="https://twitter.com/fffiloni" target="_blank">Sylvain Filoni</a>128 </p>129 </div>130 131 <div id="may-like-container" style="display: flex;justify-content: center;flex-direction: column;align-items: center;">132 <p style="font-size: 0.8em;margin-bottom: 4px;">You may also like: </p>133 <div id="may-like" style="display:flex; align-items:center; justify-content: center;height:20px;">134 <svg height="20" width="158" style="margin-left:4px"> 135 <a href="https://huggingface.co/spaces/fffiloni/img-to-music" target="_blank">136 <image href="https://img.shields.io/badge/🤗 Spaces-Image to Music-blue" src="https://img.shields.io/badge/🤗 Spaces-Image to Music-blue.png" height="20"/>137 </a>138 </svg>139 </div>140 </div>141 142"""143 144css = '''145 #col-container, #col-container-2 {max-width: 510px; margin-left: auto; margin-right: auto;}146 a {text-decoration-line: underline; font-weight: 600;}147 div#record_btn > .mt-6 {148 margin-top: 0!important;149 }150 div#record_btn > .mt-6 button {151 width: 100%;152 height: 40px;153 }154 .footer {155 margin-bottom: 45px;156 margin-top: 10px;157 text-align: center;158 border-bottom: 1px solid #e5e5e5;159 }160 .footer>p {161 font-size: .8rem;162 display: inline-block;163 padding: 0 10px;164 transform: translateY(10px);165 background: white;166 }167 .dark .footer {168 border-color: #303030;169 }170 .dark .footer>p {171 background: #0b0f19;172 }173 .animate-spin {174 animation: spin 1s linear infinite;175 }176 @keyframes spin {177 from {178 transform: rotate(0deg);179 }180 to {181 transform: rotate(360deg);182 }183 }184 #share-btn-container {185 display: flex; padding-left: 0.5rem !important; padding-right: 0.5rem !important; background-color: #000000; justify-content: center; align-items: center; border-radius: 9999px !important; width: 13rem;186 }187 #share-btn {188 all: initial; color: #ffffff;font-weight: 600; cursor:pointer; font-family: 'IBM Plex Sans', sans-serif; margin-left: 0.5rem !important; padding-top: 0.25rem !important; padding-bottom: 0.25rem !important;right:0;189 }190 #share-btn * {191 all: unset;192 }193 #share-btn-container div:nth-child(-n+2){194 width: auto !important;195 min-height: 0px !important;196 }197 #share-btn-container .wrap {198 display: none !important;199 }200 201'''202 203 204 205with gr.Blocks(css="style.css") as demo:206 207 with gr.Column(elem_id="col-container"):208 209 gr.HTML(title)210 211 prompt_input = gr.Textbox(placeholder="a cat diva singing in a New York jazz club", label="Musical prompt", elem_id="prompt-in")212 audio_input = gr.Audio(source="upload", type="filepath", visible=False)213 with gr.Row():214 negative_prompt = gr.Textbox(label="Negative prompt")215 duration_input = gr.Slider(label="Duration in seconds", minimum=5, maximum=250, step=1, value=60, elem_id="duration-slider")216 217 send_btn = gr.Button(value="Get a new spectrogram ! ", elem_id="submit-btn")218 219 with gr.Column(elem_id="col-container-2"):220 221 spectrogram_output = gr.Image(label="spectrogram image result", elem_id="img-out")222 sound_output = gr.Audio(type='filepath', label="spectrogram sound", elem_id="music-out")223 224 with gr.Group(elem_id="share-btn-container"):225 community_icon = gr.HTML(community_icon_html, visible=False)226 loading_icon = gr.HTML(loading_icon_html, visible=False)227 share_button = gr.Button("Share to community", elem_id="share-btn", visible=False)228 229 gr.HTML(article)230 231 send_btn.click(predict, inputs=[prompt_input, negative_prompt, audio_input, duration_input], outputs=[spectrogram_output, sound_output, share_button, community_icon, loading_icon])232 share_button.click(None, [], [], _js=share_js)233 234demo.queue(max_size=250).launch(debug=True)235