From 5c34afa7fa411ef6a44771d27c077ab4e465e77a Mon Sep 17 00:00:00 2001 From: Mikei386 <44135113+Mikei386@users.noreply.github.com> Date: Thu, 10 Sep 2026 22:56:24 +0200 Subject: [PATCH] Enable SheetSage2 audio remix in YuE2 UI --- experiments/yue2-3b/Dockerfile | 15 +++- experiments/yue2-3b/README.md | 23 +++++ .../community-webui-patches/patch_webui.py | 84 +++++++++++++++++++ 3 files changed, 121 insertions(+), 1 deletion(-) diff --git a/experiments/yue2-3b/Dockerfile b/experiments/yue2-3b/Dockerfile index 34d04df..a0c0026 100644 --- a/experiments/yue2-3b/Dockerfile +++ b/experiments/yue2-3b/Dockerfile @@ -4,7 +4,7 @@ ARG YUE2_COMMIT=9c6c4b349be978b06a9d0d958471a07a6cdeff4d ARG YUE2_WEBUI_COMMIT=8fc05609bde5dcd345d7c6d57fff3da0839164c5 RUN apt-get update \ - && apt-get install -y --no-install-recommends ca-certificates git libsndfile1 \ + && apt-get install -y --no-install-recommends ca-certificates ffmpeg git libsndfile1 \ && rm -rf /var/lib/apt/lists/* RUN git clone https://github.com/multimodal-art-projection/YuE.git /opt/yue2 \ @@ -21,6 +21,19 @@ RUN git clone https://github.com/Ladypoly/YuE2_WebUI.git /tmp/yue2-webui \ && python -m pip install --no-cache-dir -r /opt/yue2/webui/requirements.txt \ && rm -rf /tmp/yue2-webui +# SheetSage2 converts an uploaded recording into the ABC score YuE2 consumes. +# Keep its pinned Transformers/Numpy stack in a small virtual environment, but +# reuse the image's Blackwell-capable torch 2.10 + CUDA 12.8 installation. +# The upstream torch 2.8+cu126 recipe cannot execute RTX 50-series kernels. +RUN python -m venv --system-site-packages /opt/yue2/.venv-sheetsage2 \ + && /opt/yue2/.venv-sheetsage2/bin/pip install --no-cache-dir \ + "torchaudio==2.10.0" --index-url https://download.pytorch.org/whl/cu128 \ + && /opt/yue2/.venv-sheetsage2/bin/pip install --no-cache-dir \ + "transformers==4.45.2" "huggingface-hub==0.36.0" \ + "safetensors==0.5.3" "numpy==1.26.4" "scipy==1.13.1" \ + "mir_eval==0.8.2" "pretty_midi==0.2.10" "mido==1.3.3" \ + "setuptools==78.1.1" + COPY community-webui-patches /tmp/community-webui-patches RUN python /tmp/community-webui-patches/patch_webui.py /opt/yue2/webui \ && rm -rf /tmp/community-webui-patches diff --git a/experiments/yue2-3b/README.md b/experiments/yue2-3b/README.md index 67e1a93..19e62e5 100644 --- a/experiments/yue2-3b/README.md +++ b/experiments/yue2-3b/README.md @@ -44,6 +44,29 @@ and stable-diffusion.cpp are not part of the Athena setup. Manual composition, generation, result playback, score editing and the take library work without those optional components. +### Uploaded-audio remix (SheetSage2) + +The **Remix a take** drawer also accepts WAV, FLAC, MP3, M4A, OGG, Opus and +AAC uploads. SheetSage2 transcribes the recording into an editable ABC melody +and chord plan, then YuE2 renders that structure in a newly selected style. +It does not preserve the original samples, singer or production verbatim. + +SheetSage2 is kept in `/opt/yue2/.venv-sheetsage2`, while its persistent model +files live below `/data/models/yue2`. The environment intentionally reuses the +image's PyTorch 2.10/CUDA 12.8 runtime: the upstream cu126 recipe is not +Blackwell-capable. Required persistent directories are: + +```text +/data/models/yue2/ +├── SheetSage2/ +└── MERT-v2-FullSong/ +``` + +The local `SheetSage2/config.json` must point `base_model_name_or_path` to +`/opt/yue2/models/MERT-v2-FullSong`, allowing the complete transcription path +to run offline. Analysis and generation share the RTX 5080 and therefore run +sequentially; the UI parks YuE2 before starting SheetSage2. + The small integration patch under `community-webui-patches/` fixes the community release's missing `refreshArt()` function on Linux and permits the native YuE2 empty-lyrics request for true instrumentals. It deliberately does diff --git a/experiments/yue2-3b/community-webui-patches/patch_webui.py b/experiments/yue2-3b/community-webui-patches/patch_webui.py index 515e2f2..09c97bf 100644 --- a/experiments/yue2-3b/community-webui-patches/patch_webui.py +++ b/experiments/yue2-3b/community-webui-patches/patch_webui.py @@ -81,3 +81,87 @@ replace_once( '''Lyrics section tags on their own line''', '''Lyrics section tags on their own line · leave empty for a true instrumental''', ) + +# The backend already ships a complete SheetSage2 upload endpoint but hides it +# from the published console. Surface it beside the existing take-to-take +# remix controls so uploaded songs can seed a new melody/arrangement. +replace_once( + index_html, + '''

+ + +''', + '''

+ +

Remix an uploaded recording

+

SheetSage2 listens to an uploaded song and writes an editable melody + and chord score for YuE2. This is structural transcription, not a sample or a copy of + the original sound. After analysis, choose a new style and generate normally.

+
+ +
+ +
+
+ +

Checking SheetSage2…

+ + +''', +) + +replace_once( + app_js, + ''' $("coverStatus").textContent = takes.length + ? takes.length + " takes carry a score you can remix." + : "Make a song in Full plan or Melody only mode first; Direct mode keeps no score."; + }).catch(function () {}); +''', + ''' $("coverStatus").textContent = takes.length + ? takes.length + " takes carry a score you can remix." + : "Make a song in Full plan or Melody only mode first; Direct mode keeps no score."; + var sheetsage = data.sheetsage || {}; + $("coverFromAudio").disabled = !sheetsage.available; + $("coverAudioStatus").textContent = sheetsage.available + ? "SheetSage2 is ready. Analysis temporarily parks YuE2 because both use the GPU." + : "SheetSage2 is not installed yet; uploaded-audio remix is unavailable."; + }).catch(function (error) { + $("coverFromAudio").disabled = true; + $("coverAudioStatus").textContent = error.message; + }); +''', +) + +replace_once( + app_js, + ''' /* -------------------------------------------------------------- compose */ +''', + ''' $("coverFromAudio").addEventListener("click", function () { + var file = $("coverAudio").files[0]; + if (!file) return toast("Choose an audio file first", "bad"); + var button = $("coverFromAudio"); + var body = new FormData(); + body.append("file", file, file.name); + body.append("melody_only", $("coverAudioMelodyOnly").checked ? "true" : "false"); + button.disabled = true; + button.textContent = "Analysing…"; + $("coverAudioStatus").textContent = "Listening to “" + file.name + "”… this can take several minutes."; + api("/api/cover/from-audio", { method: "POST", body: body }).then(function (data) { + applyCoverScore(data.abc, "Melody analysed from “" + data.source + "” in " + data.seconds + " s. Choose the new style and generate."); + $("coverAudioStatus").textContent = "Analysis complete in " + data.seconds + " s. The editable score is loaded below."; + }).catch(function (error) { + $("coverAudioStatus").textContent = error.message; + toast(error.message, "bad"); + }).then(function () { + button.textContent = "Analyse uploaded melody"; + refreshCover(); + }); + }); + + /* -------------------------------------------------------------- compose */ +''', +)