Enable SheetSage2 audio remix in YuE2 UI

experiments/yue2-3b/Dockerfile

@@ -4,7 +4,7 @@ ARG YUE2_COMMIT=9c6c4b349be978b06a9d0d958471a07a6cdeff4d
 ARG YUE2_WEBUI_COMMIT=8fc05609bde5dcd345d7c6d57fff3da0839164c5
 
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends ca-certificates git libsndfile1 \
+    && apt-get install -y --no-install-recommends ca-certificates ffmpeg git libsndfile1 \
     && rm -rf /var/lib/apt/lists/*
 
 RUN git clone https://github.com/multimodal-art-projection/YuE.git /opt/yue2 \
@@ -21,6 +21,19 @@ RUN git clone https://github.com/Ladypoly/YuE2_WebUI.git /tmp/yue2-webui \
     && python -m pip install --no-cache-dir -r /opt/yue2/webui/requirements.txt \
     && rm -rf /tmp/yue2-webui
 
+# SheetSage2 converts an uploaded recording into the ABC score YuE2 consumes.
+# Keep its pinned Transformers/Numpy stack in a small virtual environment, but
+# reuse the image's Blackwell-capable torch 2.10 + CUDA 12.8 installation.
+# The upstream torch 2.8+cu126 recipe cannot execute RTX 50-series kernels.
+RUN python -m venv --system-site-packages /opt/yue2/.venv-sheetsage2 \
+    && /opt/yue2/.venv-sheetsage2/bin/pip install --no-cache-dir \
+       "torchaudio==2.10.0" --index-url https://download.pytorch.org/whl/cu128 \
+    && /opt/yue2/.venv-sheetsage2/bin/pip install --no-cache-dir \
+       "transformers==4.45.2" "huggingface-hub==0.36.0" \
+       "safetensors==0.5.3" "numpy==1.26.4" "scipy==1.13.1" \
+       "mir_eval==0.8.2" "pretty_midi==0.2.10" "mido==1.3.3" \
+       "setuptools==78.1.1"
+
 COPY community-webui-patches /tmp/community-webui-patches
 RUN python /tmp/community-webui-patches/patch_webui.py /opt/yue2/webui \
     && rm -rf /tmp/community-webui-patches

experiments/yue2-3b/README.md

@@ -44,6 +44,29 @@ and stable-diffusion.cpp are not part of the Athena setup. Manual composition,
 generation, result playback, score editing and the take library work without
 those optional components.
 
+### Uploaded-audio remix (SheetSage2)
+
+The **Remix a take** drawer also accepts WAV, FLAC, MP3, M4A, OGG, Opus and
+AAC uploads. SheetSage2 transcribes the recording into an editable ABC melody
+and chord plan, then YuE2 renders that structure in a newly selected style.
+It does not preserve the original samples, singer or production verbatim.
+
+SheetSage2 is kept in `/opt/yue2/.venv-sheetsage2`, while its persistent model
+files live below `/data/models/yue2`. The environment intentionally reuses the
+image's PyTorch 2.10/CUDA 12.8 runtime: the upstream cu126 recipe is not
+Blackwell-capable. Required persistent directories are:
+
+```text
+/data/models/yue2/
+├── SheetSage2/
+└── MERT-v2-FullSong/
+```
+
+The local `SheetSage2/config.json` must point `base_model_name_or_path` to
+`/opt/yue2/models/MERT-v2-FullSong`, allowing the complete transcription path
+to run offline. Analysis and generation share the RTX 5080 and therefore run
+sequentially; the UI parks YuE2 before starting SheetSage2.
+
 The small integration patch under `community-webui-patches/` fixes the
 community release's missing `refreshArt()` function on Linux and permits the
 native YuE2 empty-lyrics request for true instrumentals. It deliberately does

experiments/yue2-3b/community-webui-patches/patch_webui.py

@@ -81,3 +81,87 @@ replace_once(
     '''<span class="label">Lyrics <em>section tags on their own line</em></span>''',
     '''<span class="label">Lyrics <em>section tags on their own line · leave empty for a true instrumental</em></span>''',
 )
+
+# The backend already ships a complete SheetSage2 upload endpoint but hides it
+# from the published console. Surface it beside the existing take-to-take
+# remix controls so uploaded songs can seed a new melody/arrangement.
+replace_once(
+    index_html,
+    '''          <p class="row-hint" id="coverStatus"></p>
+        </div>
+      </details>
+''',
+    '''          <p class="row-hint" id="coverStatus"></p>
+
+          <h4>Remix an uploaded recording</h4>
+          <p class="hint">SheetSage2 listens to an uploaded song and writes an editable melody
+            and chord score for YuE2. This is structural transcription, not a sample or a copy of
+            the original sound. After analysis, choose a new style and generate normally.</p>
+          <div class="field-row">
+            <label class="field grow">
+              <span class="label">Source audio</span>
+              <input id="coverAudio" type="file" accept=".wav,.flac,.mp3,.m4a,.ogg,.opus,.aac,audio/*" />
+            </label>
+            <div class="field reset-cell">
+              <button type="button" class="btn ghost" id="coverFromAudio">Analyse uploaded melody</button>
+            </div>
+          </div>
+          <label class="check"><input type="checkbox" id="coverAudioMelodyOnly" checked />
+            <span>Keep the melody, but let the new style rebuild the harmony</span></label>
+          <p class="row-hint" id="coverAudioStatus">Checking SheetSage2…</p>
+        </div>
+      </details>
+''',
+)
+
+replace_once(
+    app_js,
+    '''      $("coverStatus").textContent = takes.length
+        ? takes.length + " takes carry a score you can remix."
+        : "Make a song in Full plan or Melody only mode first; Direct mode keeps no score.";
+    }).catch(function () {});
+''',
+    '''      $("coverStatus").textContent = takes.length
+        ? takes.length + " takes carry a score you can remix."
+        : "Make a song in Full plan or Melody only mode first; Direct mode keeps no score.";
+      var sheetsage = data.sheetsage || {};
+      $("coverFromAudio").disabled = !sheetsage.available;
+      $("coverAudioStatus").textContent = sheetsage.available
+        ? "SheetSage2 is ready. Analysis temporarily parks YuE2 because both use the GPU."
+        : "SheetSage2 is not installed yet; uploaded-audio remix is unavailable.";
+    }).catch(function (error) {
+      $("coverFromAudio").disabled = true;
+      $("coverAudioStatus").textContent = error.message;
+    });
+''',
+)
+
+replace_once(
+    app_js,
+    '''  /* -------------------------------------------------------------- compose */
+''',
+    '''  $("coverFromAudio").addEventListener("click", function () {
+    var file = $("coverAudio").files[0];
+    if (!file) return toast("Choose an audio file first", "bad");
+    var button = $("coverFromAudio");
+    var body = new FormData();
+    body.append("file", file, file.name);
+    body.append("melody_only", $("coverAudioMelodyOnly").checked ? "true" : "false");
+    button.disabled = true;
+    button.textContent = "Analysing…";
+    $("coverAudioStatus").textContent = "Listening to “" + file.name + "”… this can take several minutes.";
+    api("/api/cover/from-audio", { method: "POST", body: body }).then(function (data) {
+      applyCoverScore(data.abc, "Melody analysed from “" + data.source + "” in " + data.seconds + " s. Choose the new style and generate.");
+      $("coverAudioStatus").textContent = "Analysis complete in " + data.seconds + " s. The editable score is loaded below.";
+    }).catch(function (error) {
+      $("coverAudioStatus").textContent = error.message;
+      toast(error.message, "bad");
+    }).then(function () {
+      button.textContent = "Analyse uploaded melody";
+      refreshCover();
+    });
+  });
+
+  /* -------------------------------------------------------------- compose */
+''',
+)