Enable SheetSage2 audio remix in YuE2 UI
This commit is contained in:
Mikei386
@@ -4,7 +4,7 @@ ARG YUE2_COMMIT=9c6c4b349be978b06a9d0d958471a07a6cdeff4d
|
||||
ARG YUE2_WEBUI_COMMIT=8fc05609bde5dcd345d7c6d57fff3da0839164c5
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends ca-certificates git libsndfile1 \
|
||||
&& apt-get install -y --no-install-recommends ca-certificates ffmpeg git libsndfile1 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN git clone https://github.com/multimodal-art-projection/YuE.git /opt/yue2 \
|
||||
@@ -21,6 +21,19 @@ RUN git clone https://github.com/Ladypoly/YuE2_WebUI.git /tmp/yue2-webui \
|
||||
&& python -m pip install --no-cache-dir -r /opt/yue2/webui/requirements.txt \
|
||||
&& rm -rf /tmp/yue2-webui
|
||||
|
||||
# SheetSage2 converts an uploaded recording into the ABC score YuE2 consumes.
|
||||
# Keep its pinned Transformers/Numpy stack in a small virtual environment, but
|
||||
# reuse the image's Blackwell-capable torch 2.10 + CUDA 12.8 installation.
|
||||
# The upstream torch 2.8+cu126 recipe cannot execute RTX 50-series kernels.
|
||||
RUN python -m venv --system-site-packages /opt/yue2/.venv-sheetsage2 \
|
||||
&& /opt/yue2/.venv-sheetsage2/bin/pip install --no-cache-dir \
|
||||
"torchaudio==2.10.0" --index-url https://download.pytorch.org/whl/cu128 \
|
||||
&& /opt/yue2/.venv-sheetsage2/bin/pip install --no-cache-dir \
|
||||
"transformers==4.45.2" "huggingface-hub==0.36.0" \
|
||||
"safetensors==0.5.3" "numpy==1.26.4" "scipy==1.13.1" \
|
||||
"mir_eval==0.8.2" "pretty_midi==0.2.10" "mido==1.3.3" \
|
||||
"setuptools==78.1.1"
|
||||
|
||||
COPY community-webui-patches /tmp/community-webui-patches
|
||||
RUN python /tmp/community-webui-patches/patch_webui.py /opt/yue2/webui \
|
||||
&& rm -rf /tmp/community-webui-patches
|
||||
|
||||
@@ -44,6 +44,29 @@ and stable-diffusion.cpp are not part of the Athena setup. Manual composition,
|
||||
generation, result playback, score editing and the take library work without
|
||||
those optional components.
|
||||
|
||||
### Uploaded-audio remix (SheetSage2)
|
||||
|
||||
The **Remix a take** drawer also accepts WAV, FLAC, MP3, M4A, OGG, Opus and
|
||||
AAC uploads. SheetSage2 transcribes the recording into an editable ABC melody
|
||||
and chord plan, then YuE2 renders that structure in a newly selected style.
|
||||
It does not preserve the original samples, singer or production verbatim.
|
||||
|
||||
SheetSage2 is kept in `/opt/yue2/.venv-sheetsage2`, while its persistent model
|
||||
files live below `/data/models/yue2`. The environment intentionally reuses the
|
||||
image's PyTorch 2.10/CUDA 12.8 runtime: the upstream cu126 recipe is not
|
||||
Blackwell-capable. Required persistent directories are:
|
||||
|
||||
```text
|
||||
/data/models/yue2/
|
||||
├── SheetSage2/
|
||||
└── MERT-v2-FullSong/
|
||||
```
|
||||
|
||||
The local `SheetSage2/config.json` must point `base_model_name_or_path` to
|
||||
`/opt/yue2/models/MERT-v2-FullSong`, allowing the complete transcription path
|
||||
to run offline. Analysis and generation share the RTX 5080 and therefore run
|
||||
sequentially; the UI parks YuE2 before starting SheetSage2.
|
||||
|
||||
The small integration patch under `community-webui-patches/` fixes the
|
||||
community release's missing `refreshArt()` function on Linux and permits the
|
||||
native YuE2 empty-lyrics request for true instrumentals. It deliberately does
|
||||
|
||||
@@ -81,3 +81,87 @@ replace_once(
|
||||
'''<span class="label">Lyrics <em>section tags on their own line</em></span>''',
|
||||
'''<span class="label">Lyrics <em>section tags on their own line · leave empty for a true instrumental</em></span>''',
|
||||
)
|
||||
|
||||
# The backend already ships a complete SheetSage2 upload endpoint but hides it
|
||||
# from the published console. Surface it beside the existing take-to-take
|
||||
# remix controls so uploaded songs can seed a new melody/arrangement.
|
||||
replace_once(
|
||||
index_html,
|
||||
''' <p class="row-hint" id="coverStatus"></p>
|
||||
</div>
|
||||
</details>
|
||||
''',
|
||||
''' <p class="row-hint" id="coverStatus"></p>
|
||||
|
||||
<h4>Remix an uploaded recording</h4>
|
||||
<p class="hint">SheetSage2 listens to an uploaded song and writes an editable melody
|
||||
and chord score for YuE2. This is structural transcription, not a sample or a copy of
|
||||
the original sound. After analysis, choose a new style and generate normally.</p>
|
||||
<div class="field-row">
|
||||
<label class="field grow">
|
||||
<span class="label">Source audio</span>
|
||||
<input id="coverAudio" type="file" accept=".wav,.flac,.mp3,.m4a,.ogg,.opus,.aac,audio/*" />
|
||||
</label>
|
||||
<div class="field reset-cell">
|
||||
<button type="button" class="btn ghost" id="coverFromAudio">Analyse uploaded melody</button>
|
||||
</div>
|
||||
</div>
|
||||
<label class="check"><input type="checkbox" id="coverAudioMelodyOnly" checked />
|
||||
<span>Keep the melody, but let the new style rebuild the harmony</span></label>
|
||||
<p class="row-hint" id="coverAudioStatus">Checking SheetSage2…</p>
|
||||
</div>
|
||||
</details>
|
||||
''',
|
||||
)
|
||||
|
||||
replace_once(
|
||||
app_js,
|
||||
''' $("coverStatus").textContent = takes.length
|
||||
? takes.length + " takes carry a score you can remix."
|
||||
: "Make a song in Full plan or Melody only mode first; Direct mode keeps no score.";
|
||||
}).catch(function () {});
|
||||
''',
|
||||
''' $("coverStatus").textContent = takes.length
|
||||
? takes.length + " takes carry a score you can remix."
|
||||
: "Make a song in Full plan or Melody only mode first; Direct mode keeps no score.";
|
||||
var sheetsage = data.sheetsage || {};
|
||||
$("coverFromAudio").disabled = !sheetsage.available;
|
||||
$("coverAudioStatus").textContent = sheetsage.available
|
||||
? "SheetSage2 is ready. Analysis temporarily parks YuE2 because both use the GPU."
|
||||
: "SheetSage2 is not installed yet; uploaded-audio remix is unavailable.";
|
||||
}).catch(function (error) {
|
||||
$("coverFromAudio").disabled = true;
|
||||
$("coverAudioStatus").textContent = error.message;
|
||||
});
|
||||
''',
|
||||
)
|
||||
|
||||
replace_once(
|
||||
app_js,
|
||||
''' /* -------------------------------------------------------------- compose */
|
||||
''',
|
||||
''' $("coverFromAudio").addEventListener("click", function () {
|
||||
var file = $("coverAudio").files[0];
|
||||
if (!file) return toast("Choose an audio file first", "bad");
|
||||
var button = $("coverFromAudio");
|
||||
var body = new FormData();
|
||||
body.append("file", file, file.name);
|
||||
body.append("melody_only", $("coverAudioMelodyOnly").checked ? "true" : "false");
|
||||
button.disabled = true;
|
||||
button.textContent = "Analysing…";
|
||||
$("coverAudioStatus").textContent = "Listening to “" + file.name + "”… this can take several minutes.";
|
||||
api("/api/cover/from-audio", { method: "POST", body: body }).then(function (data) {
|
||||
applyCoverScore(data.abc, "Melody analysed from “" + data.source + "” in " + data.seconds + " s. Choose the new style and generate.");
|
||||
$("coverAudioStatus").textContent = "Analysis complete in " + data.seconds + " s. The editable score is loaded below.";
|
||||
}).catch(function (error) {
|
||||
$("coverAudioStatus").textContent = error.message;
|
||||
toast(error.message, "bad");
|
||||
}).then(function () {
|
||||
button.textContent = "Analyse uploaded melody";
|
||||
refreshCover();
|
||||
});
|
||||
});
|
||||
|
||||
/* -------------------------------------------------------------- compose */
|
||||
''',
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user