Merge YuE2 and SheetSage2

CMakeLists.txt

@@ -532,6 +532,7 @@ add_library(engine_core OBJECT
     src/framework/codecs/mimi_codec_runtime.cpp
     src/framework/codecs/moss_audio_tokenizer_codec_runtime.cpp
     src/framework/codecs/neural_audio.cpp
+    src/framework/codecs/oobleck_audio_vae_runtime.cpp
     src/framework/codecs/redae_codec_runtime.cpp
     src/framework/conditioners/clap_audio_conditioner_runtime.cpp
     src/framework/conditioners/cav_mae_st_conditioner_runtime.cpp
@@ -1413,6 +1414,38 @@ audiocpp_add_model(stable_audio
         engine::models::stable_audio::make_stable_audio_loader
 )
 
+audiocpp_add_model(yue2
+    SOURCES
+        src/models/yue2/ar_runtime.cpp
+        src/models/yue2/assets.cpp
+        src/models/yue2/nar_runtime.cpp
+        src/models/yue2/pipeline.cpp
+        src/models/yue2/request.cpp
+        src/models/yue2/session.cpp
+        src/models/yue2/tokenizer_text.cpp
+        src/models/yue2/types.cpp
+    INCLUDES
+        engine/models/yue2/session.h
+    LOADERS
+        engine::models::yue2::make_yue2_loader
+)
+
+audiocpp_add_model(sheetsage2
+    SOURCES
+        src/models/sheetsage/audio_frontend.cpp
+        src/models/sheetsage/processing.cpp
+        src/models/sheetsage/runtime.cpp
+        src/models/sheetsage/session.cpp
+    INCLUDES
+        engine/models/sheetsage/audio_frontend.h
+        engine/models/sheetsage/processing.h
+        engine/models/sheetsage/types.h
+        engine/models/sheetsage/runtime.h
+        engine/models/sheetsage/session.h
+    LOADERS
+        engine::models::sheetsage::make_sheetsage2_loader
+)
+
 audiocpp_add_model(supertonic
     SOURCES
         src/models/supertonic/assets.cpp
@@ -2329,6 +2362,25 @@ if (ENGINE_ENABLE_OPENMP)
     target_link_libraries(model_perf PRIVATE OpenMP::OpenMP_CXX)
 endif()
 
+if (ENGINE_BUILD_TESTS OR ENGINE_BUILD_EXTENDED_TESTS OR ENGINE_BUILD_MODEL_TESTS)
+    add_executable(yue2_vae_parity_probe
+        tests/yue2/yue2_vae_parity_probe.cpp
+    )
+    target_link_libraries(yue2_vae_parity_probe PRIVATE engine_runtime ggml)
+    if (ENGINE_ENABLE_OPENMP)
+        target_link_libraries(yue2_vae_parity_probe PRIVATE OpenMP::OpenMP_CXX)
+    endif()
+
+    add_executable(sheetsage2_decoder_parity_probe
+        src/models/sheetsage/runtime.cpp
+        tests/yue2/sheetsage2_decoder_parity_probe.cpp
+    )
+    target_link_libraries(sheetsage2_decoder_parity_probe PRIVATE engine_runtime ggml)
+    if (ENGINE_ENABLE_OPENMP)
+        target_link_libraries(sheetsage2_decoder_parity_probe PRIVATE OpenMP::OpenMP_CXX)
+    endif()
+endif()
+
 if (ENGINE_BUILD_WARMBENCH)
     function(add_engine_warmbench target_name source_file)
         add_executable(${target_name}
@@ -2468,6 +2520,13 @@ if (ENGINE_BUILD_TESTS OR ENGINE_BUILD_EXTENDED_TESTS OR ENGINE_BUILD_MODEL_TEST
         add_engine_unittest(audio_dsp_test tests/unittests/test_audio_dsp.cpp)
         add_test(NAME audio_dsp_test COMMAND audio_dsp_test)
 
+        if(sheetsage2 IN_LIST AUDIOCPP_LINKED_MODELS)
+            add_engine_unittest(sheetsage_audio_frontend_test tests/unittests/test_sheetsage_audio_frontend.cpp)
+            add_test(NAME sheetsage_audio_frontend_test COMMAND sheetsage_audio_frontend_test --log)
+            add_engine_unittest(sheetsage_processing_test tests/unittests/test_sheetsage_processing.cpp)
+            add_test(NAME sheetsage_processing_test COMMAND sheetsage_processing_test --log)
+        endif()
+
         add_engine_unittest(midi_file_test tests/unittests/test_midi_file.cpp)
         add_test(NAME midi_file_test COMMAND midi_file_test)
 

docs/models/yue2.md

@@ -0,0 +1,151 @@
+# YuE2
+
+YuE2 is wired as `--family yue2 --task gen`. It generates music from lyrics and
+a style prompt, with optional symbolic ABC conditioning.
+
+## Quick Start
+
+Default packaged GGUF layout:
+
+```bash
+./build/debug/bin/audiocpp_cli \
+  --task gen \
+  --family yue2 \
+  --model models/Yue2-3B-GGUF \
+  --backend cuda \
+  --threads 8 \
+  --lyrics "[Verse]
+Soft morning light is touching the window.
+I hear the city waking below.
+[Chorus]
+Stay with the rhythm, let it carry us home.
+Sing with the sunrise, we are never alone." \
+  --request-option style="English, indie pop, bright acoustic guitar, soft drums, warm lead vocal, polished demo mix" \
+  --request-option cot=off \
+  --seed 831001 \
+  --out yue2.wav \
+  --log
+```
+
+The default session loads `yue2-3b-q8_0.gguf` for the main AR/NAR model and
+`yue2-vae-f16.gguf` for the VAE from the model root.
+
+## Model
+
+| Field | Value |
+|---|---|
+| Family | `yue2` |
+| Model directory | `models/Yue2-3B-GGUF` |
+| Task | `gen` |
+| Main GGUF default | `yue2-3b-q8_0.gguf` |
+| VAE GGUF default | `yue2-vae-f16.gguf` |
+| Required sidecars | `sidecars/yue2-model-config.json`, `sidecars/yue2-generation-config.json`, `sidecars/yue2-qwen.tiktoken`, `sidecars/yue2-vae-config.json` |
+| Lyrics input | `--lyrics`; `--text` is accepted as a fallback |
+| Style input | `--request-option style=<prompt>` |
+
+## Component Selection
+
+Select the BF16 main model:
+
+```bash
+./build/debug/bin/audiocpp_cli \
+  --task gen \
+  --family yue2 \
+  --model models/Yue2-3B-GGUF \
+  --backend cuda \
+  --threads 8 \
+  --session-option yue2.model_gguf=yue2-3b-bf16.gguf \
+  --session-option yue2.vae_gguf=yue2-vae-f16.gguf \
+  --lyrics "[Verse]
+Soft morning light is touching the window.
+[Chorus]
+Stay with the rhythm, let it carry us home." \
+  --request-option style="English, pop rock, bright guitars, clean drums, warm vocal" \
+  --request-option cot=off \
+  --seed 831001 \
+  --out yue2-bf16.wav \
+  --log
+```
+
+Select the F32 VAE:
+
+```bash
+--session-option yue2.vae_gguf=yue2-vae-f32.gguf
+```
+
+The component paths are relative to `--model`; absolute paths are rejected.
+
+## ABC Conditioning
+
+Use `cot=melody` or `cot=full` to run the symbolic route. External ABC requires
+one of those modes:
+
+```bash
+./build/debug/bin/audiocpp_cli \
+  --task gen \
+  --family yue2 \
+  --model models/Yue2-3B-GGUF \
+  --backend cuda \
+  --threads 8 \
+  --lyrics "[Verse]
+Write the melody over this score." \
+  --request-option style="English, folk pop, acoustic guitar, steady drums" \
+  --request-option cot=melody \
+  --request-option abc_file=/path/to/score.abc \
+  --seed 831001 \
+  --out yue2-abc.wav \
+  --log
+```
+
+Inline ABC can be passed with `--request-option abc=<abc text>`.
+
+## Request Options
+
+| Option | Values | Default | Meaning |
+|---|---|---:|---|
+| `--lyrics` | text | required | Song lyrics. |
+| `--text` | text | empty | Fallback lyrics source when `--lyrics` is not supplied. |
+| `--request-option style=<text>` | text | required | Music style prompt. |
+| `--request-option cot=<mode>` | `off`, `melody`, `full` | `full` | Symbolic planning route. |
+| `--request-option abc=<text>` | ABC text | empty | Inline ABC score; requires `cot=melody` or `cot=full`. |
+| `--request-option abc_file=<path>` | path | empty | ABC score file; requires `cot=melody` or `cot=full`. |
+| `--request-option semantic_codes_file=<path>` | raw int32 file | empty | Teacher-forced semantic codec IDs for parity/debug runs. |
+| `--request-option nar_noise_file=<path>` | raw float32 file | empty | Teacher-forced NAR noise rows with 64 columns for parity/debug runs. |
+| `--request-option cfg_scale=<f>` | `0..20` | `1.01` for `cot=off`, otherwise `1.0` | Semantic classifier-free guidance scale. |
+| `--request-option num_inference_steps=<n>` | integer > 0 | `32` | NAR midpoint ODE steps. |
+| `--seed <n>` | integer in `[0, 2^63)` | `831001` | Generation seed. Equivalent to `--request-option seed=<n>`. |
+
+## Sampling Options
+
+| Option | Values | Default | Meaning |
+|---|---|---:|---|
+| `--request-option abc_temperature=<f>` | `0..5` | `0.7` | ABC planner sampling temperature. |
+| `--request-option abc_top_p=<f>` | `0..1` | `0.9` | ABC planner nucleus sampling probability. |
+| `--request-option abc_top_k=<n>` | integer >= 1 | `30` | ABC planner top-k limit. |
+| `--request-option abc_repetition_penalty=<f>` | float > 0 | `1.005` | ABC planner repetition penalty. |
+| `--request-option abc_penalty_window=<n>` | integer >= 1 | `100` | ABC planner repetition penalty window. |
+| `--request-option abc_min_tokens=<n>` | integer >= 0 | `32` | Minimum ABC planner tokens before EOS is accepted. |
+| `--request-option abc_max_tokens=<n>` | integer >= `abc_min_tokens` | `4096` | Maximum ABC planner tokens. |
+| `--request-option semantic_temperature=<f>` | `0..5` | `1.0` | Semantic codec sampling temperature. |
+| `--request-option semantic_top_p=<f>` | `0..1` | `0.95` | Semantic codec nucleus sampling probability. |
+| `--request-option semantic_top_k=<n>` | integer >= 1 | `100` | Semantic codec top-k limit. |
+| `--request-option semantic_repetition_penalty=<f>` | float > 0 | `1.2` | Semantic codec repetition penalty. |
+| `--request-option semantic_penalty_window=<n>` | integer >= 1 | `50` | Semantic codec repetition penalty window. |
+| `--request-option semantic_min_tokens=<n>` | integer >= 0 | `200` | Minimum semantic tokens before EOS is accepted. |
+| `--request-option semantic_max_tokens=<n>` | integer >= `semantic_min_tokens` | `9000` | Maximum semantic codec tokens. |
+
+## Session Options
+
+| Option | Values | Default | Meaning |
+|---|---|---:|---|
+| `--session-option yue2.model_gguf=<file>` | relative GGUF path | `yue2-3b-q8_0.gguf` | Main AR/NAR component. |
+| `--session-option yue2.vae_gguf=<file>` | relative GGUF path | `yue2-vae-f16.gguf` | VAE component. |
+| `--session-option yue2.weight_type=<type>` | `native`, `f32`, `f16`, `bf16`, `q8_0`, `q4_0`, `q4_k` | `native` | Shared weight storage fallback for the main model and VAE. |
+| `--session-option yue2.model_weight_type=<type>` | `native`, `f32`, `f16`, `bf16`, `q8_0`, `q4_0`, `q4_k` | `native` | Main model weight storage override. |
+| `--session-option yue2.vae_weight_type=<type>` | `native`, `f32`, `f16`, `bf16`, `q8_0`, `q4_0`, `q4_k` | `native` | VAE weight storage override. |
+| `--session-option yue2.model_weight_context_mb=<n>` | MiB integer >= 1 | `6144` | Main model weight context size. |
+| `--session-option yue2.vae_weight_context_mb=<n>` | MiB integer >= 1 | `1536` | VAE weight context size. |
+| `--session-option yue2.ar_prefill_graph_arena_mb=<n>` | MiB integer >= 1 | `4096` | AR prefill graph arena size. |
+| `--session-option yue2.ar_decode_graph_arena_mb=<n>` | MiB integer >= 1 | `1536` | AR one-token decode graph arena size. |
+| `--session-option yue2.nar_graph_arena_mb=<n>` | MiB integer >= 1 | `6144` | NAR acoustic flow graph arena size. |
+| `--session-option yue2.vae_graph_arena_mb=<n>` | MiB integer >= 1 | `1536` | VAE decode graph arena size. |

include/engine/framework/audio/conversion.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include "engine/framework/audio/resampling.h"
 #include "engine/framework/audio/wav_reader.h"
 
 #include <cstdint>
@@ -46,6 +47,14 @@ std::vector<float> convert_interleaved_audio_to_mono_linear_resampled(
     int channel_count,
     int target_sample_rate_hz);
 
+std::vector<float> convert_interleaved_audio_to_mono_torchaudio_sinc_hann_resampled(
+    const std::vector<float> & interleaved_samples,
+    int sample_rate_hz,
+    int channel_count,
+    int target_sample_rate_hz,
+    const TorchaudioSincHannResampleOptions & options = {},
+    MonoMixAccumulation accumulation = MonoMixAccumulation::Float32);
+
 std::vector<float> read_wav_f32_as_mono_linear_resampled(
     const std::filesystem::path & path,
     int target_sample_rate_hz);

include/engine/framework/codecs/oobleck_audio_vae_runtime.h

@@ -0,0 +1,65 @@
+#pragma once
+
+#include "engine/framework/assets/tensor_source.h"
+#include "engine/framework/core/backend.h"
+#include "engine/framework/core/execution_context.h"
+#include "engine/framework/modules/conv_modules.h"
+#include "engine/framework/runtime/session.h"
+
+#include <cstddef>
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace engine::codecs {
+
+struct OobleckAudioVaeConfig {
+    int64_t sample_rate = 48000;
+    int64_t audio_channels = 2;
+    int64_t channels = 64;
+    int64_t encoder_latent_dim = 128;
+    int64_t decoder_latent_dim = 64;
+    std::vector<int64_t> c_mults{1, 2, 4, 8, 16, 32};
+    std::vector<int64_t> strides{2, 2, 4, 4, 5, 6};
+    bool use_snake = true;
+    bool snake_logscale = true;
+    bool final_tanh = false;
+    std::string encoder_prefix = "encoder";
+    std::string decoder_prefix = "decoder";
+};
+
+struct OobleckAudioVaeRuntimeOptions {
+    size_t graph_arena_bytes = 512ull * 1024ull * 1024ull;
+    size_t weight_context_bytes = 1400ull * 1024ull * 1024ull;
+    assets::TensorStorageType weight_storage_type = assets::TensorStorageType::Native;
+};
+
+class OobleckAudioVaeRuntime {
+public:
+    OobleckAudioVaeRuntime(
+        std::shared_ptr<const assets::TensorSource> source,
+        core::ExecutionContext & execution,
+        OobleckAudioVaeConfig config = {},
+        OobleckAudioVaeRuntimeOptions options = {});
+    ~OobleckAudioVaeRuntime();
+
+    OobleckAudioVaeRuntime(const OobleckAudioVaeRuntime &) = delete;
+    OobleckAudioVaeRuntime & operator=(const OobleckAudioVaeRuntime &) = delete;
+    OobleckAudioVaeRuntime(OobleckAudioVaeRuntime &&) noexcept;
+    OobleckAudioVaeRuntime & operator=(OobleckAudioVaeRuntime &&) noexcept;
+
+    std::vector<float> encode_planar(const std::vector<float> & planar_audio, int64_t frames);
+    std::vector<float> decode_planar(const std::vector<float> & latents, int64_t batch, int64_t latent_frames);
+    std::vector<runtime::AudioBuffer> decode(const std::vector<float> & latents, int64_t batch, int64_t latent_frames);
+
+    void prepare_encode(int64_t frames);
+    void prepare_decode(int64_t batch, int64_t latent_frames);
+    void release_runtime_graphs();
+
+private:
+    struct Impl;
+    std::unique_ptr<Impl> impl_;
+};
+
+}  // namespace engine::codecs

include/engine/framework/modules/activation_modules.h

@@ -154,6 +154,32 @@ class Snake1dModule {
     Snake1dConfig config_;
 };
 
+struct SnakeBeta1dConfig {
+    int64_t hidden_size = 0;
+    bool logscale = true;
+};
+
+struct SnakeBeta1dWeights {
+    core::TensorValue alpha;
+    core::TensorValue beta;
+};
+
+class SnakeBeta1dModule {
+public:
+    explicit SnakeBeta1dModule(SnakeBeta1dConfig config);
+
+    const SnakeBeta1dConfig & config() const noexcept;
+    const core::ModuleSchema & schema() const noexcept;
+    core::TensorValue build(
+        core::ModuleBuildContext & ctx,
+        const core::TensorValue & input,
+        const SnakeBeta1dWeights & weights) const;
+    static const core::ModuleSchema & static_schema() noexcept;
+
+private:
+    SnakeBeta1dConfig config_;
+};
+
 enum class AliasFreeActivationKind {
     SnakeBeta,
 };

include/engine/framework/modules/transformers/qwen_causal_decode_runtime.h

@@ -44,6 +44,13 @@ struct QwenCausalPrefillResult {
     runtime::TransformerKVState state;
 };
 
+struct QwenCausalPrefillIntoDecodeResult {
+    std::vector<float> logits;
+    std::vector<float> hidden;
+    int64_t current_end = 0;
+    int64_t valid_steps = 0;
+};
+
 struct QwenCausalBatchedPrefillResult {
     std::vector<float> logits;
     std::vector<float> hidden;
@@ -68,6 +75,9 @@ class QwenCausalDecodeRuntime {
 
     QwenCausalPrefillResult prefill_tokens(const std::vector<int32_t> & token_ids);
     QwenCausalPrefillResult prefill_embeddings(const std::vector<float> & embeddings, int64_t steps);
+    QwenCausalPrefillIntoDecodeResult prefill_tokens_into_decode_cache(
+        const std::vector<int32_t> & token_ids,
+        int64_t required_cache_steps);
 
     // Prefill bounded blocks directly into the token-decode cache on the backend.
     // No host KV export/import; subsequent decode_token calls continue this state.
@@ -86,6 +96,7 @@ class QwenCausalDecodeRuntime {
     void start_decode_tokens(const runtime::TransformerKVState & state, int64_t required_cache_steps);
     void start_decode_embeddings(const runtime::TransformerKVState & state, int64_t required_cache_steps);
     QwenCausalDecodeStepResult decode_token(int32_t token);
+    void decode_token_into(int32_t token, QwenCausalDecodeStepResult & out);
     QwenCausalDecodeStepResult decode_embedding(const std::vector<float> & embedding);
 
     void start_decode_tokens_batched(