cornball-ai · TroyHernandez · Jul 7, 2026 · Jul 3, 2026 · Jul 3, 2026 · Jul 3, 2026
diff --git a/.Rbuildignore b/.Rbuildignore
@@ -10,3 +10,5 @@
 ^fyi\.md$
 ^man-md$
 ^inst/validation/\.venv$
+^ref$
+^tasks$
diff --git a/.gitignore b/.gitignore
@@ -48,3 +48,8 @@ po/*~
 # RStudio Connect folder
 rsconnect/
 /devman-md/
+
+# Upstream reference checkouts and working notes (never committed)
+ref/
+tasks/
+.venv/
diff --git a/CLAUDE.md b/CLAUDE.md
@@ -51,7 +51,7 @@ The `txt2img_*` and `img2img` functions default to `devices = "auto"`, which:
 
 ## GPU-Poor Execution Plan (TODO)
 
-Profile-based memory optimization for constrained GPUs. Inspired by mmgp/Wan2GP approach.
+Profile-based memory optimization for constrained GPUs.
 
 ### API
 
@@ -188,7 +188,7 @@ vram_report <- function(label) {
 }
 ```
 
-### Future Optimizations (from mmgp)
+### Future Optimizations
 
 Not implemented, but worth considering:
 
@@ -320,88 +320,44 @@ See cornyverse CLAUDE.md for safetensors package setup (use cornball-ai fork unt
 - [ ] Add SD3 model support
 - [ ] ControlNet integration
 
-### LTX-2 Video Generation (In Progress)
-- [x] FlowMatch scheduler (validated against Python)
-- [x] RoPE positional embeddings (validated against Python)
-- [x] LTX2 Video VAE (3D causal convolutions) - see learnings below
-- [x] DiT transformer (audio-video) - see learnings below
-- [x] Text encoder integration (connectors + flexible backends)
-- [x] GPU-poor optimizations (wan2GP style memory profiles)
-- [x] Pipeline integration (txt2vid_ltx2)
-- [x] Video output utilities (save_video)
-- [x] Weight loading from HuggingFace safetensors
+### LTX-2.3 Video Generation (clean-room rewrite in progress)
 
-#### LTX-2 Weight Loading
+The original LTX-2.0 port was removed and is being replaced by a ground-up
+LTX-2.3 implementation written from the Apache-2.0 diffusers reference only
+(branch `feat/ltx23`; working checklist in `tasks/todo.md`).
 
-Load LTX-2 model weights from HuggingFace safetensors:
+**Reference checkouts** (in `ref/upstream/`, gitignored):
+- `ref/upstream/diffusers` — the code reference (Apache-2.0)
+- `ref/upstream/ltx2-official` — Lightricks monorepo (LTX-2 Community License;
+  used for numeric constants and file-format facts only, never as a code base)
 
-```r
-# Load VAE (2.44 GB)
-vae <- load_ltx2_vae(
-  weights_path = "~/.cache/huggingface/hub/models--Lightricks--LTX-2/vae/diffusion_pytorch_model.safetensors",
-  config_path = "~/.cache/huggingface/hub/models--Lightricks--LTX-2/vae/config.json",
-  device = "cuda",
-  dtype = "float16"
-)
-
-# Load transformer (37.8 GB, sharded across 8 files)
-transformer <- load_ltx2_transformer(
-  weights_dir = "~/.cache/huggingface/hub/models--Lightricks--LTX-2/transformer",
-  device = "cpu",  # Start on CPU, offload to GPU layer-by-layer
-  dtype = "float16"
-)
-
-# Load text connectors (2.86 GB)
-connectors <- load_ltx2_connectors(
-  weights_path = "~/.cache/huggingface/hub/models--Lightricks--LTX-2/connectors/diffusion_pytorch_model.safetensors",
-  config_path = "~/.cache/huggingface/hub/models--Lightricks--LTX-2/connectors/config.json"
-)
-```
-
-**Model sizes:**
-| Component | Size | Notes |
-|-----------|------|-------|
-| VAE | 2.44 GB | Single safetensors file |
-| Transformer | 37.8 GB | Sharded across 8 files |
-| Connectors | 2.86 GB | Single safetensors file |
-| Total (19B) | 43.3 GB | Full precision |
-| Total FP8 | 27.1 GB | Quantized |
+**Rules:**
+- Never derive code from Wan2GP (WanGP Community License 2.0 restricts
+  commercial embedding). Do not open Wan2GP source or conversions of it.
+- Every technique must have a documented public source independent of
+  Wan2GP: record it in `inst/REFERENCES.md` when adding one.
+- Model weights are downloaded by the user from `Lightricks/LTX-2.3` under the
+  LTX-2 Community License and never redistributed with the package.
+- Kept support files with independent provenance: `gemma3_text_encoder.R`
+  (from HF transformers), `tokenizer_bpe.R` (original), `flowmatch_scheduler.R`
+  (from diffusers).
 
-#### LTX2 VAE Implementation Learnings
+#### Implementation facts carried over from the 2.0 port
 
 **Temporal dimension constraint for causal downsampling:**
-For LTX2's causal 3D convolutions with stride S downsampling, the temporal dimension T must satisfy:
-```
-(T + S - 1) % S == 0
-```
-This means T % S == 1, so **T must be odd for stride=2**.
-
-Example valid sequences for 2 spatiotemporal downsampling stages:
-- T=5 → (5+1)/2=3 → (3+1)/2=2 ✓
-- T=9 → (9+1)/2=5 → (5+1)/2=3 ✓
-- T=4 → (4+1)/2=2.5 ✗ (fails unflatten)
+For causal 3D convolutions with stride S downsampling, the temporal dimension T
+must satisfy `(T + S - 1) %% S == 0`, i.e. T %% S == 1, so **T must be odd for
+stride=2** (T=4 fails the unflatten; T=5 → 3 → 2 works).
 
 **R torch `unflatten` is 1-indexed:**
 ```r
 # Python: x.unflatten(2, (-1, stride))  # dim 2 is 0-indexed
 # R: x$unflatten(3, c(-1, stride))      # dim 3 is 1-indexed
 ```
 
-**LTX2 decoder channel flow:**
-In LTX2 up blocks, `in_channels == out_channels` always. The upsampler handles channel reduction via pixel shuffle. Test inputs must match this pattern.
-
-#### LTX2 DiT Transformer Learnings
-
-**cross_attention_dim must equal inner_dim:**
-In the LTX2 transformer, `caption_projection` projects text embeddings from `caption_channels` to `inner_dim`. The transformer blocks then expect `encoder_hidden_states` to have dimension `cross_attention_dim`. These must be equal:
-```r
-# In model config:
-inner_dim = num_attention_heads * attention_head_dim  # e.g., 32 * 128 = 4096
-cross_attention_dim = 4096  # Must equal inner_dim!
-```
-
 **R torch lacks nnf_scaled_dot_product_attention:**
-Manual scaled dot-product attention is required:
+Manual scaled dot-product attention is required (and materializes the full
+[B, H, S, S] attention matrix — chunk queries at high resolution):
 ```r
 scale <- 1.0 / sqrt(head_dim)
 attn_weights <- torch::torch_matmul(query, key$transpose(-2L, -1L)) * scale
@@ -410,32 +366,18 @@ attn_weights <- torch::nnf_softmax(attn_weights, dim = -1L)
 hidden_states <- torch::torch_matmul(attn_weights, value)
 ```
 
-**Avoid function name collisions across files:**
-The `apply_interleaved_rotary_emb` function in `rope.R` expects `freqs$cos_freqs`, while a similar function in `dit_ltx2_modules.R` expects `freqs[[1]]`. Name collision caused segfaults - renamed to `apply_interleaved_rotary_emb_list` in dit module.
-
-#### LTX2 Text Encoder Learnings
-
-**Architecture: Gemma3 + Connectors:**
-LTX-2 uses Gemma3 as the text encoder, with separate connector transformers for video and audio streams:
-```
-Gemma3 → pack_text_embeds → text_proj_in → video_connector → video_embeds
-                                        → audio_connector → audio_embeds
-```
-
 **Attention mask broadcasting:**
-When using 2D attention masks [B, S], they must be expanded to [B, 1, 1, S] for broadcasting with attention weights [B, H, S, S]:
+2D attention masks [B, S] must be expanded to [B, 1, 1, S] to broadcast with
+attention weights [B, H, S, S]:
 ```r
 if (attention_mask$ndim == 2L) {
   attention_mask <- attention_mask$unsqueeze(2L)$unsqueeze(2L)
 }
 ```
 
-**Flexible text encoding backends:**
-The `encode_text_ltx2()` function supports multiple backends:
-- `"gemma3"`: Native R torch Gemma3 encoder (no Python dependency)
-- `"precomputed"`: Load from file (cached embeddings)
-- `"api"`: HTTP request to external service (Gemma container)
-- `"random"`: Random embeddings for testing
+**Avoid function name collisions across files:**
+Two same-named helpers with different calling conventions in different files
+once caused segfaults. One name, one signature, package-wide.
 
 #### Native Gemma3 Text Encoder
 

diff --git a/DESCRIPTION b/DESCRIPTION
@@ -1,8 +1,14 @@
 Package: diffuseR
 Title: Functional Interface to Diffusion Models in R
-Version: 0.0.2
-Authors@R: 
-    person("Troy", "Hernandez", email = "troy@cornball.ai", role = c("aut", "cre"))
+Version: 0.1.0
+Authors@R: c(
+    person("Troy", "Hernandez", email = "troy@cornball.ai", role = c("aut", "cre"),
+           comment = c(ORCID = "0009-0005-4248-604X")),
+    person("cornball.ai", role = "cph"),
+    person("The HuggingFace Team", role = "cph",
+           comment = "portions ported from the diffusers library (Apache-2.0); see inst/COPYRIGHTS"),
+    person("Lightricks Ltd.", role = "cph",
+           comment = "LTX checkpoint layout and pipeline constants; see inst/COPYRIGHTS"))
 Description: A native R implementation of diffusion models providing a functional 
     interface to state-of-the-art generative AI. Inspired by Hugging Face's Python 
     'diffusers' library, 'diffuseR' allows users to generate and manipulate images 

diff --git a/NAMESPACE b/NAMESPACE
@@ -1,22 +1,16 @@
 # tinyrox says don't edit this manually, but it can't stop you!
 
-export(apply_interleaved_rotary_emb)
-export(apply_split_rotary_emb)
 export(auto_devices)
 export(bpe_tokenizer)
 export(clear_vram)
 export(CLIPTokenizer)
-export(configure_vae_for_profile)
 export(ddim_scheduler_create)
 export(ddim_scheduler_step)
 export(decode_bpe)
-export(dequantize_int4)
-export(diagonal_gaussian_distribution)
-export(dit_offloaded_forward)
 export(download_component)
+export(download_ltx2)
 export(download_model)
 export(encode_bpe)
-export(encode_text_ltx2)
 export(encode_with_gemma3)
 export(filename_from_prompt)
 export(flowmatch_calculate_shift)
@@ -28,63 +22,108 @@ export(gemma3_config_ltx2)
 export(gemma3_text_model)
 export(gemma3_tokenizer)
 export(img2img)
-export(int4_linear)
-export(int4_linear_from_quantized)
 export(is_blackwell_gpu)
 export(latents_to_video)
-export(linear_to_int4)
 export(load_decoder_weights)
 export(load_gemma3_text_encoder)
-export(load_int4_into_model)
-export(load_int4_weights)
-export(load_int4_weights_into_model)
-export(load_ltx2_connectors)
-export(load_ltx2_transformer)
-export(load_ltx2_vae)
 export(load_model_component)
 export(load_pipeline)
 export(load_text_encoder_weights)
 export(load_text_encoder2_weights)
 export(load_to_gpu)
 export(load_unet_sdxl_weights)
 export(load_unet_weights)
-export(ltx_video_downsampler3d)
-export(ltx_video_upsampler3d)
-export(ltx2_memory_profile)
-export(ltx2_text_connectors)
-export(ltx2_video_causal_conv3d)
-export(ltx2_video_decoder3d)
-export(ltx2_video_down_block3d)
-export(ltx2_video_encoder3d)
-export(ltx2_video_mid_block3d)
-export(ltx2_video_resnet_block3d)
-export(ltx2_video_transformer_3d_model)
-export(ltx2_video_up_block3d)
-export(ltx2_video_vae)
-export(make_linear)
+export(ltx23_ada_layer_norm_single)
+export(ltx23_adain_filter_latent)
+export(ltx23_antialias_act1d)
+export(ltx23_apply_interleaved_rotary_emb)
+export(ltx23_apply_split_rotary_emb)
+export(ltx23_attention)
+export(ltx23_audio_causal_conv2d)
+export(ltx23_audio_decoder)
+export(ltx23_audio_downsample)
+export(ltx23_audio_encoder)
+export(ltx23_audio_mel_frontend)
+export(ltx23_audio_resnet_block)
+export(ltx23_audio_upsample)
+export(ltx23_audio_vae)
+export(ltx23_causal_conv3d)
+export(ltx23_census)
+export(ltx23_connector_transformer_1d)
+export(ltx23_denormalize_latents)
+export(ltx23_distilled_sigmas)
+export(ltx23_downsample1d)
+export(ltx23_encode_audio)
+export(ltx23_encode_video_frames)
+export(ltx23_feed_forward)
+export(ltx23_fp8_linear)
+export(ltx23_get_timestep_embedding)
+export(ltx23_is_fp8_cast_key)
+export(ltx23_kaiser_sinc_filter1d)
+export(ltx23_latent_upsampler)
+export(ltx23_load_group)
+export(ltx23_load_pipeline)
+export(ltx23_load_transformer_fp8)
+export(ltx23_load_transformer_nf4)
+export(ltx23_load_upsampler)
+export(ltx23_map_audio_vae_key)
+export(ltx23_map_connector_key)
+export(ltx23_map_dit_key)
+export(ltx23_map_vae_key)
+export(ltx23_map_vocoder_key)
+export(ltx23_mel_stft)
+export(ltx23_memory_profile)
+export(ltx23_nf4_dequantize)
+export(ltx23_nf4_linear)
+export(ltx23_nf4_quantize)
+export(ltx23_normalize_latents)
+export(ltx23_open_checkpoint)
+export(ltx23_open_fp8_checkpoint)
+export(ltx23_per_channel_rms_norm)
+export(ltx23_per_token_rms_norm)
+export(ltx23_prepare_conditioned_latents)
+export(ltx23_preprocess_frames)
+export(ltx23_quantize_fp8)
+export(ltx23_quantize_nf4)
+export(ltx23_read_audio)
+export(ltx23_read_tail_frames)
+export(ltx23_release_dequant_buffers)
+export(ltx23_rms_norm)
+export(ltx23_rotary_pos_embed)
+export(ltx23_rotary_pos_embed_1d)
+export(ltx23_set_attn_chunk)
+export(ltx23_snake_beta)
+export(ltx23_split_keys)
+export(ltx23_stage2_distilled_sigmas)
+export(ltx23_text_connectors)
+export(ltx23_tone_map_latents)
+export(ltx23_transformer)
+export(ltx23_transformer_block)
+export(ltx23_tune_gc)
+export(ltx23_upsample1d)
+export(ltx23_video_decoder3d)
+export(ltx23_video_down_block3d)
+export(ltx23_video_downsampler3d)
+export(ltx23_video_encoder3d)
+export(ltx23_video_mid_block3d)
+export(ltx23_video_resnet_block3d)
+export(ltx23_video_up_block3d)
+export(ltx23_video_upsampler3d)
+export(ltx23_video_vae)
+export(ltx23_vocoder)
+export(ltx23_vocoder_resblock)
+export(ltx23_vocoder_with_bwe)
 export(models2devices)
 export(offload_to_cpu)
-export(pack_text_embeds)
-export(pack_video_latents)
-export(per_channel_rms_norm)
 export(post_quant_conv)
 export(preprocess_image)
 export(quant_conv)
-export(quantize_int4)
-export(quantize_ltx2_transformer)
-export(quantize_ltx2_vae)
-export(quantize_model_int4)
-export(quantize_safetensors_int4)
-export(rope_embedder_create)
-export(rope_forward)
-export(rope_prepare_video_coords)
 export(save_frames)
 export(save_image)
-export(save_int4_weights)
 export(save_video)
+export(save_video_ltx23)
 export(scheduler_add_noise)
 export(sdxl_memory_profile)
-export(sequential_cfg_forward)
 export(text_encoder_native)
 export(text_encoder2_native)
 export(tokenize_gemma3)
@@ -96,10 +135,12 @@ export(unet_native)
 export(unet_native_from_torchscript)
 export(unet_sdxl_native)
 export(unet_sdxl_native_from_torchscript)
-export(unpack_video_latents)
 export(vae_decoder_native)
-export(validate_resolution)
 export(vocab_size)
 export(vram_report)
+export(write_wav)
 
 S3method(print,bpe_tokenizer)
+S3method(print,ltx23_checkpoint)
+
+importFrom(utils,head)