From c8f5dbfdd768b43d516dcd2ace3f9396d42bb7b7 Mon Sep 17 00:00:00 2001 From: TroyHernandez Date: Fri, 3 Jul 2026 10:49:44 -0500 Subject: [PATCH 01/83] Remove LTX-2.0 implementation ahead of clean-room LTX-2.3 rewrite The LTX-2.0 port is superseded by a ground-up LTX-2.3 implementation written from the Apache-2.0 diffusers reference. Generic VRAM utilities and the SDXL memory profile move to R/vram.R and R/memory_sdxl.R; encode_with_gemma3() now returns raw stacked hidden states for the downstream connector modules. Kept: Gemma3 encoder, BPE tokenizer, FlowMatch scheduler. --- NAMESPACE | 48 +- R/dit_ltx2.R | 936 --------- R/dit_ltx2_modules.R | 802 -------- R/gemma3_text_encoder.R | 36 +- R/gpu_poor.R | 1785 ----------------- R/memory_sdxl.R | 118 ++ R/rope.R | 360 ---- R/text_encoder_ltx2.R | 714 ------- R/txt2vid_ltx2.R | 615 ------ R/vae_ltx2.R | 913 --------- R/vae_ltx2_modules.R | 746 ------- R/vram.R | 202 ++ inst/tinytest/test_dit_ltx2.R | 256 --- inst/tinytest/test_gpu_poor.R | 82 - inst/tinytest/test_ltx2_weights.R | 101 - inst/tinytest/test_rope.R | 140 -- inst/tinytest/test_rope_python_validation.R | 132 -- inst/tinytest/test_text_encoder_ltx2.R | 155 -- inst/tinytest/test_txt2vid_ltx2.R | 46 - inst/tinytest/test_vae_ltx2.R | 242 --- man/apply_interleaved_rotary_emb.Rd | 19 - man/apply_interleaved_rotary_emb_list.Rd | 11 - man/apply_split_rotary_emb.Rd | 19 - man/configure_vae_for_profile.Rd | 25 - man/ddim_scheduler_create.Rd | 18 +- man/ddim_scheduler_step.Rd | 22 +- man/dequantize_int4.Rd | 26 - man/diagonal_gaussian_distribution.Rd | 13 - man/dit_offloaded_forward.Rd | 53 - man/download_component.Rd | 9 +- man/download_model.Rd | 15 +- man/encode_bpe.Rd | 11 +- man/encode_text_ltx2.Rd | 52 - man/encode_with_gemma3.Rd | 23 +- man/feed_forward.Rd | 19 - man/flowmatch_calculate_shift.Rd | 9 +- man/flowmatch_scheduler_create.Rd | 18 +- man/flowmatch_scheduler_step.Rd | 9 +- man/flowmatch_set_timesteps.Rd | 10 +- man/gelu_activation.Rd | 11 - man/gemma3_rotary_embedding.Rd | 8 +- man/get_timestep_embedding.Rd | 18 - man/gpu_poor.Rd | 8 - man/img2img.Rd | 30 +- man/int4_linear.Rd | 43 - man/int4_linear_from_quantized.Rd | 30 - man/linear_to_int4.Rd | 20 - man/load_gemma3_text_encoder.Rd | 8 +- man/load_int4_into_model.Rd | 41 - man/load_int4_weights.Rd | 27 - man/load_int4_weights_into_model.Rd | 21 - man/load_ltx2_connector_weights.Rd | 18 - man/load_ltx2_connectors.Rd | 30 - man/load_ltx2_transformer.Rd | 31 - man/load_ltx2_transformer_sharded.Rd | 25 - man/load_ltx2_transformer_weights.Rd | 18 - man/load_ltx2_vae.Rd | 31 - man/load_ltx2_vae_weights.Rd | 21 - man/load_model_component.Rd | 10 +- man/load_pipeline.Rd | 13 +- man/ltx2_ada_layer_norm_single.Rd | 15 - man/ltx2_attention.Rd | 24 - man/ltx2_audio_video_rotary_pos_embed.Rd | 27 - man/ltx2_connector_transformer_1d.Rd | 22 - man/ltx2_memory_profile.Rd | 45 - man/ltx2_rotary_pos_embed_1d.Rd | 18 - man/ltx2_text_connectors.Rd | 60 - man/ltx2_transformer_block_1d.Rd | 18 - man/ltx2_video_causal_conv3d.Rd | 35 - man/ltx2_video_decoder3d.Rd | 54 - man/ltx2_video_down_block3d.Rd | 39 - man/ltx2_video_encoder3d.Rd | 45 - man/ltx2_video_mid_block3d.Rd | 36 - man/ltx2_video_resnet_block3d.Rd | 37 - man/ltx2_video_transformer_3d_model.Rd | 107 - man/ltx2_video_transformer_block.Rd | 27 - man/ltx2_video_up_block3d.Rd | 48 - man/ltx2_video_vae.Rd | 101 - man/ltx_video_downsampler3d.Rd | 24 - man/ltx_video_upsampler3d.Rd | 27 - man/make_linear.Rd | 27 - man/models2devices.Rd | 8 +- man/pack_text_embeds.Rd | 34 - man/pack_video_latents.Rd | 21 - man/per_channel_rms_norm.Rd | 16 - ...alpha_combined_timestep_size_embeddings.Rd | 15 - man/pixart_alpha_text_projection.Rd | 16 - man/quantize_int4.Rd | 36 - man/quantize_ltx2_transformer.Rd | 58 - man/quantize_ltx2_vae.Rd | 34 - man/quantize_model_int4.Rd | 20 - man/quantize_safetensors_int4.Rd | 31 - man/rms_norm.Rd | 11 - man/rope.Rd | 14 - man/rope_embedder_create.Rd | 54 - man/rope_forward.Rd | 25 - man/rope_prepare_video_coords.Rd | 38 - man/save_int4_weights.Rd | 46 - man/save_video.Rd | 11 +- man/save_video_ffmpeg.Rd | 10 +- man/save_video_frames.Rd | 24 - man/sequential_cfg_forward.Rd | 46 - man/text_encoder2_native.Rd | 10 +- man/text_encoder_native.Rd | 12 +- man/timestep_embedding.Rd | 8 +- man/timestep_embedding_module.Rd | 16 - man/timesteps_module.Rd | 16 - man/tokenize_gemma3.Rd | 9 +- man/txt2img_sd21.Rd | 29 +- man/txt2img_sdxl.Rd | 32 +- man/txt2vid_ltx2.Rd | 96 - man/unet_native.Rd | 12 +- man/unet_sdxl_native.Rd | 17 +- man/unpack_video_latents.Rd | 33 - man/vae_ltx2.Rd | 8 - man/vae_ltx2_modules.Rd | 8 - man/validate_resolution.Rd | 28 - man/vram.Rd | 8 + 118 files changed, 443 insertions(+), 10575 deletions(-) delete mode 100644 R/dit_ltx2.R delete mode 100644 R/dit_ltx2_modules.R delete mode 100644 R/gpu_poor.R create mode 100644 R/memory_sdxl.R delete mode 100644 R/rope.R delete mode 100644 R/text_encoder_ltx2.R delete mode 100644 R/txt2vid_ltx2.R delete mode 100644 R/vae_ltx2.R delete mode 100644 R/vae_ltx2_modules.R create mode 100644 R/vram.R delete mode 100644 inst/tinytest/test_dit_ltx2.R delete mode 100644 inst/tinytest/test_gpu_poor.R delete mode 100644 inst/tinytest/test_ltx2_weights.R delete mode 100644 inst/tinytest/test_rope.R delete mode 100644 inst/tinytest/test_rope_python_validation.R delete mode 100644 inst/tinytest/test_text_encoder_ltx2.R delete mode 100644 inst/tinytest/test_txt2vid_ltx2.R delete mode 100644 inst/tinytest/test_vae_ltx2.R delete mode 100644 man/apply_interleaved_rotary_emb.Rd delete mode 100644 man/apply_interleaved_rotary_emb_list.Rd delete mode 100644 man/apply_split_rotary_emb.Rd delete mode 100644 man/configure_vae_for_profile.Rd delete mode 100644 man/dequantize_int4.Rd delete mode 100644 man/diagonal_gaussian_distribution.Rd delete mode 100644 man/dit_offloaded_forward.Rd delete mode 100644 man/encode_text_ltx2.Rd delete mode 100644 man/feed_forward.Rd delete mode 100644 man/gelu_activation.Rd delete mode 100644 man/get_timestep_embedding.Rd delete mode 100644 man/gpu_poor.Rd delete mode 100644 man/int4_linear.Rd delete mode 100644 man/int4_linear_from_quantized.Rd delete mode 100644 man/linear_to_int4.Rd delete mode 100644 man/load_int4_into_model.Rd delete mode 100644 man/load_int4_weights.Rd delete mode 100644 man/load_int4_weights_into_model.Rd delete mode 100644 man/load_ltx2_connector_weights.Rd delete mode 100644 man/load_ltx2_connectors.Rd delete mode 100644 man/load_ltx2_transformer.Rd delete mode 100644 man/load_ltx2_transformer_sharded.Rd delete mode 100644 man/load_ltx2_transformer_weights.Rd delete mode 100644 man/load_ltx2_vae.Rd delete mode 100644 man/load_ltx2_vae_weights.Rd delete mode 100644 man/ltx2_ada_layer_norm_single.Rd delete mode 100644 man/ltx2_attention.Rd delete mode 100644 man/ltx2_audio_video_rotary_pos_embed.Rd delete mode 100644 man/ltx2_connector_transformer_1d.Rd delete mode 100644 man/ltx2_memory_profile.Rd delete mode 100644 man/ltx2_rotary_pos_embed_1d.Rd delete mode 100644 man/ltx2_text_connectors.Rd delete mode 100644 man/ltx2_transformer_block_1d.Rd delete mode 100644 man/ltx2_video_causal_conv3d.Rd delete mode 100644 man/ltx2_video_decoder3d.Rd delete mode 100644 man/ltx2_video_down_block3d.Rd delete mode 100644 man/ltx2_video_encoder3d.Rd delete mode 100644 man/ltx2_video_mid_block3d.Rd delete mode 100644 man/ltx2_video_resnet_block3d.Rd delete mode 100644 man/ltx2_video_transformer_3d_model.Rd delete mode 100644 man/ltx2_video_transformer_block.Rd delete mode 100644 man/ltx2_video_up_block3d.Rd delete mode 100644 man/ltx2_video_vae.Rd delete mode 100644 man/ltx_video_downsampler3d.Rd delete mode 100644 man/ltx_video_upsampler3d.Rd delete mode 100644 man/make_linear.Rd delete mode 100644 man/pack_text_embeds.Rd delete mode 100644 man/pack_video_latents.Rd delete mode 100644 man/per_channel_rms_norm.Rd delete mode 100644 man/pixart_alpha_combined_timestep_size_embeddings.Rd delete mode 100644 man/pixart_alpha_text_projection.Rd delete mode 100644 man/quantize_int4.Rd delete mode 100644 man/quantize_ltx2_transformer.Rd delete mode 100644 man/quantize_ltx2_vae.Rd delete mode 100644 man/quantize_model_int4.Rd delete mode 100644 man/quantize_safetensors_int4.Rd delete mode 100644 man/rms_norm.Rd delete mode 100644 man/rope.Rd delete mode 100644 man/rope_embedder_create.Rd delete mode 100644 man/rope_forward.Rd delete mode 100644 man/rope_prepare_video_coords.Rd delete mode 100644 man/save_int4_weights.Rd delete mode 100644 man/save_video_frames.Rd delete mode 100644 man/sequential_cfg_forward.Rd delete mode 100644 man/timestep_embedding_module.Rd delete mode 100644 man/timesteps_module.Rd delete mode 100644 man/txt2vid_ltx2.Rd delete mode 100644 man/unpack_video_latents.Rd delete mode 100644 man/vae_ltx2.Rd delete mode 100644 man/vae_ltx2_modules.Rd delete mode 100644 man/validate_resolution.Rd create mode 100644 man/vram.Rd diff --git a/NAMESPACE b/NAMESPACE index 8133d0b..63b2d0a 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,22 +1,15 @@ # tinyrox says don't edit this manually, but it can't stop you! -export(apply_interleaved_rotary_emb) -export(apply_split_rotary_emb) export(auto_devices) export(bpe_tokenizer) export(clear_vram) export(CLIPTokenizer) -export(configure_vae_for_profile) export(ddim_scheduler_create) export(ddim_scheduler_step) export(decode_bpe) -export(dequantize_int4) -export(diagonal_gaussian_distribution) -export(dit_offloaded_forward) export(download_component) export(download_model) export(encode_bpe) -export(encode_text_ltx2) export(encode_with_gemma3) export(filename_from_prompt) export(flowmatch_calculate_shift) @@ -28,19 +21,10 @@ export(gemma3_config_ltx2) export(gemma3_text_model) export(gemma3_tokenizer) export(img2img) -export(int4_linear) -export(int4_linear_from_quantized) export(is_blackwell_gpu) export(latents_to_video) -export(linear_to_int4) export(load_decoder_weights) export(load_gemma3_text_encoder) -export(load_int4_into_model) -export(load_int4_weights) -export(load_int4_weights_into_model) -export(load_ltx2_connectors) -export(load_ltx2_transformer) -export(load_ltx2_vae) export(load_model_component) export(load_pipeline) export(load_text_encoder_weights) @@ -48,58 +32,30 @@ export(load_text_encoder2_weights) export(load_to_gpu) export(load_unet_sdxl_weights) export(load_unet_weights) -export(ltx_video_downsampler3d) -export(ltx_video_upsampler3d) -export(ltx2_memory_profile) -export(ltx2_text_connectors) -export(ltx2_video_causal_conv3d) -export(ltx2_video_decoder3d) -export(ltx2_video_down_block3d) -export(ltx2_video_encoder3d) -export(ltx2_video_mid_block3d) -export(ltx2_video_resnet_block3d) -export(ltx2_video_transformer_3d_model) -export(ltx2_video_up_block3d) -export(ltx2_video_vae) -export(make_linear) export(models2devices) export(offload_to_cpu) -export(pack_text_embeds) -export(pack_video_latents) -export(per_channel_rms_norm) export(post_quant_conv) export(preprocess_image) export(quant_conv) -export(quantize_int4) -export(quantize_ltx2_transformer) -export(quantize_ltx2_vae) -export(quantize_model_int4) -export(quantize_safetensors_int4) -export(rope_embedder_create) -export(rope_forward) -export(rope_prepare_video_coords) export(save_frames) export(save_image) -export(save_int4_weights) export(save_video) export(scheduler_add_noise) export(sdxl_memory_profile) -export(sequential_cfg_forward) export(text_encoder_native) export(text_encoder2_native) export(tokenize_gemma3) export(txt2img) export(txt2img_sd21) export(txt2img_sdxl) -export(txt2vid_ltx2) export(unet_native) export(unet_native_from_torchscript) export(unet_sdxl_native) export(unet_sdxl_native_from_torchscript) -export(unpack_video_latents) export(vae_decoder_native) -export(validate_resolution) export(vocab_size) export(vram_report) S3method(print,bpe_tokenizer) + +importFrom(utils,head) diff --git a/R/dit_ltx2.R b/R/dit_ltx2.R deleted file mode 100644 index 94288cb..0000000 --- a/R/dit_ltx2.R +++ /dev/null @@ -1,936 +0,0 @@ -# LTX2 DiT Transformer Model -# Full audio-video transformer matching HuggingFace diffusers implementation - -# ------------------------------------------------------------------------------ -# LTX2 Audio-Video Rotary Positional Embeddings -# ------------------------------------------------------------------------------ - -#' LTX2 Audio-Video Rotary Positional Embeddings -#' @keywords internal -ltx2_audio_video_rotary_pos_embed <- torch::nn_module( - "LTX2AudioVideoRotaryPosEmbed", - initialize = function( - dim, - patch_size = 1L, - patch_size_t = 1L, - base_num_frames = 20L, - base_height = 2048L, - base_width = 2048L, - sampling_rate = 16000L, - hop_length = 160L, - scale_factors = c(8L, 32L, 32L), - theta = 10000.0, - causal_offset = 1L, - modality = "video", - double_precision = TRUE, - rope_type = "interleaved", - num_attention_heads = 32L - ) { - - self$dim <- dim - self$patch_size <- patch_size - self$patch_size_t <- patch_size_t - - if (!rope_type %in% c("interleaved", "split")) { - stop("rope_type must be 'interleaved' or 'split'") - } - self$rope_type <- rope_type - - self$base_num_frames <- base_num_frames - self$num_attention_heads <- num_attention_heads - - # Video-specific - self$base_height <- base_height - self$base_width <- base_width - - # Audio-specific - self$sampling_rate <- sampling_rate - self$hop_length <- hop_length - self$audio_latents_per_second <- as.numeric(sampling_rate) / as.numeric(hop_length) / as.numeric(scale_factors[1]) - - self$scale_factors <- scale_factors - self$theta <- theta - self$causal_offset <- causal_offset - - self$modality <- modality - if (!modality %in% c("video", "audio")) { - stop("modality must be 'video' or 'audio'") - } - self$double_precision <- double_precision - }, - - prepare_video_coords = function( - batch_size, - num_frames, - height, - width, - device, - fps = 24.0 - ) { - # Generate grid coordinates for each spatiotemporal dimension - grid_f <- torch::torch_arange(start = 0, end = num_frames - 1L, step = self$patch_size_t, - dtype = torch::torch_float32(), device = device) - grid_h <- torch::torch_arange(start = 0, end = height - 1L, step = self$patch_size, - dtype = torch::torch_float32(), device = device) - grid_w <- torch::torch_arange(start = 0, end = width - 1L, step = self$patch_size, - dtype = torch::torch_float32(), device = device) - - # Create meshgrid (F, H, W order) - grids <- torch::torch_meshgrid(list(grid_f, grid_h, grid_w), indexing = "ij") - grid <- torch::torch_stack(grids, dim = 1L) # [3, N_F, N_H, N_W] - - # Get patch boundaries - patch_size_vec <- c(self$patch_size_t, self$patch_size, self$patch_size) - patch_size_delta <- torch::torch_tensor(patch_size_vec, dtype = grid$dtype, device = grid$device) - patch_ends <- grid + patch_size_delta$view(c(3L, 1L, 1L, 1L)) - - # Combine start and end coordinates - latent_coords <- torch::torch_stack(list(grid, patch_ends), dim = - 1L) # [3, N_F, N_H, N_W, 2] - - # Reshape to [batch_size, 3, num_patches, 2] - latent_coords <- latent_coords$flatten(start_dim = 2L, end_dim = 4L) - latent_coords <- latent_coords$unsqueeze(1L)$`repeat`(c(batch_size, 1L, 1L, 1L)) - - # Scale to pixel space - scale_tensor <- torch::torch_tensor(self$scale_factors, device = latent_coords$device) - pixel_coords <- latent_coords * scale_tensor$view(c(1L, 3L, 1L, 1L)) - - # Handle causal offset for temporal dimension - pixel_coords[, 1,,] <- (pixel_coords[, 1,,] + self$causal_offset - self$scale_factors[1])$clamp(min = 0) - - # Scale temporal by FPS - pixel_coords[, 1,,] <- pixel_coords[, 1,,] / fps - - pixel_coords - }, - - prepare_audio_coords = function( - batch_size, - num_frames, - device, - shift = 0L - ) { - # Generate coordinates in the frame (time) dimension - grid_f <- torch::torch_arange(start = shift, end = num_frames + shift - 1L, step = self$patch_size_t, - dtype = torch::torch_float32(), device = device) - - audio_scale_factor <- self$scale_factors[1] - - # Calculate start timestamps - grid_start_mel <- grid_f * audio_scale_factor - grid_start_mel <- (grid_start_mel + self$causal_offset - audio_scale_factor)$clamp(min = 0) - grid_start_s <- grid_start_mel * self$hop_length / self$sampling_rate - - # Calculate end timestamps - grid_end_mel <- (grid_f + self$patch_size_t) * audio_scale_factor - grid_end_mel <- (grid_end_mel + self$causal_offset - audio_scale_factor)$clamp(min = 0) - grid_end_s <- grid_end_mel * self$hop_length / self$sampling_rate - - audio_coords <- torch::torch_stack(list(grid_start_s, grid_end_s), dim = - 1L) # [num_patches, 2] - audio_coords <- audio_coords$unsqueeze(1L)$expand(c(batch_size, - 1L, - 1L)) # [batch_size, num_patches, 2] - audio_coords <- audio_coords$unsqueeze(2L) # [batch_size, 1, num_patches, 2] - - audio_coords - }, - - prepare_coords = function(...) { - if (self$modality == "video") { - self$prepare_video_coords(...) - } else { - self$prepare_audio_coords(...) - } - }, - - forward = function( - coords, - device = NULL, - dtype = NULL - ) { - if (is.null(device)) device <- coords$device - # Store target dtype for conversion at end (RoPE computed in float32 for precision) - - num_pos_dims <- coords$shape[2] - - # If coords are patch boundaries, use midpoint - if (coords$ndim == 4L) { - coords_chunks <- coords$chunk(2L, dim = - 1L) - coords_start <- coords_chunks[[1]] - coords_end <- coords_chunks[[2]] - coords <- (coords_start + coords_end) / 2.0 - coords <- coords$squeeze(- 1L) # [B, num_pos_dims, num_patches] - } - - # Get coordinates as fraction of base data shape - if (self$modality == "video") { - max_positions <- c(self$base_num_frames, self$base_height, self$base_width) - } else { - max_positions <- c(self$base_num_frames) - } - - # [B, num_pos_dims, num_patches] -> [B, num_patches, num_pos_dims] - grid_parts <- lapply(seq_len(num_pos_dims), function(i) { - coords[, i,] / max_positions[i] - }) - grid <- torch::torch_stack(grid_parts, dim = - 1L)$to(device = device) - - num_rope_elems <- num_pos_dims * 2L - - # Create frequency grid - if (self$double_precision) { - freqs_dtype <- torch::torch_float64() - } else { - freqs_dtype <- torch::torch_float32() - } - pow_indices <- torch::torch_pow( - self$theta, - torch::torch_linspace(start = 0.0, end = 1.0, steps = self$dim %/% num_rope_elems, - dtype = freqs_dtype, device = device) - ) - freqs <- (pow_indices * pi / 2.0)$to(dtype = torch::torch_float32()) - - # Outer product - freqs <- (grid$unsqueeze(- 1L) * 2 - 1) * freqs# [B, num_patches, num_pos_dims, dim/num_elems] - freqs <- freqs$transpose(- 1L, - 2L)$flatten(start_dim = 3L) # [B, num_patches, dim/2] - - # Get cos/sin frequencies - if (self$rope_type == "interleaved") { - cos_freqs <- freqs$cos()$repeat_interleave(2L, dim = - 1L) - sin_freqs <- freqs$sin()$repeat_interleave(2L, dim = - 1L) - - if (self$dim %% num_rope_elems != 0L) { - pad_size <- self$dim %% num_rope_elems - cos_padding <- torch::torch_ones_like(cos_freqs[,, 1:pad_size]) - sin_padding <- torch::torch_zeros_like(sin_freqs[,, 1:pad_size]) - cos_freqs <- torch::torch_cat(list(cos_padding, cos_freqs), dim = - 1L) - sin_freqs <- torch::torch_cat(list(sin_padding, sin_freqs), dim = - 1L) - } - } else { - # split type - expected_freqs <- self$dim %/% 2L - current_freqs <- freqs$shape[length(freqs$shape)] - pad_size <- expected_freqs - current_freqs - - cos_freq <- freqs$cos() - sin_freq <- freqs$sin() - - if (pad_size != 0L) { - cos_padding <- torch::torch_ones_like(cos_freq[,, 1:pad_size]) - sin_padding <- torch::torch_zeros_like(sin_freq[,, 1:pad_size]) - cos_freq <- torch::torch_cat(list(cos_padding, cos_freq), dim = - 1L) - sin_freq <- torch::torch_cat(list(sin_padding, sin_freq), dim = - 1L) - } - - # Reshape for multi-head attention - b <- cos_freq$shape[1] - t <- cos_freq$shape[2] - - cos_freq <- cos_freq$reshape(c(b, t, self$num_attention_heads, - 1L)) - sin_freq <- sin_freq$reshape(c(b, t, self$num_attention_heads, - 1L)) - - cos_freqs <- cos_freq$transpose(2L, 3L) # (B, H, T, D//2) - sin_freqs <- sin_freq$transpose(2L, 3L) - } - - # Convert to target dtype if specified (for mixed precision) - if (!is.null(dtype)) { - cos_freqs <- cos_freqs$to(dtype = dtype) - sin_freqs <- sin_freqs$to(dtype = dtype) - } - - list(cos_freqs, sin_freqs) - } -) - -# ------------------------------------------------------------------------------ -# LTX2 Video Transformer 3D Model -# ------------------------------------------------------------------------------ - -#' LTX2 Video Transformer 3D Model (Audio-Video) -#' -#' Full audio-video transformer matching HuggingFace diffusers implementation. -#' -#' @param in_channels Integer. Video input channels (default: 128). -#' @param out_channels Integer. Video output channels (default: 128). -#' @param patch_size Integer. Spatial patch size (default: 1). -#' @param patch_size_t Integer. Temporal patch size (default: 1). -#' @param num_attention_heads Integer. Video attention heads (default: 32). -#' @param attention_head_dim Integer. Video attention head dimension (default: 128). -#' @param cross_attention_dim Integer. Video cross-attention dimension (default: 4096). -#' @param vae_scale_factors Integer vector. VAE scale factors (default: c(8, 32, 32)). -#' @param pos_embed_max_pos Integer. Max position for RoPE (default: 20). -#' @param base_height Integer. Base height for RoPE (default: 2048). -#' @param base_width Integer. Base width for RoPE (default: 2048). -#' @param audio_in_channels Integer. Audio input channels (default: 128). -#' @param audio_out_channels Integer. Audio output channels (default: 128). -#' @param audio_patch_size Integer. Audio patch size (default: 1). -#' @param audio_patch_size_t Integer. Audio temporal patch size (default: 1). -#' @param audio_num_attention_heads Integer. Audio attention heads (default: 32). -#' @param audio_attention_head_dim Integer. Audio head dimension (default: 64). -#' @param audio_cross_attention_dim Integer. Audio cross-attention dim (default: 2048). -#' @param audio_scale_factor Integer. Audio scale factor (default: 4). -#' @param audio_pos_embed_max_pos Integer. Audio max position (default: 20). -#' @param audio_sampling_rate Integer. Audio sampling rate (default: 16000). -#' @param audio_hop_length Integer. Audio hop length (default: 160). -#' @param num_layers Integer. Number of transformer layers (default: 48). -#' @param activation_fn Character. Activation function (default: "gelu-approximate"). -#' @param qk_norm Character. QK normalization type (default: "rms_norm_across_heads"). -#' @param norm_elementwise_affine Logical. Use elementwise affine in norms (default: FALSE). -#' @param norm_eps Numeric. Epsilon for normalization (default: 1e-6). -#' @param caption_channels Integer. Caption embedding channels (default: 3840). -#' @param attention_bias Logical. Use bias in attention (default: TRUE). -#' @param attention_out_bias Logical. Use bias in attention output (default: TRUE). -#' @param rope_theta Numeric. Theta for RoPE (default: 10000). -#' @param rope_double_precision Logical. Use double precision for RoPE (default: TRUE). -#' @param causal_offset Integer. Causal offset for RoPE (default: 1). -#' @param timestep_scale_multiplier Numeric. Timestep scale (default: 1000). -#' @param cross_attn_timestep_scale_multiplier Numeric. Cross-attn timestep scale (default: 1000). -#' @param rope_type Character. RoPE type: "interleaved" or "split" (default: "interleaved"). -#' -#' @return An nn_module representing the LTX2 video transformer. -#' @export -ltx2_video_transformer_3d_model <- torch::nn_module( - "LTX2VideoTransformer3DModel", - initialize = function( - in_channels = 128L, - out_channels = 128L, - patch_size = 1L, - patch_size_t = 1L, - num_attention_heads = 32L, - attention_head_dim = 128L, - cross_attention_dim = 4096L, - vae_scale_factors = c(8L, 32L, 32L), - pos_embed_max_pos = 20L, - base_height = 2048L, - base_width = 2048L, - audio_in_channels = 128L, - audio_out_channels = 128L, - audio_patch_size = 1L, - audio_patch_size_t = 1L, - audio_num_attention_heads = 32L, - audio_attention_head_dim = 64L, - audio_cross_attention_dim = 2048L, - audio_scale_factor = 4L, - audio_pos_embed_max_pos = 20L, - audio_sampling_rate = 16000L, - audio_hop_length = 160L, - num_layers = 48L, - activation_fn = "gelu-approximate", - qk_norm = "rms_norm_across_heads", - norm_elementwise_affine = FALSE, - norm_eps = 1e-6, - caption_channels = 3840L, - attention_bias = TRUE, - attention_out_bias = TRUE, - rope_theta = 10000.0, - rope_double_precision = TRUE, - causal_offset = 1L, - timestep_scale_multiplier = 1000L, - cross_attn_timestep_scale_multiplier = 1000L, - rope_type = "interleaved" - ) { - - if (is.null(out_channels)) out_channels <- in_channels - if (is.null(audio_out_channels)) audio_out_channels <- audio_in_channels - - self$timestep_scale_multiplier <- timestep_scale_multiplier - self$cross_attn_timestep_scale_multiplier <- cross_attn_timestep_scale_multiplier - - inner_dim <- num_attention_heads * attention_head_dim - audio_inner_dim <- audio_num_attention_heads * audio_attention_head_dim - - # 1. Patchification input projections - self$proj_in <- make_linear(in_channels, inner_dim) - self$audio_proj_in <- make_linear(audio_in_channels, audio_inner_dim) - - # 2. Prompt embeddings - self$caption_projection <- pixart_alpha_text_projection( - in_features = caption_channels, hidden_size = inner_dim - ) - self$audio_caption_projection <- pixart_alpha_text_projection( - in_features = caption_channels, hidden_size = audio_inner_dim - ) - - # 3. Timestep modulation - # 3.1 Global timestep embedding and modulation - self$time_embed <- ltx2_ada_layer_norm_single(inner_dim, num_mod_params = 6L, use_additional_conditions = FALSE) - self$audio_time_embed <- ltx2_ada_layer_norm_single(audio_inner_dim, num_mod_params = 6L, use_additional_conditions = FALSE) - - # 3.2 Global cross-attention modulation - self$av_cross_attn_video_scale_shift <- ltx2_ada_layer_norm_single(inner_dim, num_mod_params = 4L, use_additional_conditions = FALSE) - self$av_cross_attn_audio_scale_shift <- ltx2_ada_layer_norm_single(audio_inner_dim, num_mod_params = 4L, use_additional_conditions = FALSE) - self$av_cross_attn_video_a2v_gate <- ltx2_ada_layer_norm_single(inner_dim, num_mod_params = 1L, use_additional_conditions = FALSE) - self$av_cross_attn_audio_v2a_gate <- ltx2_ada_layer_norm_single(audio_inner_dim, num_mod_params = 1L, use_additional_conditions = FALSE) - - # 3.3 Output layer modulation - self$scale_shift_table <- torch::nn_parameter(torch::torch_randn(2L, inner_dim) / sqrt(inner_dim)) - self$audio_scale_shift_table <- torch::nn_parameter(torch::torch_randn(2L, audio_inner_dim) / sqrt(audio_inner_dim)) - - # 4. Rotary Positional Embeddings - # Video self-attention RoPE - self$rope <- ltx2_audio_video_rotary_pos_embed( - dim = inner_dim, - patch_size = patch_size, - patch_size_t = patch_size_t, - base_num_frames = pos_embed_max_pos, - base_height = base_height, - base_width = base_width, - scale_factors = vae_scale_factors, - theta = rope_theta, - causal_offset = causal_offset, - modality = "video", - double_precision = rope_double_precision, - rope_type = rope_type, - num_attention_heads = num_attention_heads - ) - - # Audio self-attention RoPE - self$audio_rope <- ltx2_audio_video_rotary_pos_embed( - dim = audio_inner_dim, - patch_size = audio_patch_size, - patch_size_t = audio_patch_size_t, - base_num_frames = audio_pos_embed_max_pos, - sampling_rate = audio_sampling_rate, - hop_length = audio_hop_length, - scale_factors = c(audio_scale_factor), - theta = rope_theta, - causal_offset = causal_offset, - modality = "audio", - double_precision = rope_double_precision, - rope_type = rope_type, - num_attention_heads = audio_num_attention_heads - ) - - # Cross-attention RoPE - cross_attn_pos_embed_max_pos <- max(pos_embed_max_pos, audio_pos_embed_max_pos) - - self$cross_attn_rope <- ltx2_audio_video_rotary_pos_embed( - dim = audio_cross_attention_dim, - patch_size = patch_size, - patch_size_t = patch_size_t, - base_num_frames = cross_attn_pos_embed_max_pos, - base_height = base_height, - base_width = base_width, - theta = rope_theta, - causal_offset = causal_offset, - modality = "video", - double_precision = rope_double_precision, - rope_type = rope_type, - num_attention_heads = num_attention_heads - ) - - self$cross_attn_audio_rope <- ltx2_audio_video_rotary_pos_embed( - dim = audio_cross_attention_dim, - patch_size = audio_patch_size, - patch_size_t = audio_patch_size_t, - base_num_frames = cross_attn_pos_embed_max_pos, - sampling_rate = audio_sampling_rate, - hop_length = audio_hop_length, - theta = rope_theta, - causal_offset = causal_offset, - modality = "audio", - double_precision = rope_double_precision, - rope_type = rope_type, - num_attention_heads = audio_num_attention_heads - ) - - # 5. Transformer blocks - self$transformer_blocks <- torch::nn_module_list(lapply(seq_len(num_layers), function(i) { - ltx2_video_transformer_block( - dim = inner_dim, - num_attention_heads = num_attention_heads, - attention_head_dim = attention_head_dim, - cross_attention_dim = cross_attention_dim, - audio_dim = audio_inner_dim, - audio_num_attention_heads = audio_num_attention_heads, - audio_attention_head_dim = audio_attention_head_dim, - audio_cross_attention_dim = audio_cross_attention_dim, - qk_norm = qk_norm, - activation_fn = activation_fn, - attention_bias = attention_bias, - attention_out_bias = attention_out_bias, - eps = norm_eps, - elementwise_affine = norm_elementwise_affine, - rope_type = rope_type - ) - })) - - # 6. Output layers - self$norm_out <- torch::nn_layer_norm(inner_dim, eps = 1e-6, elementwise_affine = FALSE) - self$proj_out <- make_linear(inner_dim, out_channels) - - self$audio_norm_out <- torch::nn_layer_norm(audio_inner_dim, eps = 1e-6, elementwise_affine = FALSE) - self$audio_proj_out <- make_linear(audio_inner_dim, audio_out_channels) - - self$gradient_checkpointing <- FALSE - }, - - forward = function( - hidden_states, - audio_hidden_states, - encoder_hidden_states, - audio_encoder_hidden_states, - timestep, - audio_timestep = NULL, - encoder_attention_mask = NULL, - audio_encoder_attention_mask = NULL, - num_frames = NULL, - height = NULL, - width = NULL, - fps = 24.0, - audio_num_frames = NULL, - video_coords = NULL, - audio_coords = NULL - ) { - - if (is.null(audio_timestep)) audio_timestep <- timestep - - # Convert attention masks to bias (use tensor ops to preserve dtype) - # Formula: (1 - mask) * -10000.0 -> (mask - 1) * 10000.0 - if (!is.null(encoder_attention_mask) && encoder_attention_mask$ndim == 2L) { - mask_dtype <- hidden_states$dtype - encoder_attention_mask <- encoder_attention_mask$to(dtype = mask_dtype) - # Use tensor scalar to preserve dtype - scale <- torch::torch_tensor(- 10000.0, dtype = mask_dtype, device = encoder_attention_mask$device) - encoder_attention_mask <- encoder_attention_mask$sub(1)$neg()$mul(scale) - encoder_attention_mask <- encoder_attention_mask$unsqueeze(2L) - } - - if (!is.null(audio_encoder_attention_mask) && audio_encoder_attention_mask$ndim == 2L) { - mask_dtype <- audio_hidden_states$dtype - audio_encoder_attention_mask <- audio_encoder_attention_mask$to(dtype = mask_dtype) - scale <- torch::torch_tensor(- 10000.0, dtype = mask_dtype, device = audio_encoder_attention_mask$device) - audio_encoder_attention_mask <- audio_encoder_attention_mask$sub(1)$neg()$mul(scale) - audio_encoder_attention_mask <- audio_encoder_attention_mask$unsqueeze(2L) - } - - batch_size <- hidden_states$shape[1] - - # 1. Prepare RoPE positional embeddings - if (is.null(video_coords)) { - video_coords <- self$rope$prepare_video_coords(batch_size, num_frames, height, width, hidden_states$device, fps = fps) - } - if (is.null(audio_coords)) { - audio_coords <- self$audio_rope$prepare_audio_coords(batch_size, audio_num_frames, audio_hidden_states$device) - } - - video_rotary_emb <- self$rope(video_coords, device = hidden_states$device, dtype = hidden_states$dtype) - audio_rotary_emb <- self$audio_rope(audio_coords, device = audio_hidden_states$device, dtype = audio_hidden_states$dtype) - - video_cross_attn_rotary_emb <- self$cross_attn_rope(video_coords[, 1:1,,], device = hidden_states$device, dtype = hidden_states$dtype) - audio_cross_attn_rotary_emb <- self$cross_attn_audio_rope(audio_coords[, 1:1,,], device = audio_hidden_states$device, dtype = audio_hidden_states$dtype) - - # 2. Patchify input projections - hidden_states <- self$proj_in(hidden_states) - audio_hidden_states <- self$audio_proj_in(audio_hidden_states) - - # 3. Prepare timestep embeddings - timestep_cross_attn_gate_scale_factor <- self$cross_attn_timestep_scale_multiplier / self$timestep_scale_multiplier - - # 3.1 Global timestep embedding - temb_result <- self$time_embed(timestep$flatten(), batch_size = batch_size, hidden_dtype = hidden_states$dtype) - temb <- temb_result[[1]]$view(c(batch_size, - 1L, temb_result[[1]]$shape[length(temb_result[[1]]$shape)])) - embedded_timestep <- temb_result[[2]]$view(c(batch_size, - 1L, temb_result[[2]]$shape[length(temb_result[[2]]$shape)])) - - temb_audio_result <- self$audio_time_embed(audio_timestep$flatten(), batch_size = batch_size, hidden_dtype = audio_hidden_states$dtype) - temb_audio <- temb_audio_result[[1]]$view(c(batch_size, - 1L, temb_audio_result[[1]]$shape[length(temb_audio_result[[1]]$shape)])) - audio_embedded_timestep <- temb_audio_result[[2]]$view(c(batch_size, - 1L, temb_audio_result[[2]]$shape[length(temb_audio_result[[2]]$shape)])) - - # 3.2 Cross-attention modulation - video_cross_attn_scale_shift_result <- self$av_cross_attn_video_scale_shift(timestep$flatten(), batch_size = batch_size, hidden_dtype = hidden_states$dtype) - video_cross_attn_a2v_gate_result <- self$av_cross_attn_video_a2v_gate(timestep$flatten() * timestep_cross_attn_gate_scale_factor, batch_size = batch_size, hidden_dtype = hidden_states$dtype) - - video_cross_attn_scale_shift <- video_cross_attn_scale_shift_result[[1]]$view(c(batch_size, - 1L, video_cross_attn_scale_shift_result[[1]]$shape[length(video_cross_attn_scale_shift_result[[1]]$shape)])) - video_cross_attn_a2v_gate <- video_cross_attn_a2v_gate_result[[1]]$view(c(batch_size, - 1L, video_cross_attn_a2v_gate_result[[1]]$shape[length(video_cross_attn_a2v_gate_result[[1]]$shape)])) - - audio_cross_attn_scale_shift_result <- self$av_cross_attn_audio_scale_shift(audio_timestep$flatten(), batch_size = batch_size, hidden_dtype = audio_hidden_states$dtype) - audio_cross_attn_v2a_gate_result <- self$av_cross_attn_audio_v2a_gate(audio_timestep$flatten() * timestep_cross_attn_gate_scale_factor, batch_size = batch_size, hidden_dtype = audio_hidden_states$dtype) - - audio_cross_attn_scale_shift <- audio_cross_attn_scale_shift_result[[1]]$view(c(batch_size, - 1L, audio_cross_attn_scale_shift_result[[1]]$shape[length(audio_cross_attn_scale_shift_result[[1]]$shape)])) - audio_cross_attn_v2a_gate <- audio_cross_attn_v2a_gate_result[[1]]$view(c(batch_size, - 1L, audio_cross_attn_v2a_gate_result[[1]]$shape[length(audio_cross_attn_v2a_gate_result[[1]]$shape)])) - - # 4. Prepare prompt embeddings - encoder_hidden_states <- self$caption_projection(encoder_hidden_states) - encoder_hidden_states <- encoder_hidden_states$view(c(batch_size, - 1L, hidden_states$shape[length(hidden_states$shape)])) - - audio_encoder_hidden_states <- self$audio_caption_projection(audio_encoder_hidden_states) - audio_encoder_hidden_states <- audio_encoder_hidden_states$view(c(batch_size, - 1L, audio_hidden_states$shape[length(audio_hidden_states$shape)])) - - # 5. Run transformer blocks - for (i in seq_along(self$transformer_blocks)) { - block <- self$transformer_blocks[[i]] - result <- block( - hidden_states = hidden_states, - audio_hidden_states = audio_hidden_states, - encoder_hidden_states = encoder_hidden_states, - audio_encoder_hidden_states = audio_encoder_hidden_states, - temb = temb, - temb_audio = temb_audio, - temb_ca_scale_shift = video_cross_attn_scale_shift, - temb_ca_audio_scale_shift = audio_cross_attn_scale_shift, - temb_ca_gate = video_cross_attn_a2v_gate, - temb_ca_audio_gate = audio_cross_attn_v2a_gate, - video_rotary_emb = video_rotary_emb, - audio_rotary_emb = audio_rotary_emb, - ca_video_rotary_emb = video_cross_attn_rotary_emb, - ca_audio_rotary_emb = audio_cross_attn_rotary_emb, - encoder_attention_mask = encoder_attention_mask, - audio_encoder_attention_mask = audio_encoder_attention_mask - ) - hidden_states <- result[[1]] - audio_hidden_states <- result[[2]] - } - - # 6. Output layers - scale_shift_values <- self$scale_shift_table$unsqueeze(1)$unsqueeze(1)$to(dtype = hidden_states$dtype) + embedded_timestep$unsqueeze(3) - shift <- scale_shift_values[,, 1,] - scale <- scale_shift_values[,, 2,] - - hidden_states <- self$norm_out(hidden_states) - hidden_states <- hidden_states * scale$add(1) + shift - output <- self$proj_out(hidden_states) - - audio_scale_shift_values <- self$audio_scale_shift_table$unsqueeze(1)$unsqueeze(1)$to(dtype = audio_hidden_states$dtype) + audio_embedded_timestep$unsqueeze(3) - audio_shift <- audio_scale_shift_values[,, 1,] - audio_scale <- audio_scale_shift_values[,, 2,] - - audio_hidden_states <- self$audio_norm_out(audio_hidden_states) - audio_hidden_states <- audio_hidden_states * audio_scale$add(1) + audio_shift - audio_output <- self$audio_proj_out(audio_hidden_states) - - list(sample = output, audio_sample = audio_output) - } -) - -# ------------------------------------------------------------------------------ -# Weight Loading Functions -# ------------------------------------------------------------------------------ - -#' Load LTX2 DiT Transformer from safetensors -#' -#' Load pre-trained LTX2 transformer weights from HuggingFace safetensors files. -#' Supports both single file and sharded multi-file loading. -#' -#' @param weights_dir Character. Directory containing safetensors files. -#' @param config_path Character. Optional path to config.json. If NULL, uses default config. -#' @param device Character. Device to load weights to. Default: "cpu" -#' @param dtype Character. Data type ("float32", "float16", "bfloat16"). Default: "float16" -#' @param verbose Logical. Print loading progress. Default: TRUE -#' @return Initialized ltx2_video_transformer3d module -#' @export -load_ltx2_transformer <- function( - weights_dir, - config_path = NULL, - device = "cpu", - dtype = "float16", - verbose = TRUE -) { - - # Look for config - if (is.null(config_path)) { - config_path <- file.path(weights_dir, "config.json") - } - - config <- NULL - if (file.exists(config_path)) { - config <- jsonlite::fromJSON(config_path) - if (verbose) message("Loaded config from: ", config_path) - } - - # Create transformer with config or defaults - if (!is.null(config)) { - transformer <- ltx2_video_transformer_3d_model( - in_channels = config$in_channels %||% 128L, - out_channels = config$out_channels %||% 128L, - patch_size = config$patch_size %||% 1L, - patch_size_t = config$patch_size_t %||% 1L, - num_attention_heads = config$num_attention_heads %||% 32L, - attention_head_dim = config$attention_head_dim %||% 128L, - cross_attention_dim = config$cross_attention_dim %||% 4096L, - num_layers = config$num_layers %||% 48L, - caption_channels = config$caption_channels %||% 3840L, - qk_norm = config$qk_norm %||% "rms_norm_across_heads", - norm_elementwise_affine = config$norm_elementwise_affine %||% FALSE, - norm_eps = config$norm_eps %||% 1e-6, - timestep_scale_multiplier = config$timestep_scale_multiplier %||% 1000.0, - cross_attn_timestep_scale_multiplier = config$cross_attn_timestep_scale_multiplier %||% 1000.0, - base_height = config$base_height %||% 2048L, - base_width = config$base_width %||% 2048L, - pos_embed_max_pos = config$pos_embed_max_pos %||% 20L, - rope_theta = config$rope_theta %||% 10000.0, - rope_type = config$rope_type %||% "split", - causal_offset = config$causal_offset %||% 1L, - audio_in_channels = config$audio_in_channels %||% 128L, - audio_out_channels = config$audio_out_channels %||% 128L, - audio_num_attention_heads = config$audio_num_attention_heads %||% 32L, - audio_attention_head_dim = config$audio_attention_head_dim %||% 64L, - audio_cross_attention_dim = config$audio_cross_attention_dim %||% 2048L, - audio_sampling_rate = config$audio_sampling_rate %||% 16000L, - audio_hop_length = config$audio_hop_length %||% 160L, - audio_scale_factor = config$audio_scale_factor %||% 4L, - audio_pos_embed_max_pos = config$audio_pos_embed_max_pos %||% 20L - ) - } else { - transformer <- ltx2_video_transformer_3d_model() - } - - # Find weight files - index_path <- file.path(weights_dir, "diffusion_pytorch_model.safetensors.index.json") - single_path <- file.path(weights_dir, "diffusion_pytorch_model.safetensors") - - if (file.exists(index_path)) { - # Sharded weights - if (verbose) message("Loading sharded weights from: ", weights_dir) - load_ltx2_transformer_sharded(transformer, weights_dir, index_path, verbose = verbose) - } else if (file.exists(single_path)) { - # Single file - if (verbose) message("Loading weights from: ", single_path) - weights <- safetensors::safe_load_file(single_path, framework = "torch") - load_ltx2_transformer_weights(transformer, weights, verbose = verbose) - } else { - stop("No weights found in: ", weights_dir) - } - - # Convert dtype and move to device - torch_dtype <- switch(dtype, - "float32" = torch::torch_float32(), - "float16" = torch::torch_float16(), - "bfloat16" = torch::torch_bfloat16(), - torch::torch_float16() - ) - - transformer$to(device = device, dtype = torch_dtype) - - if (verbose) message("Transformer loaded successfully on device: ", device, " dtype: ", dtype) - transformer -} - -#' Load sharded transformer weights -#' -#' @param transformer LTX2 transformer module -#' @param weights_dir Directory containing sharded safetensors -#' @param index_path Path to index.json -#' @param verbose Print progress -#' @keywords internal -load_ltx2_transformer_sharded <- function( - transformer, - weights_dir, - index_path, - verbose = TRUE -) { - # Load index - index <- jsonlite::fromJSON(index_path) - weight_map <- index$weight_map - - # Get unique shard files - shard_files <- unique(unlist(weight_map)) - if (verbose) message(sprintf("Loading %d shards...", length(shard_files))) - - total_loaded <- 0L - total_skipped <- 0L - - for (i in seq_along(shard_files)) { - shard_file <- shard_files[i] - shard_path <- file.path(weights_dir, shard_file) - - if (!file.exists(shard_path)) { - warning("Shard not found: ", shard_path) - next - } - - if (verbose) message(sprintf("[%d/%d] Loading %s...", i, length(shard_files), shard_file)) - - weights <- safetensors::safe_load_file(shard_path, framework = "torch") - result <- load_ltx2_transformer_weights(transformer, weights, verbose = FALSE) - total_loaded <- total_loaded + result$loaded - total_skipped <- total_skipped + result$skipped - - # Free memory - rm(weights) - gc() - } - - if (verbose) { - message(sprintf("Total: %d loaded, %d skipped", total_loaded, total_skipped)) - } - - invisible(list(loaded = total_loaded, skipped = total_skipped)) -} - -#' Load weights into LTX2 transformer module -#' -#' @param transformer LTX2 transformer module -#' @param weights Named list of weight tensors -#' @param verbose Print progress -#' @keywords internal -load_ltx2_transformer_weights <- function( - transformer, - weights, - verbose = TRUE -) { - native_params <- names(transformer$parameters) - - # Remap HuggingFace names to R module names - remap_transformer_key <- function(key) { - # HuggingFace uses nn.ModuleList for FeedForward: - # ff.net.0.proj.weight -> ff.act_fn.proj.weight - # ff.net.2.weight -> ff.proj_out.weight - key <- gsub("\\.ff\\.net\\.0\\.", ".ff.act_fn.", key) - key <- gsub("\\.ff\\.net\\.2\\.", ".ff.proj_out.", key) - # Same for audio_ff - key <- gsub("\\.audio_ff\\.net\\.0\\.", ".audio_ff.act_fn.", key) - key <- gsub("\\.audio_ff\\.net\\.2\\.", ".audio_ff.proj_out.", key) - - # to_out.0 is used in both HF and our module (ModuleList) - # No remapping needed for to_out - key - } - - loaded <- 0L - skipped <- 0L - unmapped <- character(0) - - torch::with_no_grad({ - for (hf_name in names(weights)) { - native_name <- remap_transformer_key(hf_name) - - if (native_name %in% native_params) { - hf_tensor <- weights[[hf_name]] - native_tensor <- transformer$parameters[[native_name]] - - if (all(as.integer(hf_tensor$shape) == as.integer(native_tensor$shape))) { - native_tensor$copy_(hf_tensor) - loaded <- loaded + 1L - } else { - if (verbose) { - message("Shape mismatch: ", native_name, - " (HF: ", paste(as.integer(hf_tensor$shape), collapse = "x"), - " vs R: ", paste(as.integer(native_tensor$shape), collapse = "x"), ")") - } - skipped <- skipped + 1L - } - } else { - skipped <- skipped + 1L - unmapped <- c(unmapped, paste0(hf_name, " -> ", native_name)) - } - } - }) - - if (verbose) { - message(sprintf("Transformer weights: %d loaded, %d skipped", loaded, skipped)) - if (length(unmapped) > 0 && length(unmapped) <= 20) { - message("Unmapped parameters:") - for (u in unmapped[1:min(20, length(unmapped))]) { - message(" ", u) - } - } - if (length(unmapped) > 20) { - message(" ... and ", length(unmapped) - 20, " more") - } - } - - invisible(list(loaded = loaded, skipped = skipped, unmapped = unmapped)) -} - -# ------------------------------------------------------------------------------ -# Latent Packing/Unpacking for DiT -# ------------------------------------------------------------------------------ - -#' Pack Video Latents for DiT -#' -#' Transforms 5D video latents \[B, C, F, H, W\] to 3D sequence \[B, F*H*W, C\] -#' for input to the DiT transformer. -#' -#' @param latents Tensor of shape \[B, C, F, H, W\]. -#' @param patch_size Integer. Spatial patch size (default 1). -#' @param patch_size_t Integer. Temporal patch size (default 1). -#' -#' @return Packed tensor of shape \[B, num_patches, C*patch_size_t*patch_size^2\]. -#' -#' @export -pack_video_latents <- function( - latents, - patch_size = 1L, - patch_size_t = 1L -) { - batch_size <- latents$shape[1] - num_channels <- latents$shape[2] - num_frames <- latents$shape[3] - height <- latents$shape[4] - width <- latents$shape[5] - - post_patch_frames <- num_frames %/% patch_size_t - post_patch_height <- height %/% patch_size - post_patch_width <- width %/% patch_size - - # Reshape: [B, C, F, H, W] -> [B, C, F//pt, pt, H//p, p, W//p, p] - latents <- latents$reshape(c( - batch_size, num_channels, - post_patch_frames, patch_size_t, - post_patch_height, patch_size, - post_patch_width, patch_size - )) - - # Permute: [B, C, F//pt, pt, H//p, p, W//p, p] -> [B, F//pt, H//p, W//p, C, pt, p, p] - latents <- latents$permute(c(1L, 3L, 5L, 7L, 2L, 4L, 6L, 8L)) - - # Flatten patches: [B, F//pt, H//p, W//p, C*pt*p*p] - latents <- latents$flatten(start_dim = 5L, end_dim = 8L) - - # Flatten sequence: [B, F//pt * H//p * W//p, C*pt*p*p] - latents <- latents$flatten(start_dim = 2L, end_dim = 4L) - - latents -} - -#' Unpack Video Latents from DiT -#' -#' Transforms 3D sequence \[B, num_patches, D\] back to 5D video latents \[B, C, F, H, W\]. -#' -#' @param latents Tensor of shape \[B, num_patches, D\]. -#' @param num_frames Integer. Target number of latent frames. -#' @param height Integer. Target latent height. -#' @param width Integer. Target latent width. -#' @param patch_size Integer. Spatial patch size (default 1). -#' @param patch_size_t Integer. Temporal patch size (default 1). -#' -#' @return Unpacked tensor of shape \[B, C, F, H, W\]. -#' -#' @export -unpack_video_latents <- function( - latents, - num_frames, - height, - width, - patch_size = 1L, - patch_size_t = 1L -) { - batch_size <- latents$shape[1] - - post_patch_frames <- num_frames %/% patch_size_t - post_patch_height <- height %/% patch_size - post_patch_width <- width %/% patch_size - - # Unflatten sequence: [B, S, D] -> [B, F//pt, H//p, W//p, D] - latents <- latents$reshape(c(batch_size, post_patch_frames, post_patch_height, post_patch_width, - 1L)) - - # Calculate channel dimension - d <- latents$shape[5] - num_channels <- d %/% (patch_size_t * patch_size * patch_size) - - # Unflatten patches: [B, F//pt, H//p, W//p, C, pt, p, p] - latents <- latents$reshape(c( - batch_size, post_patch_frames, post_patch_height, post_patch_width, - num_channels, patch_size_t, patch_size, patch_size - )) - - # Permute back: [B, C, F//pt, pt, H//p, p, W//p, p] - latents <- latents$permute(c(1L, 5L, 2L, 6L, 3L, 7L, 4L, 8L)) - - # Flatten to video: [B, C, F, H, W] - latents <- latents$flatten(start_dim = 7L, end_dim = 8L) - latents <- latents$flatten(start_dim = 5L, end_dim = 6L) - latents <- latents$flatten(start_dim = 3L, end_dim = 4L) - - latents -} - diff --git a/R/dit_ltx2_modules.R b/R/dit_ltx2_modules.R deleted file mode 100644 index 793da2f..0000000 --- a/R/dit_ltx2_modules.R +++ /dev/null @@ -1,802 +0,0 @@ -# LTX2 DiT Transformer Modules -# Audio-Video transformer matching HuggingFace diffusers implementation - -# ------------------------------------------------------------------------------ -# Helper modules: Timestep embeddings -# ------------------------------------------------------------------------------ - -#' Get timestep embedding (sinusoidal) -#' @keywords internal -get_timestep_embedding <- function( - timesteps, - embedding_dim, - flip_sin_to_cos = FALSE, - downscale_freq_shift = 1, - scale = 1, - max_period = 10000 -) { - # Ensure timesteps is 1D - if (timesteps$ndim > 1L) { - timesteps <- timesteps$flatten() - } - - # Store original dtype to convert back at end - orig_dtype <- timesteps$dtype - - half_dim <- embedding_dim %/% 2L - - # Compute in float32 for numerical precision - exponent <- - log(max_period) * torch::torch_arange( - start = 0, end = half_dim - 1L, dtype = torch::torch_float32(), device = timesteps$device - ) - exponent <- exponent / (half_dim - downscale_freq_shift) - - emb <- torch::torch_exp(exponent) - # timesteps: [N], emb: [half_dim] -> result: [N, half_dim] - emb <- timesteps$unsqueeze(- 1L)$to(dtype = torch::torch_float32()) * emb$unsqueeze(1L) - emb <- scale * emb - - # Concat sine and cosine -> [N, embedding_dim] - emb <- torch::torch_cat(list(torch::torch_sin(emb), torch::torch_cos(emb)), dim = - 1L) - - # Flip if needed (2D tensor: [N, dim]) - if (flip_sin_to_cos) { - emb <- torch::torch_cat(list( - emb[, (half_dim + 1L) :embedding_dim], - emb[, 1L:half_dim] - ), dim = - 1L) - } - - # Zero pad if odd - if (embedding_dim %% 2L == 1L) { - emb <- torch::nnf_pad(emb, c(0L, 1L, 0L, 0L)) - } - - # Convert back to original dtype (for mixed precision training) - emb <- emb$to(dtype = orig_dtype) - - emb -} - -#' Timesteps module -#' @keywords internal -timesteps_module <- torch::nn_module( - "Timesteps", - initialize = function( - num_channels, - flip_sin_to_cos = TRUE, - downscale_freq_shift = 0, - scale = 1L - ) { - self$num_channels <- num_channels - self$flip_sin_to_cos <- flip_sin_to_cos - self$downscale_freq_shift <- downscale_freq_shift - self$scale <- scale - }, - forward = function(timesteps) { - get_timestep_embedding( - timesteps, - self$num_channels, - flip_sin_to_cos = self$flip_sin_to_cos, - downscale_freq_shift = self$downscale_freq_shift, - scale = self$scale - ) - } -) - -#' Timestep embedding MLP -#' @keywords internal -timestep_embedding_module <- torch::nn_module( - "TimestepEmbedding", - initialize = function( - in_channels, - time_embed_dim, - act_fn = "silu", - out_dim = NULL - ) { - self$linear_1 <- make_linear(in_channels, time_embed_dim) - - if (act_fn == "silu") { - self$act <- torch::nn_silu() - } else if (act_fn == "gelu") { - self$act <- torch::nn_gelu() - } else { - self$act <- torch::nn_silu() - } - - if (!is.null(out_dim)) { - time_embed_dim_out <- out_dim - } else { - time_embed_dim_out <- time_embed_dim - } - self$linear_2 <- make_linear(time_embed_dim, time_embed_dim_out) - }, - forward = function(sample) { - sample <- self$linear_1(sample) - sample <- self$act(sample) - sample <- self$linear_2(sample) - sample - } -) - -#' PixArt Alpha Combined Timestep Size Embeddings -#' @keywords internal -pixart_alpha_combined_timestep_size_embeddings <- torch::nn_module( - "PixArtAlphaCombinedTimestepSizeEmbeddings", - initialize = function( - embedding_dim, - size_emb_dim, - use_additional_conditions = FALSE - ) { - self$outdim <- size_emb_dim - self$time_proj <- timesteps_module(num_channels = 256L, flip_sin_to_cos = TRUE, downscale_freq_shift = 0) - self$timestep_embedder <- timestep_embedding_module(in_channels = 256L, time_embed_dim = embedding_dim) - - self$use_additional_conditions <- use_additional_conditions - if (use_additional_conditions) { - self$additional_condition_proj <- timesteps_module(num_channels = 256L, flip_sin_to_cos = TRUE, downscale_freq_shift = 0) - self$resolution_embedder <- timestep_embedding_module(in_channels = 256L, time_embed_dim = size_emb_dim) - self$aspect_ratio_embedder <- timestep_embedding_module(in_channels = 256L, time_embed_dim = size_emb_dim) - } - }, - forward = function( - timestep, - resolution = NULL, - aspect_ratio = NULL, - batch_size = NULL, - hidden_dtype = NULL - ) { - timesteps_proj <- self$time_proj(timestep) - if (!is.null(hidden_dtype)) { - timesteps_proj <- timesteps_proj$to(dtype = hidden_dtype) - } - timesteps_emb <- self$timestep_embedder(timesteps_proj) - - if (self$use_additional_conditions && !is.null(resolution)) { - resolution_emb <- self$additional_condition_proj(resolution$flatten()) - if (!is.null(hidden_dtype)) { - resolution_emb <- resolution_emb$to(dtype = hidden_dtype) - } - resolution_emb <- self$resolution_embedder(resolution_emb)$reshape(c(batch_size, - 1L)) - - aspect_ratio_emb <- self$additional_condition_proj(aspect_ratio$flatten()) - if (!is.null(hidden_dtype)) { - aspect_ratio_emb <- aspect_ratio_emb$to(dtype = hidden_dtype) - } - aspect_ratio_emb <- self$aspect_ratio_embedder(aspect_ratio_emb)$reshape(c(batch_size, - 1L)) - - conditioning <- timesteps_emb + torch::torch_cat(list(resolution_emb, aspect_ratio_emb), dim = 2L) - } else { - conditioning <- timesteps_emb - } - - conditioning - } -) - -#' PixArt Alpha Text Projection -#' @keywords internal -pixart_alpha_text_projection <- torch::nn_module( - "PixArtAlphaTextProjection", - initialize = function( - in_features, - hidden_size, - out_features = NULL, - act_fn = "gelu_tanh" - ) { - if (is.null(out_features)) out_features <- hidden_size - - self$linear_1 <- make_linear(in_features, hidden_size) - - if (act_fn == "gelu_tanh") { - self$act_1 <- torch::nn_gelu(approximate = "tanh") - } else if (act_fn == "silu") { - self$act_1 <- torch::nn_silu() - } else { - self$act_1 <- torch::nn_gelu(approximate = "tanh") - } - - self$linear_2 <- make_linear(hidden_size, out_features) - }, - forward = function(caption) { - hidden_states <- self$linear_1(caption) - hidden_states <- self$act_1(hidden_states) - hidden_states <- self$linear_2(hidden_states) - hidden_states - } -) - -# ------------------------------------------------------------------------------ -# FeedForward module -# ------------------------------------------------------------------------------ - -#' GELU activation with optional approximation -#' @keywords internal -gelu_activation <- torch::nn_module( - "GELU", - initialize = function( - dim_in, - dim_out, - approximate = "none", - bias = TRUE - ) { - self$proj <- make_linear(dim_in, dim_out, bias = bias) - self$approximate <- approximate - }, - forward = function(hidden_states) { - hidden_states <- self$proj(hidden_states) - hidden_states <- torch::nnf_gelu(hidden_states, approximate = self$approximate) - hidden_states - } -) - -#' FeedForward module -#' @keywords internal -feed_forward <- torch::nn_module( - "FeedForward", - initialize = function( - dim, - dim_out = NULL, - mult = 4L, - dropout = 0.0, - activation_fn = "gelu-approximate", - inner_dim = NULL, - bias = TRUE - ) { - if (is.null(inner_dim)) inner_dim <- as.integer(dim * mult) - if (is.null(dim_out)) dim_out <- dim - - # Activation layer (projects in) - if (activation_fn == "gelu") { - self$act_fn <- gelu_activation(dim, inner_dim, approximate = "none", bias = bias) - } else if (activation_fn == "gelu-approximate") { - self$act_fn <- gelu_activation(dim, inner_dim, approximate = "tanh", bias = bias) - } else { - self$act_fn <- gelu_activation(dim, inner_dim, approximate = "tanh", bias = bias) - } - - self$dropout <- torch::nn_dropout(p = dropout) - self$proj_out <- make_linear(inner_dim, dim_out, bias = bias) - }, - forward = function(hidden_states) { - hidden_states <- self$act_fn(hidden_states) - hidden_states <- self$dropout(hidden_states) - hidden_states <- self$proj_out(hidden_states) - hidden_states - } -) - -# ------------------------------------------------------------------------------ -# RMSNorm -# ------------------------------------------------------------------------------ - -#' RMS Normalization -#' @keywords internal -rms_norm <- torch::nn_module( - "RMSNorm", - initialize = function( - dim, - eps = 1e-6, - elementwise_affine = TRUE - ) { - self$eps <- eps - self$elementwise_affine <- elementwise_affine - if (elementwise_affine) { - self$weight <- torch::nn_parameter(torch::torch_ones(dim)) - } - }, - forward = function(hidden_states) { - input_dtype <- hidden_states$dtype - hidden_states <- hidden_states$to(dtype = torch::torch_float32()) - variance <- hidden_states$pow(2)$mean(dim = - 1L, keepdim = TRUE) - hidden_states <- hidden_states * torch::torch_rsqrt(variance + self$eps) - if (self$elementwise_affine) { - hidden_states <- hidden_states * self$weight - } - hidden_states$to(dtype = input_dtype) - } -) - -# ------------------------------------------------------------------------------ -# LTX2 Adaptive Layer Norm Single -# ------------------------------------------------------------------------------ - -#' LTX2 AdaLayerNorm Single -#' @keywords internal -ltx2_ada_layer_norm_single <- torch::nn_module( - "LTX2AdaLayerNormSingle", - initialize = function( - embedding_dim, - num_mod_params = 6L, - use_additional_conditions = FALSE - ) { - self$num_mod_params <- num_mod_params - - self$emb <- pixart_alpha_combined_timestep_size_embeddings( - embedding_dim = embedding_dim, - size_emb_dim = embedding_dim %/% 3L, - use_additional_conditions = use_additional_conditions - ) - - self$silu <- torch::nn_silu() - self$linear <- make_linear(embedding_dim, num_mod_params * embedding_dim) - }, - forward = function( - timestep, - added_cond_kwargs = NULL, - batch_size = NULL, - hidden_dtype = NULL - ) { - if (is.null(added_cond_kwargs)) { - added_cond_kwargs <- list(resolution = NULL, aspect_ratio = NULL) - } - - embedded_timestep <- self$emb( - timestep, - resolution = added_cond_kwargs$resolution, - aspect_ratio = added_cond_kwargs$aspect_ratio, - batch_size = batch_size, - hidden_dtype = hidden_dtype - ) - - mod_params <- self$linear(self$silu(embedded_timestep)) - list(mod_params, embedded_timestep) - } -) - -# ------------------------------------------------------------------------------ -# LTX2 Attention -# ------------------------------------------------------------------------------ - -#' Apply interleaved rotary embedding -#' @keywords internal -apply_interleaved_rotary_emb_list <- function( - x, - freqs -) { - cos_freqs <- freqs[[1]] - sin_freqs <- freqs[[2]] - - # x: [B, S, C] - # Split into real and imaginary parts - x_shape <- x$shape - x_reshaped <- x$unflatten(3, c(- 1L, 2L)) # [B, S, C//2, 2] - x_real <- x_reshaped[,,, 1] - x_imag <- x_reshaped[,,, 2] - - # Rotate: [-x_imag, x_real] - x_rotated <- torch::torch_stack(list(- x_imag, x_real), dim = - 1L)$flatten(start_dim = 3L) - - # Apply rotation - out <- (x$to(dtype = torch::torch_float32()) * cos_freqs + - x_rotated$to(dtype = torch::torch_float32()) * sin_freqs)$to(dtype = x$dtype) - - out -} - -#' LTX2 Attention module -#' @keywords internal -ltx2_attention <- torch::nn_module( - "LTX2Attention", - initialize = function( - query_dim, - heads = 8L, - kv_heads = 8L, - dim_head = 64L, - dropout = 0.0, - bias = TRUE, - cross_attention_dim = NULL, - out_bias = TRUE, - qk_norm = "rms_norm_across_heads", - norm_eps = 1e-6, - norm_elementwise_affine = TRUE, - rope_type = "interleaved" - ) { - - self$head_dim <- dim_head - self$inner_dim <- dim_head * heads - if (is.null(kv_heads)) { - self$inner_kv_dim <- self$inner_dim - } else { - self$inner_kv_dim <- dim_head * kv_heads - } - self$query_dim <- query_dim - if (!is.null(cross_attention_dim)) { - self$cross_attention_dim <- cross_attention_dim - } else { - self$cross_attention_dim <- query_dim - } - self$use_bias <- bias - self$dropout_p <- dropout - self$out_dim <- query_dim - self$heads <- heads - self$rope_type <- rope_type - - # QK normalization - self$norm_q <- rms_norm(dim_head * heads, eps = norm_eps, elementwise_affine = norm_elementwise_affine) - if (is.null(kv_heads)) { - self$norm_k <- rms_norm(dim_head * heads, eps = norm_eps, elementwise_affine = norm_elementwise_affine) - } else { - self$norm_k <- rms_norm(dim_head * kv_heads, eps = norm_eps, elementwise_affine = norm_elementwise_affine) - } - - # Projections - self$to_q <- make_linear(query_dim, self$inner_dim, bias = bias) - self$to_k <- make_linear(self$cross_attention_dim, self$inner_kv_dim, bias = bias) - self$to_v <- make_linear(self$cross_attention_dim, self$inner_kv_dim, bias = bias) - - # Output projection - self$to_out <- torch::nn_sequential( - make_linear(self$inner_dim, self$out_dim, bias = out_bias), - torch::nn_dropout(p = dropout) - ) - }, - forward = function( - hidden_states, - encoder_hidden_states = NULL, - attention_mask = NULL, - query_rotary_emb = NULL, - key_rotary_emb = NULL - ) { - batch_size <- hidden_states$shape[1] - - if (is.null(encoder_hidden_states)) { - encoder_hidden_states <- hidden_states - } - - # Project Q, K, V - query <- self$to_q(hidden_states) - key <- self$to_k(encoder_hidden_states) - value <- self$to_v(encoder_hidden_states) - - # Normalize Q, K - query <- self$norm_q(query) - key <- self$norm_k(key) - - # Apply RoPE - if (!is.null(query_rotary_emb)) { - if (self$rope_type == "interleaved") { - query <- apply_interleaved_rotary_emb_list(query, query_rotary_emb) - if (!is.null(key_rotary_emb)) { - key_rope <- key_rotary_emb - } else { - key_rope <- query_rotary_emb - } - key <- apply_interleaved_rotary_emb_list(key, key_rope) - } - } - - # Reshape for multi-head attention [B, S, H, D] - query <- query$unflatten(3, c(self$heads, - 1L)) - key <- key$unflatten(3, c(self$heads, - 1L)) - value <- value$unflatten(3, c(self$heads, - 1L)) - - # Transpose to [B, H, S, D] - query <- query$transpose(2L, 3L) - key <- key$transpose(2L, 3L) - value <- value$transpose(2L, 3L) - - # Scaled dot-product attention (manual implementation) - scale <- 1.0 / sqrt(self$head_dim) - attn_weights <- torch::torch_matmul(query, key$transpose(- 2L, - 1L)) * scale - - if (!is.null(attention_mask)) { - # Expand attention mask to [B, 1, 1, S] for broadcasting with [B, H, S, S] - if (attention_mask$ndim == 2L) { - attention_mask <- attention_mask$unsqueeze(2L)$unsqueeze(2L) # [B, S] -> [B, 1, 1, S] - } else if (attention_mask$ndim == 3L) { - attention_mask <- attention_mask$unsqueeze(2L) # [B, 1, S] -> [B, 1, 1, S] - } - attn_weights <- attn_weights + attention_mask - } - - attn_weights <- torch::nnf_softmax(attn_weights, dim = - 1L) - - if (self$training && self$dropout_p > 0) { - attn_weights <- torch::nnf_dropout(attn_weights, p = self$dropout_p) - } - - hidden_states <- torch::torch_matmul(attn_weights, value) - - # Reshape back [B, H, S, D] -> [B, S, H*D] - hidden_states <- hidden_states$transpose(2L, 3L)$flatten(start_dim = 3L) - hidden_states <- hidden_states$to(dtype = query$dtype) - - # Output projection - hidden_states <- self$to_out(hidden_states) - - hidden_states - } -) - -# ------------------------------------------------------------------------------ -# LTX2 Video Transformer Block -# ------------------------------------------------------------------------------ - -#' LTX2 Video Transformer Block (Audio-Video) -#' @keywords internal -ltx2_video_transformer_block <- torch::nn_module( - "LTX2VideoTransformerBlock", - initialize = function( - dim, - num_attention_heads, - attention_head_dim, - cross_attention_dim, - audio_dim, - audio_num_attention_heads, - audio_attention_head_dim, - audio_cross_attention_dim, - qk_norm = "rms_norm_across_heads", - activation_fn = "gelu-approximate", - attention_bias = TRUE, - attention_out_bias = TRUE, - eps = 1e-6, - elementwise_affine = FALSE, - rope_type = "interleaved" - ) { - - # 1. Video Self-Attention - self$norm1 <- rms_norm(dim, eps = eps, elementwise_affine = elementwise_affine) - self$attn1 <- ltx2_attention( - query_dim = dim, - heads = num_attention_heads, - kv_heads = num_attention_heads, - dim_head = attention_head_dim, - bias = attention_bias, - cross_attention_dim = NULL, - out_bias = attention_out_bias, - qk_norm = qk_norm, - rope_type = rope_type - ) - - # Audio Self-Attention - self$audio_norm1 <- rms_norm(audio_dim, eps = eps, elementwise_affine = elementwise_affine) - self$audio_attn1 <- ltx2_attention( - query_dim = audio_dim, - heads = audio_num_attention_heads, - kv_heads = audio_num_attention_heads, - dim_head = audio_attention_head_dim, - bias = attention_bias, - cross_attention_dim = NULL, - out_bias = attention_out_bias, - qk_norm = qk_norm, - rope_type = rope_type - ) - - # 2. Video Cross-Attention (with text) - self$norm2 <- rms_norm(dim, eps = eps, elementwise_affine = elementwise_affine) - self$attn2 <- ltx2_attention( - query_dim = dim, - cross_attention_dim = cross_attention_dim, - heads = num_attention_heads, - kv_heads = num_attention_heads, - dim_head = attention_head_dim, - bias = attention_bias, - out_bias = attention_out_bias, - qk_norm = qk_norm, - rope_type = rope_type - ) - - # Audio Cross-Attention (with text) - self$audio_norm2 <- rms_norm(audio_dim, eps = eps, elementwise_affine = elementwise_affine) - self$audio_attn2 <- ltx2_attention( - query_dim = audio_dim, - cross_attention_dim = audio_cross_attention_dim, - heads = audio_num_attention_heads, - kv_heads = audio_num_attention_heads, - dim_head = audio_attention_head_dim, - bias = attention_bias, - out_bias = attention_out_bias, - qk_norm = qk_norm, - rope_type = rope_type - ) - - # 3. Audio-to-Video Cross-Attention (Q: Video, K/V: Audio) - self$audio_to_video_norm <- rms_norm(dim, eps = eps, elementwise_affine = elementwise_affine) - self$audio_to_video_attn <- ltx2_attention( - query_dim = dim, - cross_attention_dim = audio_dim, - heads = audio_num_attention_heads, - kv_heads = audio_num_attention_heads, - dim_head = audio_attention_head_dim, - bias = attention_bias, - out_bias = attention_out_bias, - qk_norm = qk_norm, - rope_type = rope_type - ) - - # Video-to-Audio Cross-Attention (Q: Audio, K/V: Video) - self$video_to_audio_norm <- rms_norm(audio_dim, eps = eps, elementwise_affine = elementwise_affine) - self$video_to_audio_attn <- ltx2_attention( - query_dim = audio_dim, - cross_attention_dim = dim, - heads = audio_num_attention_heads, - kv_heads = audio_num_attention_heads, - dim_head = audio_attention_head_dim, - bias = attention_bias, - out_bias = attention_out_bias, - qk_norm = qk_norm, - rope_type = rope_type - ) - - # 4. Feedforward layers - self$norm3 <- rms_norm(dim, eps = eps, elementwise_affine = elementwise_affine) - self$ff <- feed_forward(dim, activation_fn = activation_fn) - - self$audio_norm3 <- rms_norm(audio_dim, eps = eps, elementwise_affine = elementwise_affine) - self$audio_ff <- feed_forward(audio_dim, activation_fn = activation_fn) - - # 5. Per-layer modulation parameters - self$scale_shift_table <- torch::nn_parameter( - torch::torch_randn(6L, dim) / sqrt(dim) - ) - self$audio_scale_shift_table <- torch::nn_parameter( - torch::torch_randn(6L, audio_dim) / sqrt(audio_dim) - ) - - # Cross-attention modulation parameters - self$video_a2v_cross_attn_scale_shift_table <- torch::nn_parameter(torch::torch_randn(5L, dim)) - self$audio_a2v_cross_attn_scale_shift_table <- torch::nn_parameter(torch::torch_randn(5L, audio_dim)) - }, - forward = function( - hidden_states, - audio_hidden_states, - encoder_hidden_states, - audio_encoder_hidden_states, - temb, - temb_audio, - temb_ca_scale_shift, - temb_ca_audio_scale_shift, - temb_ca_gate, - temb_ca_audio_gate, - video_rotary_emb = NULL, - audio_rotary_emb = NULL, - ca_video_rotary_emb = NULL, - ca_audio_rotary_emb = NULL, - encoder_attention_mask = NULL, - audio_encoder_attention_mask = NULL, - a2v_cross_attention_mask = NULL, - v2a_cross_attention_mask = NULL - ) { - - batch_size <- hidden_states$shape[1] - - # 1. Video and Audio Self-Attention - norm_hidden_states <- self$norm1(hidden_states) - - # Ada values for video - num_ada_params <- self$scale_shift_table$shape[1] - ada_values <- self$scale_shift_table$unsqueeze(1)$unsqueeze(1)$to(device = temb$device, dtype = temb$dtype) + - temb$reshape(c(batch_size, temb$shape[2], num_ada_params, - 1L)) - - shift_msa <- ada_values[,, 1,] - scale_msa <- ada_values[,, 2,] - gate_msa <- ada_values[,, 3,] - shift_mlp <- ada_values[,, 4,] - scale_mlp <- ada_values[,, 5,] - gate_mlp <- ada_values[,, 6,] - - norm_hidden_states <- norm_hidden_states * scale_msa$add(1) + shift_msa - - attn_hidden_states <- self$attn1( - hidden_states = norm_hidden_states, - encoder_hidden_states = NULL, - query_rotary_emb = video_rotary_emb - ) - hidden_states <- hidden_states + attn_hidden_states * gate_msa - - # Audio self-attention - norm_audio_hidden_states <- self$audio_norm1(audio_hidden_states) - - num_audio_ada_params <- self$audio_scale_shift_table$shape[1] - audio_ada_values <- self$audio_scale_shift_table$unsqueeze(1)$unsqueeze(1)$to(device = temb_audio$device, dtype = temb_audio$dtype) + - temb_audio$reshape(c(batch_size, temb_audio$shape[2], num_audio_ada_params, - 1L)) - - audio_shift_msa <- audio_ada_values[,, 1,] - audio_scale_msa <- audio_ada_values[,, 2,] - audio_gate_msa <- audio_ada_values[,, 3,] - audio_shift_mlp <- audio_ada_values[,, 4,] - audio_scale_mlp <- audio_ada_values[,, 5,] - audio_gate_mlp <- audio_ada_values[,, 6,] - - norm_audio_hidden_states <- norm_audio_hidden_states * audio_scale_msa$add(1) + audio_shift_msa - - attn_audio_hidden_states <- self$audio_attn1( - hidden_states = norm_audio_hidden_states, - encoder_hidden_states = NULL, - query_rotary_emb = audio_rotary_emb - ) - audio_hidden_states <- audio_hidden_states + attn_audio_hidden_states * audio_gate_msa - - # 2. Video and Audio Cross-Attention with text - norm_hidden_states <- self$norm2(hidden_states) - attn_hidden_states <- self$attn2( - norm_hidden_states, - encoder_hidden_states = encoder_hidden_states, - query_rotary_emb = NULL, - attention_mask = encoder_attention_mask - ) - hidden_states <- hidden_states + attn_hidden_states - - norm_audio_hidden_states <- self$audio_norm2(audio_hidden_states) - attn_audio_hidden_states <- self$audio_attn2( - norm_audio_hidden_states, - encoder_hidden_states = audio_encoder_hidden_states, - query_rotary_emb = NULL, - attention_mask = audio_encoder_attention_mask - ) - audio_hidden_states <- audio_hidden_states + attn_audio_hidden_states - - # 3. Audio-to-Video and Video-to-Audio Cross-Attention - norm_hidden_states <- self$audio_to_video_norm(hidden_states) - norm_audio_hidden_states <- self$video_to_audio_norm(audio_hidden_states) - - # Video cross-attention modulation - video_per_layer_ca_scale_shift <- self$video_a2v_cross_attn_scale_shift_table[1:4,] - video_per_layer_ca_gate <- self$video_a2v_cross_attn_scale_shift_table[5:5,] - - video_ca_scale_shift_table <- video_per_layer_ca_scale_shift$unsqueeze(1)$unsqueeze(1)$to(dtype = temb_ca_scale_shift$dtype) + - temb_ca_scale_shift$reshape(c(batch_size, temb_ca_scale_shift$shape[2], 4L, - 1L)) - video_ca_gate <- video_per_layer_ca_gate$unsqueeze(1)$unsqueeze(1)$to(dtype = temb_ca_gate$dtype) + - temb_ca_gate$reshape(c(batch_size, temb_ca_gate$shape[2], 1L, - 1L)) - - video_a2v_ca_scale <- video_ca_scale_shift_table[,, 1,] - video_a2v_ca_shift <- video_ca_scale_shift_table[,, 2,] - video_v2a_ca_scale <- video_ca_scale_shift_table[,, 3,] - video_v2a_ca_shift <- video_ca_scale_shift_table[,, 4,] - a2v_gate <- video_ca_gate[,, 1,] - - # Audio cross-attention modulation - audio_per_layer_ca_scale_shift <- self$audio_a2v_cross_attn_scale_shift_table[1:4,] - audio_per_layer_ca_gate <- self$audio_a2v_cross_attn_scale_shift_table[5:5,] - - audio_ca_scale_shift_table <- audio_per_layer_ca_scale_shift$unsqueeze(1)$unsqueeze(1)$to(dtype = temb_ca_audio_scale_shift$dtype) + - temb_ca_audio_scale_shift$reshape(c(batch_size, temb_ca_audio_scale_shift$shape[2], 4L, - 1L)) - audio_ca_gate <- audio_per_layer_ca_gate$unsqueeze(1)$unsqueeze(1)$to(dtype = temb_ca_audio_gate$dtype) + - temb_ca_audio_gate$reshape(c(batch_size, temb_ca_audio_gate$shape[2], 1L, - 1L)) - - audio_a2v_ca_scale <- audio_ca_scale_shift_table[,, 1,] - audio_a2v_ca_shift <- audio_ca_scale_shift_table[,, 2,] - audio_v2a_ca_scale <- audio_ca_scale_shift_table[,, 3,] - audio_v2a_ca_shift <- audio_ca_scale_shift_table[,, 4,] - v2a_gate <- audio_ca_gate[,, 1,] - - # Audio-to-Video Cross Attention - mod_norm_hidden_states <- norm_hidden_states * video_a2v_ca_scale$add(1) + video_a2v_ca_shift - mod_norm_audio_hidden_states <- norm_audio_hidden_states * audio_a2v_ca_scale$add(1) + audio_a2v_ca_shift - - a2v_attn_hidden_states <- self$audio_to_video_attn( - mod_norm_hidden_states, - encoder_hidden_states = mod_norm_audio_hidden_states, - query_rotary_emb = ca_video_rotary_emb, - key_rotary_emb = ca_audio_rotary_emb, - attention_mask = a2v_cross_attention_mask - ) - hidden_states <- hidden_states + a2v_gate * a2v_attn_hidden_states - - # Video-to-Audio Cross Attention - mod_norm_hidden_states <- norm_hidden_states * video_v2a_ca_scale$add(1) + video_v2a_ca_shift - mod_norm_audio_hidden_states <- norm_audio_hidden_states * audio_v2a_ca_scale$add(1) + audio_v2a_ca_shift - - v2a_attn_hidden_states <- self$video_to_audio_attn( - mod_norm_audio_hidden_states, - encoder_hidden_states = mod_norm_hidden_states, - query_rotary_emb = ca_audio_rotary_emb, - key_rotary_emb = ca_video_rotary_emb, - attention_mask = v2a_cross_attention_mask - ) - audio_hidden_states <- audio_hidden_states + v2a_gate * v2a_attn_hidden_states - - # 4. Feedforward - norm_hidden_states <- self$norm3(hidden_states) * scale_mlp$add(1) + shift_mlp - ff_output <- self$ff(norm_hidden_states) - hidden_states <- hidden_states + ff_output * gate_mlp - - norm_audio_hidden_states <- self$audio_norm3(audio_hidden_states) * audio_scale_mlp$add(1) + audio_shift_mlp - audio_ff_output <- self$audio_ff(norm_audio_hidden_states) - audio_hidden_states <- audio_hidden_states + audio_ff_output * audio_gate_mlp - - list(hidden_states, audio_hidden_states) - } -) - diff --git a/R/gemma3_text_encoder.R b/R/gemma3_text_encoder.R index 5d111f3..613d9b9 100644 --- a/R/gemma3_text_encoder.R +++ b/R/gemma3_text_encoder.R @@ -796,24 +796,26 @@ tokenize_gemma3 <- function( #' Encode text with Gemma3 for LTX-2 #' #' Full pipeline for encoding text prompts using Gemma3 text encoder. -#' Returns packed embeddings ready for LTX-2 connectors. +#' Returns the raw stacked per-layer hidden states (embedding layer plus all +#' transformer layers) for downstream connector modules, which handle +#' normalization and projection themselves. #' #' @param prompts Character vector of prompts. #' @param model Gemma3 text model (or path to load from). #' @param tokenizer Gemma3 tokenizer (or path to load from). #' @param max_sequence_length Integer. Maximum sequence length. -#' @param scale_factor Numeric. Scale factor for packing (default 8). #' @param device Character. Device for computation. #' @param dtype Character. Data type. #' @param verbose Logical. Print progress. -#' @return List with prompt_embeds and prompt_attention_mask. +#' @return List with prompt_embeds (raw stacked hidden states, +#' shape \code{[batch, seq_len, hidden_size, num_layers + 1]}) and +#' prompt_attention_mask. #' @export encode_with_gemma3 <- function( prompts, model = NULL, tokenizer = NULL, max_sequence_length = 1024L, - scale_factor = 8, device = "cuda", dtype = "float16", verbose = TRUE @@ -852,29 +854,15 @@ encode_with_gemma3 <- function( output <- model(input_ids, attention_mask = attention_mask, output_hidden_states = TRUE) }) - # Stack hidden states from transformer layers (skip embedding layer at index 1) + # Stack ALL hidden states: embedding layer + every transformer layer. + # The LTX connectors consume the full stack (their projection expects + # hidden_size * (num_layers + 1) inputs) and normalize it themselves. hidden_states_list <- output$hidden_states - # hidden_states_list[[1]] is embedding layer, [[2]] onwards are transformer layers - # LTX-2 connectors expect 49 layers (text_proj_in_factor=49) - transformer_hidden_states <- hidden_states_list[2:length(hidden_states_list)] - # Stack: [batch, seq_len, hidden_size, num_layers] - hidden_states_stacked <- torch::torch_stack(transformer_hidden_states, dim = - 1L) - - # Compute sequence lengths from attention mask - sequence_lengths <- as.integer(attention_mask$sum(dim = 2L)$cpu()) - - # Pack embeddings - if (verbose) message("Packing embeddings...") - prompt_embeds <- pack_text_embeds( - hidden_states_stacked, - sequence_lengths = sequence_lengths, - padding_side = "left", - scale_factor = scale_factor, - device = device - ) + # Stack: [batch, seq_len, hidden_size, num_layers + 1] + hidden_states_stacked <- torch::torch_stack(hidden_states_list, dim = - 1L) list( - prompt_embeds = prompt_embeds, + prompt_embeds = hidden_states_stacked, prompt_attention_mask = attention_mask ) } diff --git a/R/gpu_poor.R b/R/gpu_poor.R deleted file mode 100644 index 39d7db0..0000000 --- a/R/gpu_poor.R +++ /dev/null @@ -1,1785 +0,0 @@ -#' GPU-Poor Memory Management for LTX-2 -#' -#' wan2GP-style memory optimizations for running LTX-2 video generation -#' on limited VRAM (6-16GB). -#' -#' @name gpu_poor -NULL - -#' Get LTX-2 Memory Profile -#' -#' Determines optimal memory configuration based on available VRAM. -#' -#' @param vram_gb Numeric. Available VRAM in GB, or NULL for auto-detection. -#' @param model Character. Model variant: "ltx2-19b-fp4" (default), "ltx2-19b-fp8", -#' or "ltx2-19b-distilled". -#' -#' @return A list with memory profile settings. -#' -#' @details -#' LTX-2 is a 19B parameter model. Even with FP4 quantization (~10GB weights), -#' it requires careful memory management. The GPU-poor approach: -#' -#' 1. Text encoding runs on CPU (cached) -#' 2. DiT loaded in chunks, processed layer-by-layer, unloaded -#' 3. VAE loaded after DiT unload, decode with tiling, unload -#' -#' Memory profiles: -#' \describe{ -#' \item{high}{16GB+ - FP4 DiT with chunk loading, VAE on GPU} -#' \item{medium}{12GB - FP4 DiT chunk loading, VAE tiled} -#' \item{low}{8GB - FP4 DiT layer-by-layer, VAE tiled small} -#' \item{very_low}{6GB - FP4 layer-by-layer, VAE on CPU} -#' \item{cpu_only}{All on CPU} -#' } -#' -#' @export -#' -#' @examples -#' \dontrun{ -#' # Auto-detect profile -#' profile <- ltx2_memory_profile() -#' -#' # Specific VRAM -#' profile <- ltx2_memory_profile(vram_gb = 8) -#' } -ltx2_memory_profile <- function( - vram_gb = NULL, - model = "ltx2-19b-fp4" -) { - # Auto-detect free VRAM if not provided - if (is.null(vram_gb)) { - vram_gb <- .detect_vram(use_free = TRUE) - message(sprintf("Detected %.1f GB free VRAM", vram_gb)) - } - - # Determine profile level - if (vram_gb >= 16) { - profile <- "high" - } else if (vram_gb >= 12) { - profile <- "medium" - } else if (vram_gb >= 8) { - profile <- "low" - } else if (vram_gb >= 6) { - profile <- "very_low" - } else { - profile <- "cpu_only" - } - - # Build profile config - # Note: LTX-2 19B has 48 transformer layers - # At FP4, ~10GB total model weights - # Layer chunk size determines how many layers loaded at once - - profiles <- list( - high = list( - name = "high", - # Stage 1: Text encoding (always CPU) - text_device = "cpu", - text_backend = "native", # Native Gemma3 encoder - # Stage 2: DiT denoising - dit_device = "cuda", - dit_offload = "chunk", # Load layers in chunks - dit_chunk_size = 12L, # 12 layers at a time (~2.5GB) - # Stage 3: VAE decode - vae_device = "cuda", - vae_tiling = FALSE, - vae_tile_size = c(512L, 512L), - vae_tile_frames = 16L, - # General settings - dtype = "float16", - model_precision = "fp4", # Preferred quantization - max_resolution = c(720L, 1280L), # height, width - max_frames = 121L, - cfg_mode = "batched"# Distilled uses CFG=1, so this is moot - ), - medium = list( - name = "medium", - text_device = "cpu", - text_backend = "native", - dit_device = "cuda", - dit_offload = "chunk", - dit_chunk_size = 8L, # 8 layers at a time (~1.7GB) - vae_device = "cuda", - vae_tiling = TRUE, - vae_tile_size = c(512L, 512L), - vae_tile_frames = 16L, - dtype = "float16", - model_precision = "fp4", - max_resolution = c(720L, 1280L), - max_frames = 121L, - cfg_mode = "batched" - ), - low = list( - name = "low", - text_device = "cpu", - text_backend = "native", - dit_device = "cuda", - dit_offload = "layer", # One layer at a time - dit_chunk_size = 1L, - vae_device = "cuda", - vae_tiling = TRUE, - vae_tile_size = c(256L, 256L), - vae_tile_frames = 8L, - dtype = "float16", - model_precision = "fp4", - max_resolution = c(480L, 854L), - max_frames = 61L, - cfg_mode = "sequential" - ), - very_low = list( - name = "very_low", - text_device = "cpu", - text_backend = "native", - dit_device = "cuda", - dit_offload = "layer", - dit_chunk_size = 1L, - vae_device = "cpu", # VAE on CPU - vae_tiling = TRUE, - vae_tile_size = c(128L, 128L), - vae_tile_frames = 4L, - dtype = "float16", - model_precision = "fp4", - max_resolution = c(480L, 640L), - max_frames = 33L, - cfg_mode = "sequential" - ), - cpu_only = list( - name = "cpu_only", - text_device = "cpu", - text_backend = "native", - dit_device = "cpu", - dit_offload = "none", - dit_chunk_size = 48L, # All layers (CPU has more RAM) - vae_device = "cpu", - vae_tiling = TRUE, - vae_tile_size = c(256L, 256L), - vae_tile_frames = 8L, - dtype = "float32", # CPU often faster with float32 - model_precision = "fp4", - max_resolution = c(480L, 640L), - max_frames = 33L, - cfg_mode = "sequential" - ) - ) - - profiles[[profile]] -} - -#' Get SDXL Memory Profile -#' -#' Determines optimal memory configuration for SDXL image generation -#' based on available VRAM. -#' -#' @param vram_gb Numeric. Available VRAM in GB, or NULL for auto-detection. -#' -#' @return A list with memory profile settings. -#' -#' @details -#' Memory profiles for SDXL: -#' \describe{ -#' \item{full_gpu}{16GB+ - All components on CUDA} -#' \item{balanced}{10-12GB - UNet + decoder on CUDA, text encoders on CPU} -#' \item{unet_gpu}{6-10GB - Only UNet on CUDA, everything else CPU} -#' \item{cpu_only}{<6GB - All on CPU} -#' } -#' -#' Each profile also specifies: -#' - cfg_mode: "batched" or "sequential" (sequential halves peak memory) -#' - cleanup: "none", "phase", or "step" (when to clear VRAM) -#' - dtype: "float16" or "float32" -#' - max_resolution: maximum image dimension -#' -#' @export -#' -#' @examples -#' \dontrun{ -#' # Auto-detect profile -#' profile <- sdxl_memory_profile() -#' -#' # Specific VRAM -#' profile <- sdxl_memory_profile(vram_gb = 8) -#' } -sdxl_memory_profile <- function(vram_gb = NULL) { - # Auto-detect free VRAM if not provided - if (is.null(vram_gb)) { - vram_gb <- .detect_vram(use_free = TRUE) - message(sprintf("Detected %.1f GB free VRAM", vram_gb)) - } - - # Determine profile level - if (vram_gb >= 16) { - profile <- "full_gpu" - } else if (vram_gb >= 10) { - profile <- "balanced" - } else if (vram_gb >= 6) { - profile <- "unet_gpu" - } else { - profile <- "cpu_only" - } - - # Build profile config - profiles <- list( - full_gpu = list( - name = "full_gpu", - devices = list( - unet = "cuda", - decoder = "cuda", - text_encoder = "cuda", - text_encoder2 = "cuda", - encoder = "cuda" - ), - dtype = "float16", - cfg_mode = "batched", - cleanup = "none", - max_resolution = 1536L, - step_cleanup_interval = 0L# No step cleanup - ), - balanced = list( - name = "balanced", - devices = list( - unet = "cuda", - decoder = "cuda", - text_encoder = "cpu", - text_encoder2 = "cpu", - encoder = "cpu" - ), - dtype = "float16", - cfg_mode = "batched", - cleanup = "phase", # Cleanup between text encoding and denoising - max_resolution = 1024L, - step_cleanup_interval = 0L - ), - unet_gpu = list( - name = "unet_gpu", - devices = list( - unet = "cuda", - decoder = "cpu", - text_encoder = "cpu", - text_encoder2 = "cpu", - encoder = "cpu" - ), - dtype = "float16", - cfg_mode = "sequential", # Sequential CFG halves peak memory - cleanup = "phase", - max_resolution = 1024L, - step_cleanup_interval = 10L# Cleanup every 10 steps - ), - cpu_only = list( - name = "cpu_only", - devices = list( - unet = "cpu", - decoder = "cpu", - text_encoder = "cpu", - text_encoder2 = "cpu", - encoder = "cpu" - ), - dtype = "float32", # CPU often faster with float32 - cfg_mode = "sequential", - cleanup = "none", # No GPU to clean - max_resolution = 768L, - step_cleanup_interval = 0L - ) - ) - - profiles[[profile]] -} - -#' Check if GPU is Blackwell Architecture -#' -#' Blackwell GPUs (RTX 50xx) may need special handling. -#' -#' @return Logical. TRUE if Blackwell GPU detected. -#' -#' @export -#' -#' @examples -#' \dontrun{ -#' if (is_blackwell_gpu()) { -#' message("Using Blackwell-compatible settings") -#' } -#' } -is_blackwell_gpu <- function() { - # Use gpuctl if available - if (requireNamespace("gpu.ctl", quietly = TRUE)) { - return(gpu.ctl::gpu_is_blackwell()) - } - - # Fallback: check compute capability via torch - if (torch::cuda_is_available()) { - props <- tryCatch( - torch::cuda_get_device_properties(0L), - error = function(e) NULL - ) - if (!is.null(props)) { - # Blackwell is compute 12.x - major <- props$major - return(major >= 12) - } - } - - FALSE -} - -#' Detect Available VRAM -#' -#' Uses gpuctl if available. -#' -#' @param use_free Logical. If TRUE, return free VRAM. If FALSE, return total. -#' -#' @return Numeric. VRAM in GB, or 0 if no GPU detected. -#' @keywords internal -.detect_vram <- function(use_free = FALSE) { - # Try gpuctl (preferred - uses nvidia-smi) - if (requireNamespace("gpu.ctl", quietly = TRUE)) { - info <- gpu.ctl::gpu_detect() - if (!is.null(info)) { - if (use_free && !is.null(info$vram_free_gb)) { - return(info$vram_free_gb) - } - if (!is.null(info$vram_total_gb)) { - return(info$vram_total_gb) - } - } - } - - # Fallback: check if CUDA available but can't determine VRAM - if (torch::cuda_is_available()) { - # Conservative estimate - assume 8GB if we can't detect - message("Could not detect VRAM. Install gpuctl for accurate detection.") - return(8) - } - - # No GPU detected - 0 -} - -#' Offload Module to CPU -#' -#' Moves a torch module and all its parameters to CPU. -#' -#' @param module A torch nn_module. -#' @param gc Logical. Run garbage collection after offload. -#' -#' @return The module (modified in place). -#' -#' @export -#' -#' @examples -#' \dontrun{ -#' model$to(device = "cuda") -#' output <- model(x) -#' offload_to_cpu(model) -#' } -offload_to_cpu <- function( - module, - gc = TRUE -) { - module$to(device = "cpu") - if (gc && torch::cuda_is_available()) { - gc() - torch::cuda_empty_cache() - } - invisible(module) -} - -#' Load Module to GPU -#' -#' Moves a torch module and all its parameters to CUDA. -#' -#' @param module A torch nn_module. -#' @param device Character. Target device (default "cuda"). -#' -#' @return The module (modified in place). -#' -#' @export -#' -#' @examples -#' \dontrun{ -#' load_to_gpu(model) -#' output <- model(x) -#' offload_to_cpu(model) -#' } -load_to_gpu <- function( - module, - device = "cuda" -) { - module$to(device = device) - invisible(module) -} - -#' Report VRAM Usage -#' -#' Prints current VRAM usage using gpuctl. -#' -#' @param label Character. Label for the report. -#' -#' @return Invisibly returns a list with used and free VRAM in GB. -#' -#' @export -#' -#' @examples -#' \dontrun{ -#' vram_report("After model load") -#' } -vram_report <- function(label = "") { - if (!torch::cuda_is_available()) { - message("[", label, "] No CUDA available") - return(invisible(list(used = 0, free = 0))) - } - - # Use gpuctl for accurate reporting - if (requireNamespace("gpu.ctl", quietly = TRUE)) { - info <- gpu.ctl::gpu_detect() - if (!is.null(info)) { - used <- info$vram_used_gb - free <- info$vram_free_gb - message(sprintf("[%s] VRAM: %.2f GB used, %.2f GB free", - label, used, free)) - return(invisible(list(used = used, free = free))) - } - } - - message("[", label, "] VRAM: (install gpuctl for detailed stats)") - invisible(list(used = NA, free = NA)) -} - -#' Clear VRAM Cache -#' -#' Forces garbage collection and clears CUDA memory cache. -#' -#' @param verbose Logical. Print memory status before/after. -#' -#' @return Invisibly returns NULL. -#' -#' @export -#' -#' @examples -#' \dontrun{ -#' clear_vram() -#' } -clear_vram <- function(verbose = FALSE) { - if (!torch::cuda_is_available()) { - return(invisible(NULL)) - } - - if (verbose) { - vram_report("Before clear") - } - - gc() - tryCatch( - torch::cuda_empty_cache(), - error = function(e) NULL - ) - - if (verbose) { - vram_report("After clear") - } - - invisible(NULL) -} - -#' DiT Chunk-based Forward Pass -#' -#' Runs transformer layers in chunks, moving each chunk to GPU before -#' computation and back to CPU after. Balances memory usage with speed. -#' -#' @param hidden_states Input tensor. -#' @param layers List of transformer layers (on CPU). -#' @param chunk_size Integer. Number of layers to load at once (default 1). -#' @param device Target device for computation. -#' @param verbose Logical. Print progress. -#' @param ... Additional arguments passed to each layer. -#' -#' @return Output tensor (on CPU). -#' -#' @export -#' -#' @examples -#' \dontrun{ -#' # Layer-by-layer for 8GB VRAM -#' output <- dit_offloaded_forward( -#' hidden_states, -#' model$transformer_blocks, -#' chunk_size = 1, -#' device = "cuda" -#' ) -#' -#' # Chunk-based for 16GB VRAM -#' output <- dit_offloaded_forward( -#' hidden_states, -#' model$transformer_blocks, -#' chunk_size = 12, # 12 layers at a time -#' device = "cuda" -#' ) -#' } -dit_offloaded_forward <- function( - hidden_states, - layers, - chunk_size = 1L, - device = "cuda", - verbose = FALSE, - ... -) { - n_layers <- length(layers) - chunk_size <- as.integer(chunk_size) - - # Move input to target device - x <- hidden_states$to(device = device) - - # Process in chunks - chunk_start <- 1L - while (chunk_start <= n_layers) { - chunk_end <- min(chunk_start + chunk_size - 1L, n_layers) - - if (verbose) { - message(sprintf(" Processing layers %d-%d of %d", chunk_start, chunk_end, n_layers)) - } - - # Load chunk to GPU - for (i in chunk_start:chunk_end) { - layers[[i]]$to(device = device) - } - - # Forward pass through chunk - for (i in chunk_start:chunk_end) { - x <- layers[[i]](x, ...) - } - - # Offload chunk back to CPU - for (i in chunk_start:chunk_end) { - layers[[i]]$to(device = "cpu") - } - - # Clear cache after each chunk - if (device != "cpu") { - torch::cuda_empty_cache() - } - - chunk_start <- chunk_end + 1L - } - - # Return result on CPU - x$to(device = "cpu") -} - -#' Sequential CFG Forward Pass -#' -#' Runs unconditional and conditional forward passes separately to halve -#' peak activation memory. For GPU-poor scenarios. -#' -#' @param model The DiT model. -#' @param latents Current latent tensor. -#' @param timestep Current timestep tensor. -#' @param prompt_embeds Conditional prompt embeddings. -#' @param negative_prompt_embeds Unconditional prompt embeddings. -#' @param guidance_scale CFG scale. -#' @param ... Additional arguments to model forward pass. -#' -#' @return The CFG-combined noise prediction. -#' -#' @export -#' -#' @examples -#' \dontrun{ -#' noise_pred <- sequential_cfg_forward( -#' model, latents, timestep, -#' prompt_embeds, negative_prompt_embeds, -#' guidance_scale = 4.0 -#' ) -#' } -sequential_cfg_forward <- function( - model, - latents, - timestep, - prompt_embeds, - negative_prompt_embeds, - guidance_scale, - ... -) { - torch::with_no_grad({ - # Unconditional pass - noise_pred_uncond <- model( - hidden_states = latents, - encoder_hidden_states = negative_prompt_embeds, - timestep = timestep, - ... - )$sample - - # Conditional pass - noise_pred_cond <- model( - hidden_states = latents, - encoder_hidden_states = prompt_embeds, - timestep = timestep, - ... - )$sample - - # CFG combination - noise_pred <- noise_pred_uncond + guidance_scale * (noise_pred_cond - noise_pred_uncond) - - # Clean up intermediate tensors - rm(noise_pred_uncond, noise_pred_cond) - }) - - noise_pred -} - -#' Validate Resolution Against Profile -#' -#' Checks if requested resolution fits within memory profile limits. -#' -#' @param height Integer. Requested height. -#' @param width Integer. Requested width. -#' @param num_frames Integer. Requested number of frames. -#' @param profile Memory profile from `ltx2_memory_profile()`. -#' -#' @return List with adjusted height, width, num_frames and warning if adjusted. -#' -#' @export -#' -#' @examples -#' \dontrun{ -#' profile <- ltx2_memory_profile(vram_gb = 8) -#' validated <- validate_resolution(720, 1280, 60, profile) -#' } -validate_resolution <- function( - height, - width, - num_frames, - profile -) { - adjusted <- FALSE - warnings <- character(0) - - max_h <- profile$max_resolution[1] - max_w <- profile$max_resolution[2] - max_f <- profile$max_frames - - if (height > max_h) { - warnings <- c(warnings, sprintf("Height %d exceeds profile max %d", height, max_h)) - height <- max_h - adjusted <- TRUE - } - - if (width > max_w) { - warnings <- c(warnings, sprintf("Width %d exceeds profile max %d", width, max_w)) - width <- max_w - adjusted <- TRUE - } - - if (num_frames > max_f) { - warnings <- c(warnings, sprintf("Frames %d exceeds profile max %d", num_frames, max_f)) - num_frames <- max_f - adjusted <- TRUE - } - - if (adjusted && length(warnings) > 0) { - warning("Resolution adjusted for memory profile '", profile$name, "':\n ", - paste(warnings, collapse = "\n ")) - } - - list( - height = height, - width = width, - num_frames = num_frames, - adjusted = adjusted - ) -} - -#' Configure VAE for Memory Profile -#' -#' Sets VAE tiling parameters based on memory profile. -#' -#' @param vae The LTX2 VAE module. -#' @param profile Memory profile from `ltx2_memory_profile()`. -#' -#' @return The VAE (modified in place). -#' -#' @export -#' -#' @examples -#' \dontrun{ -#' profile <- ltx2_memory_profile(vram_gb = 8) -#' vae <- load_ltx2_vae(...) -#' configure_vae_for_profile(vae, profile) -#' } -configure_vae_for_profile <- function( - vae, - profile -) { - if (profile$vae_tiling) { - vae$enable_tiling( - tile_sample_min_height = profile$vae_tile_size[1], - tile_sample_min_width = profile$vae_tile_size[2], - tile_sample_min_num_frames = profile$vae_tile_frames - ) - } else { - vae$disable_tiling() - } - - invisible(vae) -} - -#' Quantize Tensor to INT4 -#' -#' Quantizes a float tensor to 4-bit integers with block-wise scaling. -#' Two INT4 values are packed per byte for 7-8x compression. -#' -#' @param x Tensor. Input float tensor. -#' @param block_size Integer. Number of values per scale factor (default 64). -#' -#' @return A list with: -#' - `packed`: uint8 tensor with packed INT4 values -#' - `scales`: float tensor with per-block scale factors -#' - `orig_shape`: original tensor shape -#' - `orig_numel`: original number of elements -#' - `block_size`: block size used -#' -#' @details -#' INT4 range is -8 to 7. Values are scaled per block, quantized, shifted to -#' unsigned (0-15), and packed two per byte. Compression is ~7x vs float32, -#' ~3.5x vs float16. Typical reconstruction error is 10-12% of std. -#' -#' @export -#' -#' @examples -#' \dontrun{ -#' x <- torch_randn(c(4096, 4096)) * 0.02 -#' q <- quantize_int4(x) -#' x_back <- dequantize_int4(q) -#' } -quantize_int4 <- function( - x, - block_size = 64L -) { - orig_shape <- x$shape - orig_dtype <- x$dtype - x_flat <- x$to(dtype = torch::torch_float32())$flatten() - n <- x_flat$shape[1] - - # Pad to multiple of block_size * 2 (2 values per byte) - pad_to <- ceiling(n / (block_size * 2)) * block_size * 2 - if (pad_to > n) { - x_flat <- torch::torch_cat(list( - x_flat, - torch::torch_zeros(pad_to - n, dtype = torch::torch_float32(), device = x$device) - )) - } - - # Reshape into blocks - n_blocks <- as.integer(pad_to / block_size) - x_blocks <- x_flat$reshape(c(n_blocks, block_size)) - - # Compute scale per block (absmax / 7) - scales <- x_blocks$abs()$max(dim = 2) [[1]] / 7.0 - scales <- scales$clamp(min = 1e-10) - - # Quantize: scale, round, clamp to -8..7, shift to 0..15 - x_scaled <- x_blocks / scales$unsqueeze(2) - x_int <- x_scaled$round()$clamp(- 8, 7) + 8L - x_uint <- x_int$to(dtype = torch::torch_uint8()) - - # Pack pairs into bytes (high nibble * 16 + low nibble) - x_uint <- x_uint$reshape(c(n_blocks, block_size %/% 2L, 2L)) - high <- x_uint[,, 1]$to(torch::torch_int32()) * 16L - low <- x_uint[,, 2]$to(torch::torch_int32()) - packed <- (high + low)$to(torch::torch_uint8()) - - list( - packed = packed$flatten(), - scales = scales, - orig_shape = orig_shape, - orig_numel = n, - block_size = block_size - ) -} - -#' Dequantize INT4 Tensor -#' -#' Reconstructs a float tensor from INT4-quantized data. -#' -#' @param q List. Quantized data from `quantize_int4()`. -#' @param dtype Torch dtype. Output dtype (default float16). -#' @param device Character. Target device. -#' -#' @return Tensor with original shape and specified dtype. -#' -#' @export -#' -#' @examples -#' \dontrun{ -#' q <- quantize_int4(weights) -#' weights_approx <- dequantize_int4(q, dtype = torch_float16(), device = "cuda") -#' } -dequantize_int4 <- function( - q, - dtype = torch::torch_float16(), - device = "cpu" -) { - packed <- q$packed$to(dtype = torch::torch_int32(), device = device) - - # Unpack bytes: high nibble = floor(x/16), low nibble = x mod 16 - high <- torch::torch_floor(packed$to(torch::torch_float32()) / 16)$to(torch::torch_int32()) - low <- packed - high * 16L - - # Interleave high and low nibbles - x_uint <- torch::torch_stack(list(high, low), dim = 2L)$flatten() - - # Shift back to signed (-8 to 7) - x_int <- x_uint - 8L - - # Apply per-block scales - block_size <- q$block_size - x_blocks <- x_int$reshape(c(- 1L, block_size))$to(dtype = dtype) - scales_dev <- q$scales$to(dtype = dtype, device = device) - x_scaled <- x_blocks * scales_dev$unsqueeze(2) - - # Flatten and trim to original size - x_flat <- x_scaled$flatten()[1:q$orig_numel] - x_flat$reshape(q$orig_shape) -} - -#' Create Linear Layer (Standard or INT4) -#' -#' Factory function that creates either a standard nn_linear or INT4 linear layer -#' based on package options. Use this instead of torch::nn_linear() in model code. -#' -#' @param in_features Integer. Input dimension. -#' @param out_features Integer. Output dimension. -#' @param bias Logical. Include bias term. -#' -#' @return nn_linear or int4_linear module. -#' -#' @details -#' Behavior controlled by options: -#' - `diffuseR.use_int4`: If TRUE, create INT4 layer (default FALSE) -#' - `diffuseR.int4_device`: Device for INT4 layers (default "cuda") -#' - `diffuseR.int4_dtype`: Dtype for INT4 operations (default torch_float16()) -#' -#' @export -make_linear <- function( - in_features, - out_features, - bias = TRUE -) { - - if (getOption("diffuseR.use_int4", FALSE)) { - device <- getOption("diffuseR.int4_device", "cuda") - dtype <- getOption("diffuseR.int4_dtype", torch::torch_float16()) - int4_linear(in_features, out_features, bias = bias, device = device, dtype = dtype) - } else { - torch::nn_linear(in_features, out_features, bias = bias) - } -} - -#' INT4 Linear Layer -#' -#' A linear layer that stores weights in INT4 format and dequantizes on-the-fly -#' during forward pass. This enables running large models on limited VRAM by -#' keeping weights compressed on GPU. -#' -#' @param in_features Integer. Size of each input sample. -#' @param out_features Integer. Size of each output sample. -#' @param bias Logical. If TRUE, adds a learnable bias (stored in float16). -#' @param device Character. Device for the layer. -#' @param dtype torch_dtype. Data type for dequantized operations. -#' -#' @return nn_module with INT4 weight storage. -#' -#' @details -#' The layer stores: -#' - `weight_packed`: uint8 tensor with packed INT4 values -#' - `weight_scales`: float32 tensor with per-block scales -#' - `weight_shape`: original weight shape -#' - `bias`: optional float16 bias -#' -#' During forward(), weights are dequantized to the target dtype, the matmul -#' is performed, and the dequantized tensor is freed. This allows ~40GB models -#' to run with ~10GB of VRAM for weights. -#' -#' @export -int4_linear <- torch::nn_module( - "INT4Linear", - initialize = function( - in_features, - out_features, - bias = TRUE, - device = "cuda", - dtype = torch::torch_float16() - ) { - self$in_features <- in_features - self$out_features <- out_features - self$dtype <- dtype - self$device <- device - self$block_size <- 64L - - # Placeholder - will be set by load_int4_weight() - self$weight_packed <- NULL - self$weight_scales <- NULL - self$weight_shape <- c(out_features, in_features) - self$weight_numel <- out_features * in_features - - if (bias) { - self$bias <- torch::nn_parameter(torch::torch_zeros(out_features, - dtype = dtype, device = device)) - } else { - self$bias <- NULL - } - }, - -#' Load INT4 quantized weight into this layer -#' @param q List with packed, scales, orig_shape from quantize_int4() - load_int4_weight = function(q) { - # Store INT4 data as buffers (not parameters) - self$weight_packed <- q$packed$to(device = self$device) - self$weight_scales <- q$scales$to(device = self$device) - self$weight_shape <- q$orig_shape - self$weight_numel <- q$orig_numel - invisible(self) - }, - - forward = function(x) { - if (is.null(self$weight_packed)) { - stop("INT4 weight not loaded. Call load_int4_weight() first.") - } - - # Dequantize weight on-the-fly - q <- list( - packed = self$weight_packed, - scales = self$weight_scales, - orig_shape = self$weight_shape, - orig_numel = self$weight_numel, - block_size = self$block_size - ) - weight <- dequantize_int4(q, dtype = self$dtype, device = self$device) - - # Linear operation - out <- torch::nnf_linear(x, weight, self$bias) - - # Weight tensor goes out of scope and will be freed - out - } -) - -#' Create INT4 Linear from Standard Linear -#' -#' Converts a standard nn_linear layer to an INT4 linear layer. -#' -#' @param linear_module nn_linear module to convert. -#' @param device Character. Target device for INT4 weights. -#' @param dtype torch_dtype. Target dtype for dequantized operations. -#' -#' @return int4_linear module with quantized weights. -#' -#' @export -linear_to_int4 <- function( - linear_module, - device = "cuda", - dtype = torch::torch_float16() -) { - in_features <- linear_module$in_features - out_features <- linear_module$out_features - has_bias <- !is.null(linear_module$bias) - - # Create INT4 layer - int4_layer <- int4_linear(in_features, out_features, bias = has_bias, - device = device, dtype = dtype) - - # Quantize and load weight - q <- quantize_int4(linear_module$weight) - int4_layer$load_int4_weight(q) - - # Copy bias if present (use with_no_grad to avoid in-place error on parameter) - if (has_bias) { - torch::with_no_grad({ - int4_layer$bias$copy_(linear_module$bias$to(dtype = dtype, device = device)) - }) - } - - int4_layer -} - -#' Create INT4 Linear from Pre-quantized Weights -#' -#' Creates an INT4 linear layer from pre-quantized weight data. -#' -#' @param q_weight List with packed, scales, orig_shape from load_int4_weights(). -#' @param q_bias Optional. Quantized bias (or NULL for no bias). -#' @param bias_tensor Optional. Float tensor for bias (if not quantized). -#' @param device Character. Target device. -#' @param dtype torch_dtype. Target dtype for operations. -#' -#' @return int4_linear module with loaded weights. -#' -#' @export -int4_linear_from_quantized <- function( - q_weight, - q_bias = NULL, - bias_tensor = NULL, - device = "cuda", - dtype = torch::torch_float16() -) { - out_features <- q_weight$orig_shape[1] - in_features <- q_weight$orig_shape[2] - has_bias <- !is.null(q_bias) || !is.null(bias_tensor) - - # Create INT4 layer - int4_layer <- int4_linear(in_features, out_features, bias = has_bias, - device = device, dtype = dtype) - - # Load quantized weight - int4_layer$load_int4_weight(q_weight) - - # Load bias if present (use with_no_grad to avoid in-place error on parameter) - if (!is.null(bias_tensor)) { - torch::with_no_grad({ - int4_layer$bias$copy_(bias_tensor$to(dtype = dtype, device = device)) - }) - } else if (!is.null(q_bias)) { - # Dequantize bias - bias_dequant <- dequantize_int4(q_bias, dtype = dtype, device = device) - torch::with_no_grad({ - int4_layer$bias$copy_(bias_dequant) - }) - } - - int4_layer -} - -#' Load INT4 Weights into Model -#' -#' Replaces linear layers in a model with INT4 versions and loads quantized weights. -#' This is the main entry point for running large models with INT4 quantization. -#' -#' @param model nn_module. The model to convert. -#' @param int4_weights List of quantized weights from `load_int4_weights()`. -#' @param device Character. Target device for INT4 weights. -#' @param dtype torch_dtype. Target dtype for dequantized operations. -#' @param verbose Logical. Print progress. -#' -#' @return The model with linear layers replaced by INT4 versions. -#' -#' @details -#' This function: -#' 1. Identifies linear layers by matching parameter names ending in ".weight" -#' 2. Creates INT4Linear layers with matching dimensions -#' 3. Loads quantized weights and biases -#' 4. Replaces the original layers in the model -#' -#' The INT4 weights stay compressed on GPU (~10GB for a 40GB model). -#' During forward(), each layer dequantizes on-the-fly, keeping memory usage low. -#' -#' @export -load_int4_into_model <- function( - model, - int4_weights, - device = "cuda", - dtype = torch::torch_float16(), - verbose = TRUE -) { - # Get all module names that have .weight in the quantized weights - weight_names <- grep("\\.weight$", names(int4_weights), value = TRUE) - - if (verbose) { - message(sprintf("Loading %d INT4 weights into model...", length(weight_names))) - } - - loaded <- 0 - skipped <- 0 - - for (weight_name in weight_names) { - # Extract module path (e.g., "transformer_blocks.0.attn1.to_q") - module_path <- sub("\\.weight$", "", weight_name) - bias_name <- paste0(module_path, ".bias") - - q_weight <- int4_weights[[weight_name]] - if (bias_name %in% names(int4_weights)) { - q_bias <- int4_weights[[bias_name]] - } else { - q_bias <- NULL - } - - # Check dimensions - only process 2D weights (linear layers) - if (length(q_weight$orig_shape) != 2) { - skipped <- skipped + 1 - next - } - - # Create INT4 layer - out_features <- q_weight$orig_shape[1] - in_features <- q_weight$orig_shape[2] - has_bias <- !is.null(q_bias) - - int4_layer <- int4_linear(in_features, out_features, bias = has_bias, - device = device, dtype = dtype) - int4_layer$load_int4_weight(q_weight) - - # Load bias if present - if (has_bias) { - bias_dequant <- dequantize_int4(q_bias, dtype = dtype, device = device) - torch::with_no_grad({ - int4_layer$bias$copy_(bias_dequant) - }) - } - - # Store the INT4 layer for later assignment - # Note: Direct module replacement in R torch is complex - # For now, store in a separate list that can be used during forward - if (!exists("int4_layers", where = model)) { - model$int4_layers <- list() - } - model$int4_layers[[module_path]] <- int4_layer - - loaded <- loaded + 1 - } - - if (verbose) { - message(sprintf("Loaded %d INT4 layers, skipped %d non-linear", loaded, skipped)) - } - - invisible(model) -} - -#' Load INT4 Weights into INT4 Model -#' -#' Loads pre-quantized INT4 weights into a model created with `make_linear()` -#' when `diffuseR.use_int4 = TRUE`. -#' -#' @param model nn_module created with INT4 layers. -#' @param int4_weights List from `load_int4_weights()`. -#' @param verbose Logical. Print progress. -#' -#' @return Model with INT4 weights loaded (invisibly). -#' -#' @export -load_int4_weights_into_model <- function( - model, - int4_weights, - verbose = TRUE -) { - # Get model's named modules (flattened) - params <- model$parameters - param_names <- names(params) - - loaded <- 0 - skipped <- 0 - - # Name mapping from HuggingFace to R model structure - # FFN layers have different naming: - # HF: ff.net.0.proj, ff.net.2 - # R: ff.act_fn.proj, ff.proj_out - map_hf_to_r_name <- function(hf_name) { - r_name <- hf_name - # Map FFN layer names - r_name <- gsub("\\.ff\\.net\\.0\\.proj\\.", ".ff.act_fn.proj.", r_name) - r_name <- gsub("\\.ff\\.net\\.2\\.", ".ff.proj_out.", r_name) - r_name <- gsub("\\.audio_ff\\.net\\.0\\.proj\\.", ".audio_ff.act_fn.proj.", r_name) - r_name <- gsub("\\.audio_ff\\.net\\.2\\.", ".audio_ff.proj_out.", r_name) - # Handle end-of-string cases - r_name <- gsub("\\.ff\\.net\\.0\\.proj$", ".ff.act_fn.proj", r_name) - r_name <- gsub("\\.ff\\.net\\.2$", ".ff.proj_out", r_name) - r_name <- gsub("\\.audio_ff\\.net\\.0\\.proj$", ".audio_ff.act_fn.proj", r_name) - r_name <- gsub("\\.audio_ff\\.net\\.2$", ".audio_ff.proj_out", r_name) - r_name - } - - for (int4_name in names(int4_weights)) { - # Map HuggingFace name to R model name - r_name <- map_hf_to_r_name(int4_name) - # Check if this weight (with mapped name) exists in model - if (r_name %in% param_names) { - # This is a regular parameter (bias, norm weights, etc.) - # Dequantize and copy - q <- int4_weights[[int4_name]] - if (length(q$orig_shape) == 1) { - # 1D tensor (bias, norm) - dequantize to model's device/dtype - param <- params[[r_name]] - dequant <- dequantize_int4(q, dtype = param$dtype, device = as.character(param$device)) - torch::with_no_grad({ - param$copy_(dequant) - }) - loaded <- loaded + 1 - } - } else if (grepl("\\.weight$", r_name)) { - # This might be an INT4 linear weight - find the layer - # Weight name: "module.path.weight" -> layer path: "module.path" - layer_path <- sub("\\.weight$", "", r_name) - - # Try to find corresponding INT4 layer by navigating module tree - tryCatch({ - # Navigate to the layer using R model path - parts <- strsplit(layer_path, "\\.") [[1]] - current <- model - - for (part in parts) { - if (grepl("^[0-9]+$", part)) { - # Numeric index (0-based in Python, 1-based in R) - idx <- as.integer(part) + 1L - current <- current[[idx]] - } else { - current <- current[[part]] - } - } - - # Check if this is an INT4Linear layer (has load_int4_weight method) - if (!is.null(current$load_int4_weight)) { - # Load INT4 weight directly (using original name for data access) - q <- int4_weights[[int4_name]] - current$load_int4_weight(q) - loaded <- loaded + 1 - } else { - skipped <- skipped + 1 - } - }, error = function(e) { - skipped <<- skipped + 1 - }) - } else { - skipped <- skipped + 1 - } - } - - if (verbose) { - message(sprintf("Loaded %d weights, skipped %d", loaded, skipped)) - } - - invisible(model) -} - -#' Quantize Model Weights to INT4 -#' -#' Quantizes all parameters in a torch module to INT4 format. -#' -#' @param module nn_module. The model to quantize. -#' @param block_size Integer. Block size for quantization. -#' @param verbose Logical. Print progress. -#' -#' @return List of quantized parameters (does not modify original module). -#' -#' @export -quantize_model_int4 <- function( - module, - block_size = 64L, - verbose = TRUE -) { - params <- module$parameters - quantized <- list() - - total_orig <- 0 - total_quant <- 0 - - for (name in names(params)) { - p <- params[[name]] - orig_bytes <- prod(p$shape) * 2# Assume float16 - - q <- quantize_int4(p, block_size = block_size) - quant_bytes <- q$packed$shape[1] + prod(q$scales$shape) * 4 - - quantized[[name]] <- q - total_orig <- total_orig + orig_bytes - total_quant <- total_quant + quant_bytes - - if (verbose) { - message(sprintf(" %s: %.2f MB -> %.2f MB", - name, orig_bytes / 1e6, quant_bytes / 1e6)) - } - } - - if (verbose) { - message(sprintf("Total: %.2f MB -> %.2f MB (%.1fx compression)", - total_orig / 1e6, total_quant / 1e6, total_orig / total_quant)) - } - - quantized -} - -#' Save INT4 Quantized Weights -#' -#' Saves INT4 quantized weights to disk as sharded safetensors files. -#' -#' @param quantized_params List of quantized parameters from `quantize_model_int4()`. -#' @param path Character. Base path for safetensors files. If multiple shards needed, -#' files will be named `path-00001-of-NNNNN.safetensors`. -#' @param max_shard_size Numeric. Maximum bytes per shard (default 2GB to avoid R -#' integer overflow issues). -#' @param verbose Logical. Print progress. -#' -#' @return Invisible character vector of saved file paths. -#' -#' @details -#' Weights are saved in safetensors format with the following structure: -#' \itemize{ -#' \item `{name}::packed` - uint8 tensor with packed INT4 values -#' \item `{name}::scales` - float32 tensor with per-block scales -#' \item `{name}::shape` - int64 tensor with original shape -#' } -#' -#' Large models are automatically sharded to avoid R's 2GB vector limit. -#' The block size is fixed at 64 (standard for INT4 quantization). -#' -#' @export -#' -#' @examples -#' \dontrun{ -#' q <- quantize_model_int4(model) -#' save_int4_weights(q, "model_int4.safetensors") -#' } -save_int4_weights <- function( - quantized_params, - path, - max_shard_size = 2e9, - verbose = TRUE -) { - if (verbose) message(sprintf("Preparing %d parameters for safetensors...", length(quantized_params))) - - # Calculate total size and estimate number of shards - total_bytes <- 0 - param_sizes <- list() - for (name in names(quantized_params)) { - q <- quantized_params[[name]] - size <- prod(q$packed$shape) + prod(q$scales$shape) * 4 + length(q$orig_shape) * 8 - param_sizes[[name]] <- size - total_bytes <- total_bytes + size - } - - n_shards <- max(1L, ceiling(total_bytes / max_shard_size)) - - if (n_shards == 1) { - # Single file - use original path - tensors <- list() - for (name in names(quantized_params)) { - q <- quantized_params[[name]] - tensors[[paste0(name, "::packed")]] <- q$packed$cpu() - tensors[[paste0(name, "::scales")]] <- q$scales$cpu() - tensors[[paste0(name, "::shape")]] <- torch::torch_tensor(q$orig_shape, dtype = torch::torch_int64()) - } - if (verbose) message(sprintf("Saving %d tensors to %s...", length(tensors), path)) - safetensors::safe_save_file(tensors, path) - file_size <- file.info(path)$size / 1e6 - if (verbose) message(sprintf("Saved %.2f MB", file_size)) - return(invisible(path)) - } - - # Multiple shards - split params across files - if (verbose) message(sprintf("Sharding into %d files (max %.1f GB each)...", n_shards, max_shard_size / 1e9)) - - # Remove extension for base path - base_path <- sub("\\.safetensors$", "", path) - param_names <- names(quantized_params) - params_per_shard <- ceiling(length(param_names) / n_shards) - saved_paths <- character(0) - - for (shard_idx in seq_len(n_shards)) { - start_idx <- (shard_idx - 1) * params_per_shard + 1 - end_idx <- min(shard_idx * params_per_shard, length(param_names)) - - if (start_idx > length(param_names)) break - - shard_names <- param_names[start_idx:end_idx] - tensors <- list() - - for (name in shard_names) { - q <- quantized_params[[name]] - tensors[[paste0(name, "::packed")]] <- q$packed$cpu() - tensors[[paste0(name, "::scales")]] <- q$scales$cpu() - tensors[[paste0(name, "::shape")]] <- torch::torch_tensor(q$orig_shape, dtype = torch::torch_int64()) - } - - shard_path <- sprintf("%s-%05d-of-%05d.safetensors", base_path, shard_idx, n_shards) - if (verbose) message(sprintf(" [%d/%d] Saving %d params to %s...", - shard_idx, n_shards, length(shard_names), basename(shard_path))) - safetensors::safe_save_file(tensors, shard_path) - saved_paths <- c(saved_paths, shard_path) - } - - total_size <- sum(file.info(saved_paths)$size) / 1e6 - if (verbose) message(sprintf("Total: %.2f MB across %d shards", total_size, length(saved_paths))) - - invisible(saved_paths) -} - -#' Load INT4 Quantized Weights -#' -#' Loads INT4 quantized weights from safetensors file(s). -#' -#' @param path Character. Path to safetensors file or base path for sharded files. -#' For sharded files, pass the base path (e.g., "model_int4.safetensors") and -#' the function will find all shards matching the pattern. -#' @param verbose Logical. Print progress. -#' -#' @return List of quantized parameter structures ready for `dequantize_int4()`. -#' -#' @export -#' -#' @examples -#' \dontrun{ -#' q <- load_int4_weights("model_int4.safetensors") -#' # Dequantize specific parameter on GPU -#' weight <- dequantize_int4(q[["linear.weight"]], device = "cuda") -#' } -load_int4_weights <- function( - path, - verbose = TRUE -) { - path <- path.expand(path) - - # Check for sharded files - base_path <- sub("\\.safetensors$", "", path) - shard_pattern <- sprintf("%s-[0-9]+-of-[0-9]+\\.safetensors$", basename(base_path)) - shard_dir <- dirname(path) - shard_files <- list.files(shard_dir, pattern = shard_pattern, full.names = TRUE) - - if (length(shard_files) > 0) { - # Load sharded files - shard_files <- sort(shard_files) - if (verbose) { - total_size <- sum(file.info(shard_files)$size) / 1e6 - message(sprintf("Loading INT4 weights from %d shards (%.2f MB total)...", - length(shard_files), total_size)) - } - paths <- shard_files - } else if (file.exists(path)) { - # Single file - if (verbose) { - size_mb <- file.info(path)$size / 1e6 - message(sprintf("Loading INT4 weights from %s (%.2f MB)...", path, size_mb)) - } - paths <- path - } else { - stop("File not found: ", path) - } - - # Load all files - quantized <- list() - for (i in seq_along(paths)) { - p <- paths[i] - if (verbose && length(paths) > 1) { - message(sprintf(" [%d/%d] Loading %s...", i, length(paths), basename(p))) - } - - tensors <- safetensors::safe_load_file(p, framework = "torch") - - # Parse tensor names to reconstruct parameter structures - packed_names <- grep("::packed$", names(tensors), value = TRUE) - param_names <- sub("::packed$", "", packed_names) - - for (name in param_names) { - packed <- tensors[[paste0(name, "::packed")]] - scales <- tensors[[paste0(name, "::scales")]] - shape_tensor <- tensors[[paste0(name, "::shape")]] - orig_shape <- as.integer(as.array(shape_tensor)) - orig_numel <- prod(orig_shape) - - quantized[[name]] <- list( - packed = packed, - scales = scales, - orig_shape = orig_shape, - orig_numel = orig_numel, - block_size = 64L# Standard block size - ) - } - } - - if (verbose) message(sprintf("Done. Loaded %d parameters.", length(quantized))) - quantized -} - -#' Quantize Safetensor Weights to INT4 -#' -#' Loads weights from safetensors file(s) and quantizes to INT4. -#' Useful for quantizing large models without loading the full module. -#' -#' @param paths Character vector. Paths to safetensor files. -#' @param output_path Character. Path to save INT4 weights (.safetensors). -#' @param block_size Integer. Block size for quantization (default 64). -#' @param verbose Logical. Print progress. -#' -#' @return Invisible NULL. Writes quantized weights to output_path. -#' -#' @export -#' -#' @examples -#' \dontrun{ -#' # Quantize sharded transformer weights -#' paths <- list.files("~/.cache/huggingface/.../transformer", -#' pattern = "safetensors$", full.names = TRUE) -#' quantize_safetensors_int4(paths, "dit_int4.safetensors") -#' } -quantize_safetensors_int4 <- function( - paths, - output_path, - block_size = 64L, - verbose = TRUE -) { - all_quantized <- list() - total_orig <- 0 - total_quant <- 0 - - for (path in paths) { - if (verbose) message(sprintf("Loading %s...", basename(path))) - - weights <- safetensors::safe_load_file(path, framework = "torch") - - for (name in names(weights)) { - w <- weights[[name]] - orig_bytes <- prod(w$shape) * 2# Assume float16 - - q <- quantize_int4(w, block_size = block_size) - quant_bytes <- length(as.array(q$packed)) + prod(q$scales$shape) * 4 - - all_quantized[[name]] <- q - total_orig <- total_orig + orig_bytes - total_quant <- total_quant + quant_bytes - - if (verbose && prod(w$shape) > 1e6) { - message(sprintf(" %s: %.2f MB -> %.2f MB", - name, orig_bytes / 1e6, quant_bytes / 1e6)) - } - } - - # Clear memory between shards - rm(weights) - gc() - } - - if (verbose) { - message(sprintf("\nTotal: %.2f GB -> %.2f GB (%.1fx compression)", - total_orig / 1e9, total_quant / 1e9, total_orig / total_quant)) - } - - save_int4_weights(all_quantized, output_path, verbose = verbose) -} - -#' Quantize LTX-2 Transformer to INT4 -#' -#' Downloads (if needed) and quantizes the LTX-2 19B transformer to INT4 format. -#' The quantized weights are cached for future use. -#' -#' @param model_id Character. HuggingFace model ID (default "Lightricks/LTX-2"). -#' @param output_dir Character. Directory to save quantized weights. -#' Default uses `tools::R_user_dir("diffuseR", "cache")`. -#' @param block_size Integer. Block size for INT4 quantization (default 64). -#' @param force Logical. Re-quantize even if cached file exists. -#' @param download Logical. If TRUE, download model from HuggingFace if not cached. -#' @param verbose Logical. Print progress. -#' -#' @return Character. Path to the quantized weights file. -#' -#' @details -#' LTX-2 is a 19B parameter model (~40GB in BF16). INT4 quantization reduces -#' this to ~5.7GB, fitting in 16GB VRAM with room for activations. -#' -#' The function: -#' 1. Uses hfhub to locate/download the model from HuggingFace -#' 2. Loads each safetensor shard -#' 3. Quantizes all weights to INT4 (block-wise, 64 values per scale) -#' 4. Saves as safetensors file -#' -#' @export -#' -#' @examples -#' \dontrun{ -#' # Quantize and cache (first run takes ~10-20 minutes) -#' weights_path <- quantize_ltx2_transformer() -#' -#' # Load quantized weights -#' q <- load_int4_weights(weights_path) -#' -#' # Dequantize specific layer on GPU -#' layer_weight <- dequantize_int4(q[["transformer_blocks.0.attn1.to_q.weight"]], -#' device = "cuda", dtype = torch_float16()) -#' } -quantize_ltx2_transformer <- function( - model_id = "Lightricks/LTX-2", - output_dir = NULL, - block_size = 64L, - force = FALSE, - download = FALSE, - verbose = TRUE -) { - if (!requireNamespace("hfhub", quietly = TRUE)) { - stop("Package 'hfhub' is required. Install with: install.packages('hfhub')") - } - - # Use R_user_dir for CRAN-compliant cache - - if (is.null(output_dir)) { - output_dir <- tools::R_user_dir("diffuseR", "cache") - } - - output_file <- file.path(output_dir, "ltx2_transformer_int4.safetensors") - - # Check if already cached - if (file.exists(output_file) && !force) { - if (verbose) { - size_gb <- file.info(output_file)$size / 1e9 - message(sprintf("Using cached INT4 weights: %s (%.2f GB)", output_file, size_gb)) - } - return(output_file) - } - - # Check if model is available locally via transformer/config.json - transformer_dir <- tryCatch({ - config_path <- hfhub::hub_download(model_id, "transformer/config.json", - local_files_only = TRUE) - dirname(config_path) - }, error = function(e) NULL) - - if (is.null(transformer_dir)) { - if (!download) { - stop("Model '", model_id, "' transformer not found in HuggingFace cache.\n", - "Run with download = TRUE to download, or use:\n", - " huggingface-cli download ", model_id) - } - - # Interactive consent before downloading - if (interactive()) { - ans <- utils::askYesNo( - paste0("Download '", model_id, "' transformer (~40GB) from HuggingFace?"), - default = TRUE - ) - if (!isTRUE(ans)) { - stop("Download cancelled.", call. = FALSE) - } - } - - if (verbose) message("Downloading transformer weights from HuggingFace...") - model_path <- hfhub::hub_snapshot(model_id, - allow_patterns = "transformer/*") - transformer_dir <- file.path(model_path, "transformer") - } - if (!dir.exists(transformer_dir)) { - stop("Transformer directory not found: ", transformer_dir) - } - - safetensor_files <- list.files(transformer_dir, pattern = "\\.safetensors$", - full.names = TRUE) - if (length(safetensor_files) == 0) { - stop("No safetensor files found in: ", transformer_dir) - } - - if (verbose) { - message(sprintf("Found %d safetensor files in: %s", length(safetensor_files), transformer_dir)) - total_size <- sum(file.info(safetensor_files)$size) / 1e9 - message(sprintf("Total size: %.2f GB (will compress to ~%.2f GB)", - total_size, total_size / 7)) - } - - # Create output directory only when actually writing (CRAN policy) - dir.create(output_dir, showWarnings = FALSE, recursive = TRUE) - - # Quantize - quantize_safetensors_int4( - paths = safetensor_files, - output_path = output_file, - block_size = block_size, - verbose = verbose - ) - - output_file -} - -#' Quantize LTX-2 VAE to INT4 -#' -#' Quantizes the LTX-2 video VAE to INT4 format. -#' -#' @param model_id Character. HuggingFace model ID (default "Lightricks/LTX-2"). -#' @param output_dir Character. Directory to save quantized weights. -#' Default uses `tools::R_user_dir("diffuseR", "cache")`. -#' @param block_size Integer. Block size for INT4 quantization. -#' @param force Logical. Re-quantize even if cached file exists. -#' @param download Logical. If TRUE, download model from HuggingFace if not cached. -#' @param verbose Logical. Print progress. -#' -#' @return Character. Path to the quantized weights file. -#' -#' @export -quantize_ltx2_vae <- function( - model_id = "Lightricks/LTX-2", - output_dir = NULL, - block_size = 64L, - force = FALSE, - download = FALSE, - verbose = TRUE -) { - if (!requireNamespace("hfhub", quietly = TRUE)) { - stop("Package 'hfhub' is required. Install with: install.packages('hfhub')") - } - - # Use R_user_dir for CRAN-compliant cache - - if (is.null(output_dir)) { - output_dir <- tools::R_user_dir("diffuseR", "cache") - } - - output_file <- file.path(output_dir, "ltx2_vae_int4.safetensors") - - # Check if already cached - if (file.exists(output_file) && !force) { - if (verbose) { - size_mb <- file.info(output_file)$size / 1e6 - message(sprintf("Using cached INT4 VAE weights: %s (%.2f MB)", output_file, size_mb)) - } - return(output_file) - } - - # Check if model is available locally via vae/config.json - vae_dir <- tryCatch({ - config_path <- hfhub::hub_download(model_id, "vae/config.json", - local_files_only = TRUE) - dirname(config_path) - }, error = function(e) NULL) - - if (is.null(vae_dir)) { - if (!download) { - stop("Model '", model_id, "' VAE not found in HuggingFace cache.\n", - "Run with download = TRUE to download, or use:\n", - " huggingface-cli download ", model_id) - } - - # Interactive consent before downloading - if (interactive()) { - ans <- utils::askYesNo( - paste0("Download '", model_id, "' VAE from HuggingFace?"), - default = TRUE - ) - if (!isTRUE(ans)) { - stop("Download cancelled.", call. = FALSE) - } - } - - if (verbose) message("Downloading VAE weights from HuggingFace...") - model_path <- hfhub::hub_snapshot(model_id, - allow_patterns = "vae/*") - vae_dir <- file.path(model_path, "vae") - } - if (!dir.exists(vae_dir)) { - stop("VAE directory not found: ", vae_dir) - } - - safetensor_files <- list.files(vae_dir, pattern = "\\.safetensors$", - full.names = TRUE) - if (length(safetensor_files) == 0) { - stop("No safetensor files found in: ", vae_dir) - } - - if (verbose) { - total_size <- sum(file.info(safetensor_files)$size) / 1e6 - message(sprintf("Found %d VAE safetensor files (%.2f MB)", - length(safetensor_files), total_size)) - } - - # Create output directory only when actually writing (CRAN policy) - dir.create(output_dir, showWarnings = FALSE, recursive = TRUE) - - # Quantize - quantize_safetensors_int4( - paths = safetensor_files, - output_path = output_file, - block_size = block_size, - verbose = verbose - ) - - output_file -} - diff --git a/R/memory_sdxl.R b/R/memory_sdxl.R new file mode 100644 index 0000000..82da915 --- /dev/null +++ b/R/memory_sdxl.R @@ -0,0 +1,118 @@ +#' Get SDXL Memory Profile +#' +#' Determines optimal memory configuration for SDXL image generation +#' based on available VRAM. +#' +#' @param vram_gb Numeric. Available VRAM in GB, or NULL for auto-detection. +#' +#' @return A list with memory profile settings. +#' +#' @details +#' Memory profiles for SDXL: +#' \describe{ +#' \item{full_gpu}{16GB+ - All components on CUDA} +#' \item{balanced}{10-12GB - UNet + decoder on CUDA, text encoders on CPU} +#' \item{unet_gpu}{6-10GB - Only UNet on CUDA, everything else CPU} +#' \item{cpu_only}{<6GB - All on CPU} +#' } +#' +#' Each profile also specifies: +#' - cfg_mode: "batched" or "sequential" (sequential halves peak memory) +#' - cleanup: "none", "phase", or "step" (when to clear VRAM) +#' - dtype: "float16" or "float32" +#' - max_resolution: maximum image dimension +#' +#' @export +#' +#' @examples +#' \dontrun{ +#' # Auto-detect profile +#' profile <- sdxl_memory_profile() +#' +#' # Specific VRAM +#' profile <- sdxl_memory_profile(vram_gb = 8) +#' } +sdxl_memory_profile <- function(vram_gb = NULL) { + # Auto-detect free VRAM if not provided + if (is.null(vram_gb)) { + vram_gb <- .detect_vram(use_free = TRUE) + message(sprintf("Detected %.1f GB free VRAM", vram_gb)) + } + + # Determine profile level + if (vram_gb >= 16) { + profile <- "full_gpu" + } else if (vram_gb >= 10) { + profile <- "balanced" + } else if (vram_gb >= 6) { + profile <- "unet_gpu" + } else { + profile <- "cpu_only" + } + + # Build profile config + profiles <- list( + full_gpu = list( + name = "full_gpu", + devices = list( + unet = "cuda", + decoder = "cuda", + text_encoder = "cuda", + text_encoder2 = "cuda", + encoder = "cuda" + ), + dtype = "float16", + cfg_mode = "batched", + cleanup = "none", + max_resolution = 1536L, + step_cleanup_interval = 0L# No step cleanup + ), + balanced = list( + name = "balanced", + devices = list( + unet = "cuda", + decoder = "cuda", + text_encoder = "cpu", + text_encoder2 = "cpu", + encoder = "cpu" + ), + dtype = "float16", + cfg_mode = "batched", + cleanup = "phase", # Cleanup between text encoding and denoising + max_resolution = 1024L, + step_cleanup_interval = 0L + ), + unet_gpu = list( + name = "unet_gpu", + devices = list( + unet = "cuda", + decoder = "cpu", + text_encoder = "cpu", + text_encoder2 = "cpu", + encoder = "cpu" + ), + dtype = "float16", + cfg_mode = "sequential", # Sequential CFG halves peak memory + cleanup = "phase", + max_resolution = 1024L, + step_cleanup_interval = 10L# Cleanup every 10 steps + ), + cpu_only = list( + name = "cpu_only", + devices = list( + unet = "cpu", + decoder = "cpu", + text_encoder = "cpu", + text_encoder2 = "cpu", + encoder = "cpu" + ), + dtype = "float32", # CPU often faster with float32 + cfg_mode = "sequential", + cleanup = "none", # No GPU to clean + max_resolution = 768L, + step_cleanup_interval = 0L + ) + ) + + profiles[[profile]] +} diff --git a/R/rope.R b/R/rope.R deleted file mode 100644 index 925ecf9..0000000 --- a/R/rope.R +++ /dev/null @@ -1,360 +0,0 @@ -#' Rotary Position Embeddings (RoPE) for LTX-2 Video Models -#' -#' Implementation of rotary positional embeddings for 3D video (spatiotemporal) -#' coordinates as used in LTX-2. RoPE encodes position information directly -#' into the attention queries and keys without adding position embeddings. -#' -#' @name rope -#' @references -#' Su, J., Lu, Y., Pan, S., Murtadha, A., Wen, B., & Liu, Y. (2021). -#' "RoFormer: Enhanced Transformer with Rotary Position Embedding." -#' \url{https://arxiv.org/abs/2104.09864} -NULL - -#' Create RoPE position embedder for video -#' -#' Creates a RoPE embedder configured for video generation, handling -#' spatiotemporal coordinates (frames, height, width). -#' -#' @param dim Integer. Dimension for RoPE (typically attention head dim * num_heads). -#' @param patch_size Integer. Spatial patch size. Default: 1 -#' @param patch_size_t Integer. Temporal patch size. Default: 1 -#' @param base_num_frames Integer. Base number of frames for normalization. Default: 20 -#' @param base_height Integer. Base height for normalization. Default: 2048 -#' @param base_width Integer. Base width for normalization. Default: 2048 -#' @param scale_factors Numeric vector of length 3. VAE scale factors for -#' (temporal, height, width). Default: c(8, 32, 32) -#' @param theta Numeric. Base frequency for RoPE. Default: 10000.0 -#' @param causal_offset Integer. Offset for causal VAE modeling. Default: 1 -#' @param double_precision Logical. Whether to use float64 for frequency -#' computation. Default: TRUE -#' @param rope_type Character. Type of RoPE: "interleaved" or "split". Default: "interleaved" -#' @param num_attention_heads Integer. Number of attention heads (for split RoPE). Default: 32 -#' -#' @return A list containing RoPE configuration and methods. -#' @export -rope_embedder_create <- function( - dim, - patch_size = 1L, - patch_size_t = 1L, - base_num_frames = 20L, - base_height = 2048L, - base_width = 2048L, - scale_factors = c(8, 32, 32), - theta = 10000.0, - causal_offset = 1L, - double_precision = TRUE, - rope_type = c("interleaved", "split"), - num_attention_heads = 32L -) { - rope_type <- match.arg(rope_type) - - list( - dim = dim, - patch_size = patch_size, - patch_size_t = patch_size_t, - base_num_frames = base_num_frames, - base_height = base_height, - base_width = base_width, - scale_factors = scale_factors, - theta = theta, - causal_offset = causal_offset, - double_precision = double_precision, - rope_type = rope_type, - num_attention_heads = num_attention_heads - ) -} - -#' Prepare video coordinates for RoPE -#' -#' Creates per-dimension patch boundaries for video coordinates in pixel space. -#' Returns tensor of shape (batch_size, 3, num_patches, 2) where dimension 1 -#' represents (frame, height, width) and dimension 3 represents (start, end). -#' -#' @param embedder List. RoPE embedder configuration. -#' @param batch_size Integer. Batch size. -#' @param num_frames Integer. Number of latent frames. -#' @param height Integer. Latent height. -#' @param width Integer. Latent width. -#' @param device Character or torch device. Device for tensors. -#' @param fps Numeric. Video frames per second. Default: 24.0 -#' -#' @return torch tensor of shape (batch_size, 3, num_patches, 2). -#' @export -rope_prepare_video_coords <- function( - embedder, - batch_size, - num_frames, - height, - width, - device = "cpu", - fps = 24.0 -) { - patch_size <- embedder$patch_size - patch_size_t <- embedder$patch_size_t - scale_factors <- embedder$scale_factors - causal_offset <- embedder$causal_offset - - # 1. Generate grid coordinates for each spatiotemporal dimension - # Note: R's torch_arange is inclusive, so use end-step to match Python's exclusive behavior - grid_f <- torch::torch_arange( - start = 0, end = num_frames - patch_size_t, step = patch_size_t, - dtype = torch::torch_float32(), device = device - ) - grid_h <- torch::torch_arange( - start = 0, end = height - patch_size, step = patch_size, - dtype = torch::torch_float32(), device = device - ) - grid_w <- torch::torch_arange( - start = 0, end = width - patch_size, step = patch_size, - dtype = torch::torch_float32(), device = device - ) - - # Create meshgrid (ij indexing keeps order as frames, height, width) - grid <- torch::torch_meshgrid(list(grid_f, grid_h, grid_w), indexing = "ij") - grid <- torch::torch_stack(grid, dim = 1) # [3, N_F, N_H, N_W] - - # 2. Get patch boundaries with respect to latent grid - patch_size_vec <- c(patch_size_t, patch_size, patch_size) - patch_size_delta <- torch::torch_tensor( - patch_size_vec, dtype = torch::torch_float32(), device = device - )$view(c(3, 1, 1, 1)) - - patch_ends <- grid + patch_size_delta - - # Combine start and end coordinates [3, N_F, N_H, N_W, 2] - latent_coords <- torch::torch_stack(list(grid, patch_ends), dim = - 1) - - # Reshape to (1, 3, num_patches, 2) - latent_coords <- torch::torch_flatten(latent_coords, start_dim = 2, end_dim = 4) # [3, num_patches, 2] - latent_coords <- latent_coords$unsqueeze(1) # [1, 3, num_patches, 2] - latent_coords <- latent_coords$`repeat`(c(batch_size, 1, 1, 1)) - - # 3. Convert to pixel space using VAE scale factors - scale_tensor <- torch::torch_tensor( - scale_factors, dtype = torch::torch_float32(), device = device - )$view(c(1, 3, 1, 1)) - - pixel_coords <- latent_coords * scale_tensor - - # Handle causal offset for first frame - # Frame coordinates need special handling for causal VAE - frame_coords <- pixel_coords[, 1,,] - frame_coords <- (frame_coords + causal_offset - scale_factors[1])$clamp(min = 0) - pixel_coords[, 1,,] <- frame_coords - - # Scale temporal coordinates by FPS (convert to seconds) - pixel_coords[, 1,,] <- pixel_coords[, 1,,] / fps - - pixel_coords -} - -#' Compute RoPE frequencies from coordinates -#' -#' Converts spatiotemporal coordinates to (cos, sin) frequency tensors -#' for applying rotary embeddings. -#' -#' @param embedder List. RoPE embedder configuration. -#' @param coords torch tensor. Coordinate tensor from rope_prepare_video_coords(). -#' @param device Character or torch device. Device for output tensors. -#' -#' @return A list with: -#' \describe{ -#' \item{cos_freqs}{Cosine frequencies tensor} -#' \item{sin_freqs}{Sine frequencies tensor} -#' } -#' @export -rope_forward <- function( - embedder, - coords, - device = NULL -) { - if (is.null(device)) { - device <- coords$device - } - - dim <- embedder$dim - theta <- embedder$theta - base_num_frames <- embedder$base_num_frames - base_height <- embedder$base_height - base_width <- embedder$base_width - double_precision <- embedder$double_precision - rope_type <- embedder$rope_type - num_attention_heads <- embedder$num_attention_heads - - # Number of spatiotemporal dimensions (3 for video) - num_pos_dims <- as.numeric(coords$shape[2]) - - # 1. If coords are patch boundaries [start, end), use midpoint - if (length(coords$shape) == 4) { - # Split into start and end - coords_split <- coords$chunk(2, dim = - 1) - coords_start <- coords_split[[1]] - coords_end <- coords_split[[2]] - coords <- (coords_start + coords_end) / 2.0 - coords <- coords$squeeze(- 1) # [B, num_pos_dims, num_patches] - } - - # 2. Get coordinates as fraction of base shape - max_positions <- c(base_num_frames, base_height, base_width) - - # Create grid tensor [B, num_patches, num_pos_dims] - grid_list <- list() - for (i in seq_len(num_pos_dims)) { - grid_list[[i]] <- coords[, i,] / max_positions[i] - } - grid <- torch::torch_stack(grid_list, dim = - 1)$to(device = device) - - # Number of RoPE elements (3 dims * 2 for cos/sin) - num_rope_elems <- num_pos_dims * 2L - - # 3. Create 1D grid of frequencies - if (double_precision) { - freqs_dtype <- torch::torch_float64() - } else { - freqs_dtype <- torch::torch_float32() - } - - pow_indices <- torch::torch_pow( - theta, - torch::torch_linspace( - start = 0.0, end = 1.0, - steps = as.integer(dim %/% num_rope_elems), - dtype = freqs_dtype, device = device - ) - ) - freqs <- (pow_indices * pi / 2.0)$to(dtype = torch::torch_float32()) - - # 4. Compute position-specific frequencies - # [B, num_patches, num_pos_dims, dim // num_rope_elems] - freqs <- (grid$unsqueeze(- 1) * 2 - 1) * freqs - - # Transpose and flatten [B, num_patches, dim // 2] - freqs <- freqs$transpose(- 1, - 2)$flatten(start_dim = 3) - - # 5. Get interleaved (cos, sin) frequencies - if (rope_type == "interleaved") { - cos_freqs <- freqs$cos()$repeat_interleave(2L, dim = - 1L) - sin_freqs <- freqs$sin()$repeat_interleave(2L, dim = - 1L) - - # Handle padding if dim not divisible by num_rope_elems - if (dim %% num_rope_elems != 0) { - pad_size <- dim %% num_rope_elems - cos_padding <- torch::torch_ones_like(cos_freqs[,, 1:pad_size]) - sin_padding <- torch::torch_zeros_like(sin_freqs[,, 1:pad_size]) - cos_freqs <- torch::torch_cat(list(cos_padding, cos_freqs), dim = - 1) - sin_freqs <- torch::torch_cat(list(sin_padding, sin_freqs), dim = - 1) - } - } else if (rope_type == "split") { - expected_freqs <- dim %/% 2L - current_freqs <- as.numeric(freqs$shape[length(freqs$shape)]) - pad_size <- expected_freqs - current_freqs - - cos_freq <- freqs$cos() - sin_freq <- freqs$sin() - - if (pad_size > 0) { - cos_padding <- torch::torch_ones_like(cos_freq[,, 1:pad_size]) - sin_padding <- torch::torch_zeros_like(sin_freq[,, 1:pad_size]) - cos_freq <- torch::torch_cat(list(cos_padding, cos_freq), dim = - 1) - sin_freq <- torch::torch_cat(list(sin_padding, sin_freq), dim = - 1) - } - - # Reshape for multi-head attention - b <- as.numeric(cos_freq$shape[1]) - t <- as.numeric(cos_freq$shape[2]) - - cos_freq <- cos_freq$reshape(c(b, t, num_attention_heads, - 1)) - sin_freq <- sin_freq$reshape(c(b, t, num_attention_heads, - 1)) - - cos_freqs <- cos_freq$swapaxes(2, 3) # (B, H, T, D//2) - sin_freqs <- sin_freq$swapaxes(2, 3) # (B, H, T, D//2) - } - - list(cos_freqs = cos_freqs, sin_freqs = sin_freqs) -} - -#' Apply interleaved rotary embeddings -#' -#' Applies rotary position embeddings to query or key tensors using the -#' interleaved format where real and imaginary components alternate. -#' -#' @param x torch tensor. Query or key tensor of shape (B, S, C). -#' @param freqs List. Contains cos_freqs and sin_freqs from rope_forward(). -#' -#' @return torch tensor. Rotated tensor with same shape as input. -#' @export -apply_interleaved_rotary_emb <- function( - x, - freqs -) { - cos_freqs <- freqs$cos_freqs - sin_freqs <- freqs$sin_freqs - - # Split x into interleaved real/imaginary pairs - # x has shape [B, S, C], unflatten to [B, S, C//2, 2] - x_unflat <- x$unflatten(3, c(- 1, 2)) - x_split <- x_unflat$unbind(- 1) - x_real <- x_split[[1]]# [B, S, C // 2] - x_imag <- x_split[[2]]# [B, S, C // 2] - - # Rotate: stack [-x_imag, x_real] and flatten back - x_rotated <- torch::torch_stack(list(- x_imag, x_real), dim = - 1)$flatten(start_dim = 3) - - # Apply rotation formula: x * cos + rotate(x) * sin - out <- (x$to(dtype = torch::torch_float32()) * cos_freqs + - x_rotated$to(dtype = torch::torch_float32()) * sin_freqs) - out <- out$to(dtype = x$dtype) - - out -} - -#' Apply split rotary embeddings -#' -#' Applies rotary position embeddings to query or key tensors using the -#' split format where first half is real and second half is imaginary. -#' -#' @param x torch tensor. Query or key tensor. -#' @param freqs List. Contains cos_freqs and sin_freqs from rope_forward(). -#' -#' @return torch tensor. Rotated tensor with same shape as input. -#' @export -apply_split_rotary_emb <- function( - x, - freqs -) { - cos_freqs <- freqs$cos_freqs - sin_freqs <- freqs$sin_freqs - - x_dtype <- x$dtype - needs_reshape <- FALSE - - # Handle dimension mismatch - if (length(x$shape) != 4 && length(cos_freqs$shape) == 4) { - b <- as.numeric(x$shape[1]) - cos_shape <- as.numeric(cos_freqs$shape) - h <- cos_shape[2] - t <- cos_shape[3] - x <- x$reshape(c(b, t, h, - 1))$swapaxes(2, 3) - needs_reshape <- TRUE - } - - # Split into real and imaginary halves - half_dim <- as.numeric(x$shape[length(x$shape)]) %/% 2L - x_real <- x[,,, 1:half_dim] - x_imag <- x[,,, (half_dim + 1) :(half_dim * 2)] - - # Rotate: [-x_imag, x_real] - x_rotated <- torch::torch_cat(list(- x_imag, x_real), dim = - 1) - - # Apply rotation - out <- (x$to(dtype = torch::torch_float32()) * cos_freqs + - x_rotated$to(dtype = torch::torch_float32()) * sin_freqs) - - if (needs_reshape) { - out <- out$swapaxes(2, 3)$flatten(start_dim = 2, end_dim = 3) - } - - out$to(dtype = x_dtype) -} - diff --git a/R/text_encoder_ltx2.R b/R/text_encoder_ltx2.R deleted file mode 100644 index 958e26a..0000000 --- a/R/text_encoder_ltx2.R +++ /dev/null @@ -1,714 +0,0 @@ -# LTX2 Text Encoder and Connectors -# -# This file implements: -# 1. LTX2TextConnectors - Transforms text embeddings for video/audio streams -# 2. Text encoder wrapper - Supports pre-computed embeddings and API-based encoding - -# ----------------------------------------------------------------------------- -# 1D RoPE for text connectors -# ----------------------------------------------------------------------------- - -#' 1D Rotary Position Embeddings for LTX2 Text Connectors -#' @keywords internal -ltx2_rotary_pos_embed_1d <- torch::nn_module( - "LTX2RotaryPosEmbed1d", - initialize = function( - dim, - base_seq_len = 4096L, - theta = 10000.0, - double_precision = TRUE, - rope_type = "interleaved", - num_attention_heads = 32L - ) { - self$dim <- dim - self$base_seq_len <- base_seq_len - self$theta <- theta - self$double_precision <- double_precision - self$rope_type <- rope_type - self$num_attention_heads <- num_attention_heads - }, - - forward = function( - batch_size, - seq_len, - device - ) { - # 1. Get 1D position ids as fractions of base_seq_len - grid_1d <- torch::torch_arange(start = 0, end = seq_len - 1L, - dtype = torch::torch_float32(), device = device) - grid_1d <- grid_1d / self$base_seq_len - grid <- grid_1d$unsqueeze(1L)$`repeat`(c(batch_size, 1L)) # [batch_size, seq_len] - - # 2. Calculate 1D RoPE frequencies - num_rope_elems <- 2L# 1D * 2 (for cos, sin) - if (self$double_precision) { - freqs_dtype <- torch::torch_float64() - } else { - freqs_dtype <- torch::torch_float32() - } - - pow_indices <- torch::torch_pow( - self$theta, - torch::torch_linspace(start = 0.0, end = 1.0, steps = self$dim %/% num_rope_elems, - dtype = freqs_dtype, device = device) - ) - freqs <- (pow_indices * pi / 2.0)$to(dtype = torch::torch_float32()) - - # 3. Outer product: [batch_size, seq_len] x [dim/2] -> [batch_size, seq_len, dim/2] - freqs_outer <- torch::torch_einsum("bs,d->bsd", list(grid, freqs)) - - # 4. Compute cos and sin - cos_freqs <- torch::torch_cos(freqs_outer) - sin_freqs <- torch::torch_sin(freqs_outer) - - # 5. Interleave or split based on rope_type - if (self$rope_type == "interleaved") { - # Repeat each element: [B, S, D/2] -> [B, S, D] - cos_freqs <- cos_freqs$unsqueeze(- 1L)$`repeat`(c(1L, 1L, 1L, 2L))$flatten(start_dim = 3L) - sin_freqs <- sin_freqs$unsqueeze(- 1L)$`repeat`(c(1L, 1L, 1L, 2L))$flatten(start_dim = 3L) - } else { - # Concatenate: [B, S, D/2] -> [B, S, D] - cos_freqs <- torch::torch_cat(list(cos_freqs, cos_freqs), dim = - 1L) - sin_freqs <- torch::torch_cat(list(sin_freqs, sin_freqs), dim = - 1L) - } - - list(cos_freqs, sin_freqs) - } -) - -# ----------------------------------------------------------------------------- -# 1D Transformer Block for Connectors -# ----------------------------------------------------------------------------- - -#' 1D Transformer Block for LTX2 Text Connectors -#' @keywords internal -ltx2_transformer_block_1d <- torch::nn_module( - "LTX2TransformerBlock1d", - initialize = function( - dim, - num_attention_heads, - attention_head_dim, - activation_fn = "gelu-approximate", - eps = 1e-6, - rope_type = "interleaved" - ) { - self$norm1 <- rms_norm(dim, eps = eps) - self$attn1 <- ltx2_attention( - query_dim = dim, - heads = num_attention_heads, - kv_heads = num_attention_heads, - dim_head = attention_head_dim, - rope_type = rope_type - ) - - self$norm2 <- rms_norm(dim, eps = eps) - self$ff <- feed_forward(dim, mult = 4L, activation_fn = activation_fn) - }, - - forward = function( - hidden_states, - attention_mask = NULL, - rotary_emb = NULL - ) { - norm_hidden_states <- self$norm1(hidden_states) - attn_hidden_states <- self$attn1(norm_hidden_states, - attention_mask = attention_mask, - query_rotary_emb = rotary_emb) - hidden_states <- hidden_states + attn_hidden_states - - norm_hidden_states <- self$norm2(hidden_states) - ff_hidden_states <- self$ff(norm_hidden_states) - hidden_states <- hidden_states + ff_hidden_states - - hidden_states - } -) - -# ----------------------------------------------------------------------------- -# Connector Transformer 1D -# ----------------------------------------------------------------------------- - -#' 1D Connector Transformer for LTX2 -#' @keywords internal -ltx2_connector_transformer_1d <- torch::nn_module( - "LTX2ConnectorTransformer1d", - initialize = function( - num_attention_heads = 30L, - attention_head_dim = 128L, - num_layers = 2L, - num_learnable_registers = 128L, - rope_base_seq_len = 4096L, - rope_theta = 10000.0, - rope_double_precision = TRUE, - eps = 1e-6, - causal_temporal_positioning = FALSE, - rope_type = "interleaved" - ) { - self$num_attention_heads <- num_attention_heads - self$inner_dim <- num_attention_heads * attention_head_dim - self$causal_temporal_positioning <- causal_temporal_positioning - self$num_learnable_registers <- num_learnable_registers - - # Learnable registers (replaces padding tokens) - if (!is.null(num_learnable_registers) && num_learnable_registers > 0L) { - init_registers <- torch::torch_rand(c(num_learnable_registers, self$inner_dim)) * 2.0 - 1.0 - self$learnable_registers <- torch::nn_parameter(init_registers) - } else { - self$learnable_registers <- NULL - } - - # 1D RoPE - self$rope <- ltx2_rotary_pos_embed_1d( - dim = self$inner_dim, - base_seq_len = rope_base_seq_len, - theta = rope_theta, - double_precision = rope_double_precision, - rope_type = rope_type, - num_attention_heads = num_attention_heads - ) - - # Transformer blocks - self$transformer_blocks <- torch::nn_module_list(lapply(seq_len(num_layers), function(i) { - ltx2_transformer_block_1d( - dim = self$inner_dim, - num_attention_heads = num_attention_heads, - attention_head_dim = attention_head_dim, - rope_type = rope_type - ) - })) - - self$norm_out <- rms_norm(self$inner_dim, eps = eps) - }, - - forward = function( - hidden_states, - attention_mask = NULL, - attn_mask_binarize_threshold = - 9000.0 - ) { - batch_size <- hidden_states$shape[1] - sequence_length <- hidden_states$shape[2] - - # 1. Replace padding with learned registers, if using - if (!is.null(self$learnable_registers)) { - if (sequence_length %% self$num_learnable_registers != 0L) { - stop(sprintf("Sequence length %d must be divisible by num_learnable_registers %d", - sequence_length, self$num_learnable_registers)) - } - - num_register_repeats <- sequence_length %/% self$num_learnable_registers - registers <- self$learnable_registers$`repeat`(c(num_register_repeats, 1L)) # [seq_len, inner_dim] - - # Binarize attention mask - binary_attn_mask <- (attention_mask >= attn_mask_binarize_threshold)$to(dtype = torch::torch_int32()) - if (binary_attn_mask$ndim == 4L) { - binary_attn_mask <- binary_attn_mask$squeeze(2L)$squeeze(2L) # [B, 1, 1, L] -> [B, L] - } - - # Extract non-padded tokens and re-pad with registers - padded_list <- list() - valid_seq_lens <- numeric(batch_size) - - for (i in seq_len(batch_size)) { - mask_i <- binary_attn_mask[i,]$to(dtype = torch::torch_bool()) - hs_i <- hidden_states[i, mask_i,] - valid_len <- as.integer(hs_i$shape[1]) - valid_seq_lens[i] <- valid_len - pad_len <- sequence_length - valid_len - - if (pad_len > 0L) { - # Pad with zeros on the right - hs_i <- torch::nnf_pad(hs_i, c(0L, 0L, 0L, pad_len)) - } - padded_list[[i]] <- hs_i$unsqueeze(1L) - } - - padded_hidden_states <- torch::torch_cat(padded_list, dim = 1L) # [B, L, D] - - # Flip mask along sequence dimension and blend with registers - # In R torch, flip requires a vector for dims - flipped_mask <- torch::torch_flip(binary_attn_mask, c(2L))$unsqueeze(- 1L)$to(dtype = hidden_states$dtype) # [B, L, 1] - # Expand registers to batch dimension for broadcasting - registers_expanded <- registers$unsqueeze(1L) # [L, D] -> [1, L, D] - broadcasts to [B, L, D] - hidden_states <- flipped_mask * padded_hidden_states + (1 - flipped_mask) * registers_expanded - - # Zero out attention mask when using registers - attention_mask <- torch::torch_zeros_like(attention_mask) - } - - # 2. Calculate 1D RoPE - rotary_emb <- self$rope(batch_size, sequence_length, device = hidden_states$device) - - # 3. Run transformer blocks - for (i in seq_along(self$transformer_blocks)) { - block <- self$transformer_blocks[[i]] - hidden_states <- block(hidden_states, attention_mask = attention_mask, rotary_emb = rotary_emb) - } - - hidden_states <- self$norm_out(hidden_states) - - list(hidden_states, attention_mask) - } -) - -# ----------------------------------------------------------------------------- -# Full Text Connectors -# ----------------------------------------------------------------------------- - -#' LTX2 Text Connectors -#' -#' Transforms packed text encoder hidden states for video and audio streams. -#' -#' @param caption_channels Integer. Dimension of caption embeddings (default 3840). -#' @param text_proj_in_factor Integer. Factor for input projection (default 1). -#' @param video_connector_num_attention_heads Integer. Number of attention heads for video connector. -#' @param video_connector_attention_head_dim Integer. Attention head dimension for video. -#' @param video_connector_num_layers Integer. Number of transformer layers for video. -#' @param video_connector_num_learnable_registers Integer. Number of learnable registers for video. -#' @param audio_connector_num_attention_heads Integer. Number of attention heads for audio connector. -#' @param audio_connector_attention_head_dim Integer. Attention head dimension for audio. -#' @param audio_connector_num_layers Integer. Number of transformer layers for audio. -#' @param audio_connector_num_learnable_registers Integer. Number of learnable registers for audio. -#' @param connector_rope_base_seq_len Integer. Base sequence length for RoPE. -#' @param rope_theta Numeric. RoPE theta parameter. -#' @param rope_double_precision Logical. Use double precision for RoPE. -#' @param causal_temporal_positioning Logical. Use causal temporal positioning. -#' @param rope_type Character. RoPE type ("interleaved" or "split"). -#' -#' @return nn_module for text connectors. -#' @export -ltx2_text_connectors <- torch::nn_module( - "LTX2TextConnectors", - initialize = function( - caption_channels = 3840L, - text_proj_in_factor = 49L, - video_connector_num_attention_heads = 30L, - video_connector_attention_head_dim = 128L, - video_connector_num_layers = 2L, - video_connector_num_learnable_registers = NULL, - audio_connector_num_attention_heads = 30L, - audio_connector_attention_head_dim = 128L, - audio_connector_num_layers = 2L, - audio_connector_num_learnable_registers = NULL, - connector_rope_base_seq_len = 4096L, - rope_theta = 10000.0, - rope_double_precision = TRUE, - causal_temporal_positioning = FALSE, - rope_type = "split" - ) { - - self$caption_channels <- caption_channels - - # Input projection (projects packed embeddings to caption_channels) - self$text_proj_in <- torch::nn_linear( - in_features = caption_channels * text_proj_in_factor, - out_features = caption_channels, - bias = FALSE - ) - - # Video connector - self$video_connector <- ltx2_connector_transformer_1d( - num_attention_heads = video_connector_num_attention_heads, - attention_head_dim = video_connector_attention_head_dim, - num_layers = video_connector_num_layers, - num_learnable_registers = video_connector_num_learnable_registers, - rope_base_seq_len = connector_rope_base_seq_len, - rope_theta = rope_theta, - rope_double_precision = rope_double_precision, - causal_temporal_positioning = causal_temporal_positioning, - rope_type = rope_type - ) - - # Audio connector - self$audio_connector <- ltx2_connector_transformer_1d( - num_attention_heads = audio_connector_num_attention_heads, - attention_head_dim = audio_connector_attention_head_dim, - num_layers = audio_connector_num_layers, - num_learnable_registers = audio_connector_num_learnable_registers, - rope_base_seq_len = connector_rope_base_seq_len, - rope_theta = rope_theta, - rope_double_precision = rope_double_precision, - causal_temporal_positioning = causal_temporal_positioning, - rope_type = rope_type - ) - }, - - forward = function( - text_encoder_hidden_states, - attention_mask, - additive_mask = FALSE - ) { - # Convert to additive attention mask if necessary - if (!additive_mask) { - text_dtype <- text_encoder_hidden_states$dtype - attention_mask <- (attention_mask - 1)$reshape(c(attention_mask$shape[1], 1L, - 1L, attention_mask$shape[length(attention_mask$shape)])) - attention_mask <- attention_mask$to(dtype = text_dtype) * torch::torch_finfo(text_dtype)$max - } - - # Project input - text_encoder_hidden_states <- self$text_proj_in(text_encoder_hidden_states) - - # Video connector - video_result <- self$video_connector(text_encoder_hidden_states, attention_mask) - video_text_embedding <- video_result[[1]] - new_attn_mask <- video_result[[2]] - - # Apply attention mask - attn_mask <- (new_attn_mask < 1e-6)$to(dtype = torch::torch_int64()) - attn_mask <- attn_mask$reshape(c(video_text_embedding$shape[1], video_text_embedding$shape[2], 1L)) - video_text_embedding <- video_text_embedding * attn_mask - new_attn_mask <- attn_mask$squeeze(- 1L) - - # Audio connector - audio_result <- self$audio_connector(text_encoder_hidden_states, attention_mask) - audio_text_embedding <- audio_result[[1]] - - list(video_text_embedding, audio_text_embedding, new_attn_mask) - } -) - -# ----------------------------------------------------------------------------- -# Text Encoder Wrapper -# ----------------------------------------------------------------------------- - -#' Encode Text for LTX2 -#' -#' Encodes text prompts for LTX2 video generation. Supports multiple backends: -#' - "gemma3": Native R torch Gemma3 text encoder -#' - "precomputed": Load pre-computed embeddings from file -#' - "api": Call an HTTP API for text encoding -#' - "random": Generate random embeddings (for testing only) -#' -#' @param prompt Character vector of prompts. -#' @param backend Character. Backend to use ("gemma3", "precomputed", "api", "random"). -#' @param model_path Character. Path to Gemma3 model directory (for "gemma3" backend). -#' @param tokenizer_path Character. Path to tokenizer (for "gemma3" backend, defaults to model_path). -#' @param text_encoder Pre-loaded Gemma3 text encoder module (for "gemma3" backend). -#' @param embeddings_file Character. Path to pre-computed embeddings (for "precomputed" backend). -#' @param api_url Character. URL of text encoding API (for "api" backend). -#' @param max_sequence_length Integer. Maximum sequence length (default 1024). -#' @param caption_channels Integer. Caption embedding dimension (default 3840). -#' @param device Character. Device for tensors. -#' @param dtype torch_dtype. Data type for tensors. -#' -#' @return List with prompt_embeds and prompt_attention_mask tensors. -#' @export -encode_text_ltx2 <- function( - prompt, - backend = "random", - model_path = NULL, - tokenizer_path = NULL, - text_encoder = NULL, - embeddings_file = NULL, - api_url = NULL, - max_sequence_length = 1024L, - caption_channels = 3840L, - device = "cpu", - dtype = torch::torch_float32() -) { - - if (is.character(prompt) && length(prompt) == 1) { - prompt <- list(prompt) - } else { - prompt <- as.list(prompt) - } - batch_size <- length(prompt) - - if (backend == "gemma3") { - # Native Gemma3 text encoding - if (identical(dtype, torch::torch_float16())) { - dtype_str <- "float16" - } else { - dtype_str <- "float32" - } - - result <- encode_with_gemma3( - prompts = unlist(prompt), - model = text_encoder %||% model_path, - tokenizer = tokenizer_path %||% model_path, - max_sequence_length = max_sequence_length, - device = device, - dtype = dtype_str, - verbose = FALSE - ) - - prompt_embeds <- result$prompt_embeds$to(dtype = dtype) - prompt_attention_mask <- result$prompt_attention_mask - - } else if (backend == "precomputed") { - if (is.null(embeddings_file)) { - stop("embeddings_file required for precomputed backend") - } - # Load pre-computed embeddings - data <- readRDS(embeddings_file) - prompt_embeds <- torch::torch_tensor(data$embeddings, device = device, dtype = dtype) - prompt_attention_mask <- torch::torch_tensor(data$attention_mask, device = device, dtype = torch::torch_int64()) - - } else if (backend == "api") { - if (is.null(api_url)) { - stop("api_url required for api backend") - } - # Call HTTP API - response <- httr::POST( - api_url, - body = jsonlite::toJSON(list( - prompts = prompt, - max_sequence_length = max_sequence_length - ), auto_unbox = TRUE), - httr::content_type_json() - ) - if (httr::status_code(response) != 200) { - stop("Text encoding API failed: ", httr::content(response, "text")) - } - data <- jsonlite::fromJSON(httr::content(response, "text")) - prompt_embeds <- torch::torch_tensor(data$embeddings, device = device, dtype = dtype) - prompt_attention_mask <- torch::torch_tensor(data$attention_mask, device = device, dtype = torch::torch_int64()) - - } else if (backend == "random") { - # Generate random embeddings (for testing) - # Shape: [batch, seq_len, caption_channels * num_layers] = [B, L, 3840*49] - # This mimics packed Gemma3 output for testing connectors - message("Using random embeddings - for testing only") - packed_dim <- caption_channels * 49L# 49 layers from Gemma3 - prompt_embeds <- torch::torch_randn(c(batch_size, max_sequence_length, packed_dim), - device = device, dtype = dtype) - prompt_attention_mask <- torch::torch_ones(c(batch_size, max_sequence_length), - device = device, dtype = torch::torch_int64()) - - } else { - stop("Unknown backend: ", backend, ". Use 'gemma3', 'precomputed', 'api', or 'random'") - } - - list( - prompt_embeds = prompt_embeds, - prompt_attention_mask = prompt_attention_mask - ) -} - -#' Pack Text Embeddings (Gemma-style) -#' -#' Normalizes and packs text encoder hidden states from multiple layers. -#' This is used when working with raw Gemma outputs. -#' -#' @param text_hidden_states Tensor of shape [batch, seq_len, hidden_dim, num_layers]. -#' @param sequence_lengths Integer vector of valid sequence lengths per batch item. -#' @param padding_side Character. "left" or "right". -#' @param scale_factor Numeric. Scale factor for normalization (default 8). -#' @param eps Numeric. Epsilon for numerical stability. -#' @param device Character. Device for tensors. -#' -#' @return Tensor of shape [batch, seq_len, hidden_dim * num_layers]. -#' @export -pack_text_embeds <- function( - text_hidden_states, - sequence_lengths, - padding_side = "left", - scale_factor = 8, - eps = 1e-6, - device = "cpu" -) { - - dims <- text_hidden_states$shape - batch_size <- dims[1] - seq_len <- dims[2] - hidden_dim <- dims[3] - num_layers <- dims[4] - - original_dtype <- text_hidden_states$dtype - - # Create padding mask - token_indices <- torch::torch_arange(start = 0, end = seq_len - 1L, device = device)$unsqueeze(1L) - sequence_lengths_t <- torch::torch_tensor(sequence_lengths, device = device) - - if (padding_side == "right") { - mask <- token_indices < sequence_lengths_t$unsqueeze(2L) - } else if (padding_side == "left") { - start_indices <- seq_len - sequence_lengths_t$unsqueeze(2L) - mask <- token_indices >= start_indices - } else { - stop("padding_side must be 'left' or 'right'") - } - mask <- mask$unsqueeze(- 1L)$unsqueeze(- 1L) # [B, seq_len, 1, 1] - - # Compute masked mean - masked_states <- text_hidden_states$masked_fill(!mask, 0.0) - num_valid <- (sequence_lengths_t * hidden_dim)$view(c(batch_size, 1L, 1L, 1L)) - masked_mean <- masked_states$sum(dim = c(2L, 3L), keepdim = TRUE) / (num_valid + eps) - - # Compute min/max - x_min <- text_hidden_states$masked_fill(!mask, Inf)$amin(dim = c(2L, 3L), keepdim = TRUE) - x_max <- text_hidden_states$masked_fill(!mask, - Inf)$amax(dim = c(2L, 3L), keepdim = TRUE) - - # Normalize - normalized <- (text_hidden_states - masked_mean) / (x_max - x_min + eps) - normalized <- normalized * scale_factor - - # Flatten layers dimension - normalized <- normalized$flatten(start_dim = 3L) - mask_flat <- mask$squeeze(- 1L)$expand(c(- 1L, - 1L, hidden_dim * num_layers)) - normalized <- normalized$masked_fill(!mask_flat, 0.0) - normalized <- normalized$to(dtype = original_dtype) - - normalized -} - -# ----------------------------------------------------------------------------- -# Weight Loading -# ----------------------------------------------------------------------------- - -#' Load LTX2 Text Connectors from safetensors -#' -#' Load pre-trained LTX2 connector weights from HuggingFace safetensors file. -#' -#' @param weights_path Character. Path to safetensors file. -#' @param config_path Character. Optional path to config.json. -#' @param device Character. Device to load weights to. Default: "cpu" -#' @param dtype Character. Data type ("float32", "float16"). Default: "float32" -#' @param verbose Logical. Print loading progress. Default: TRUE -#' @return Initialized ltx2_text_connectors module -#' @export -load_ltx2_connectors <- function( - weights_path, - config_path = NULL, - device = "cpu", - dtype = "float32", - verbose = TRUE -) { - if (!file.exists(weights_path)) { - stop("Weights file not found: ", weights_path) - } - - # Load config - config <- NULL - # Auto-detect config.json in same directory if not specified - if (is.null(config_path)) { - auto_config <- file.path(dirname(weights_path), "config.json") - if (file.exists(auto_config)) { - config_path <- auto_config - } - } - if (!is.null(config_path) && file.exists(config_path)) { - config <- jsonlite::fromJSON(config_path) - if (verbose) message("Loaded config from: ", config_path) - } - - # Create connectors with config or defaults - if (!is.null(config)) { - connectors <- ltx2_text_connectors( - caption_channels = config$caption_channels %||% 3840L, - text_proj_in_factor = config$text_proj_in_factor %||% 49L, - video_connector_num_attention_heads = config$video_connector_num_attention_heads %||% 30L, - video_connector_attention_head_dim = config$video_connector_attention_head_dim %||% 128L, - video_connector_num_layers = config$video_connector_num_layers %||% 2L, - video_connector_num_learnable_registers = as.integer(config$video_connector_num_learnable_registers), - audio_connector_num_attention_heads = config$audio_connector_num_attention_heads %||% 30L, - audio_connector_attention_head_dim = config$audio_connector_attention_head_dim %||% 128L, - audio_connector_num_layers = config$audio_connector_num_layers %||% 2L, - audio_connector_num_learnable_registers = as.integer(config$audio_connector_num_learnable_registers), - connector_rope_base_seq_len = config$connector_rope_base_seq_len %||% 4096L, - rope_theta = config$rope_theta %||% 10000.0, - rope_double_precision = config$rope_double_precision %||% TRUE, - causal_temporal_positioning = config$causal_temporal_positioning %||% FALSE, - rope_type = config$rope_type %||% "split" - ) - } else { - connectors <- ltx2_text_connectors() - } - - # Load weights - if (verbose) message("Loading weights from: ", weights_path) - weights <- safetensors::safe_load_file(weights_path, framework = "torch") - - load_ltx2_connector_weights(connectors, weights, verbose = verbose) - - # Move to device - torch_dtype <- switch(dtype, - "float32" = torch::torch_float32(), - "float16" = torch::torch_float16(), - "bfloat16" = torch::torch_bfloat16(), - torch::torch_float32() - ) - - connectors$to(device = device, dtype = torch_dtype) - - if (verbose) message("Connectors loaded successfully on device: ", device) - connectors -} - -#' Load weights into LTX2 connectors module -#' -#' @param connectors LTX2 connectors module -#' @param weights Named list of weight tensors -#' @param verbose Print progress -#' @keywords internal -load_ltx2_connector_weights <- function( - connectors, - weights, - verbose = TRUE -) { - native_params <- names(connectors$parameters) - - remap_connector_key <- function(key) { - # HuggingFace uses nn.ModuleList for FeedForward: - # ff.net.0.proj.weight -> ff.act_fn.proj.weight - # ff.net.2.weight -> ff.proj_out.weight - key <- gsub("\\.ff\\.net\\.0\\.", ".ff.act_fn.", key) - key <- gsub("\\.ff\\.net\\.2\\.", ".ff.proj_out.", key) - - # to_out.0 is correct - both HF and our module use ModuleList - key - } - - loaded <- 0L - skipped <- 0L - unmapped <- character(0) - - torch::with_no_grad({ - for (hf_name in names(weights)) { - native_name <- remap_connector_key(hf_name) - - if (native_name %in% native_params) { - hf_tensor <- weights[[hf_name]] - native_tensor <- connectors$parameters[[native_name]] - - if (all(as.integer(hf_tensor$shape) == as.integer(native_tensor$shape))) { - native_tensor$copy_(hf_tensor) - loaded <- loaded + 1L - } else { - if (verbose) { - message("Shape mismatch: ", native_name, - " (HF: ", paste(as.integer(hf_tensor$shape), collapse = "x"), - " vs R: ", paste(as.integer(native_tensor$shape), collapse = "x"), ")") - } - skipped <- skipped + 1L - } - } else { - skipped <- skipped + 1L - unmapped <- c(unmapped, paste0(hf_name, " -> ", native_name)) - } - } - }) - - if (verbose) { - message(sprintf("Connector weights: %d loaded, %d skipped", loaded, skipped)) - if (length(unmapped) > 0 && length(unmapped) <= 20) { - message("Unmapped parameters:") - for (u in unmapped[1:min(20, length(unmapped))]) { - message(" ", u) - } - } - if (length(unmapped) > 20) { - message(" ... and ", length(unmapped) - 20, " more") - } - } - - invisible(list(loaded = loaded, skipped = skipped, unmapped = unmapped)) -} - -# Null-coalescing operator (if not already defined) -if (!exists("%||%", mode = "function")) { - `%||%` <- function( - x, - y - ) if (is.null(x)) y else x -} - diff --git a/R/txt2vid_ltx2.R b/R/txt2vid_ltx2.R deleted file mode 100644 index 68517b6..0000000 --- a/R/txt2vid_ltx2.R +++ /dev/null @@ -1,615 +0,0 @@ -#' Generate Video from Text Prompt using LTX-2 -#' -#' Generates video using the LTX-2 diffusion transformer model. -#' -#' @param prompt Character. Text prompt describing the video to generate. -#' @param negative_prompt Character. Optional negative prompt. -#' @param width Integer. Video width in pixels (default 768). -#' @param height Integer. Video height in pixels (default 512). -#' @param num_frames Integer. Number of frames to generate (default 121). -#' @param fps Numeric. Frames per second (default 24). -#' @param num_inference_steps Integer. Number of denoising steps (default 8 for distilled). -#' @param guidance_scale Numeric. CFG scale (default 4.0). -#' @param memory_profile Character or list. Memory profile: "auto" for auto-detection, -#' or a profile from `ltx2_memory_profile()`. -#' @param text_backend Character. Text encoding backend: "gemma3" (native), "api", "precomputed", or "random". -#' @param text_model_path Character. Path to Gemma3 model (for "gemma3" backend). Supports glob patterns. -#' @param text_api_url Character. URL for text encoding API (if backend = "api"). -#' @param vae Optional. Pre-loaded VAE module. -#' @param dit Optional. Pre-loaded DiT transformer module. -#' @param connectors Optional. Pre-loaded text connectors module. -#' @param seed Integer. Random seed for reproducibility. -#' @param output_file Character. Path to save output video (NULL for no save). -#' @param output_format Character. Output format: "mp4", "gif", or "frames". -#' @param return_latents Logical. If TRUE, also return final latents. -#' @param verbose Logical. Print progress messages. -#' -#' @return A list with: -#' - `video`: Array of video frames [frames, height, width, channels] -#' - `latents`: (if return_latents=TRUE) Final latent tensor -#' - `metadata`: Generation metadata -#' -#' @export -#' -#' @examples -#' \dontrun{ -#' # Basic usage -#' result <- txt2vid_ltx2("A cat walking on a beach at sunset") -#' -#' # With specific settings -#' result <- txt2vid_ltx2( -#' prompt = "A timelapse of clouds moving over mountains", -#' width = 512, -#' height = 320, -#' num_frames = 61, -#' num_inference_steps = 8, -#' seed = 42, -#' output_file = "clouds.mp4" -#' ) -#' } -txt2vid_ltx2 <- function( - prompt, - negative_prompt = NULL, - width = 768L, - height = 512L, - num_frames = 121L, - fps = 24.0, - num_inference_steps = 8L, - guidance_scale = 4.0, - memory_profile = "auto", - text_backend = "gemma3", - text_model_path = NULL, - text_api_url = NULL, - vae = NULL, - dit = NULL, - connectors = NULL, - seed = NULL, - output_file = NULL, - output_format = "mp4", - return_latents = FALSE, - verbose = TRUE -) { - # Start timing - start_time <- Sys.time() - - # Ensure integers - width <- as.integer(width) - height <- as.integer(height) - num_frames <- as.integer(num_frames) - num_inference_steps <- as.integer(num_inference_steps) - - # Set seed if provided - if (!is.null(seed)) { - torch::torch_manual_seed(seed) - # torch_manual_seed sets both CPU and CUDA seeds - } - - # Resolve memory profile - if (identical(memory_profile, "auto")) { - memory_profile <- ltx2_memory_profile() - } else if (is.character(memory_profile)) { - # Named profile - vram <- switch(memory_profile, - "high" = 20, - "medium" = 12, - "low" = 8, - "very_low" = 6, - "cpu_only" = 0, - 8# default - ) - memory_profile <- ltx2_memory_profile(vram_gb = vram) - } - - if (verbose) { - message(sprintf("Using memory profile: %s", memory_profile$name)) - } - - # Validate and adjust resolution for profile - validated <- validate_resolution(height, width, num_frames, memory_profile) - if (validated$adjusted) { - height <- validated$height - width <- validated$width - num_frames <- validated$num_frames - } - - # LTX-2 VAE compression ratios - spatial_ratio <- 32L - temporal_ratio <- 8L - - # Calculate latent dimensions - latent_height <- height %/% spatial_ratio - latent_width <- width %/% spatial_ratio - latent_frames <- (num_frames - 1L) %/% temporal_ratio + 1L - - if (verbose) { - message(sprintf("Video: %dx%d, %d frames @ %.1f fps", width, height, num_frames, fps)) - message(sprintf("Latents: %dx%d, %d frames", latent_width, latent_height, latent_frames)) - } - - # Device setup - dit_device <- memory_profile$dit_device - vae_device <- memory_profile$vae_device - - torch::with_no_grad({ - - # ---- Step 1: Text Encoding ---- - if (verbose) message("Encoding text prompt...") - - # Determine dtype based on profile - latent_dtype <- if (memory_profile$dtype == "float16") { - torch::torch_float16() - } else { - torch::torch_float32() - } - - # Resolve model path (use hfhub or explicit path) - resolved_model_path <- NULL - if (text_backend == "gemma3") { - if (!is.null(text_model_path)) { - # Explicit path provided - expanded_path <- path.expand(text_model_path) - if (grepl("\\*", expanded_path)) { - # Glob pattern - find matching directories - matches <- Sys.glob(expanded_path) - if (length(matches) > 0) { - resolved_model_path <- matches[1] - } - } else if (dir.exists(expanded_path)) { - resolved_model_path <- expanded_path - } - if (is.null(resolved_model_path)) { - stop("Gemma3 model not found at: ", text_model_path) - } - } else { - # Use hfhub to find model via config.json - if (!requireNamespace("hfhub", quietly = TRUE)) { - stop("Package 'hfhub' is required. Install with: install.packages('hfhub')") - } - gemma_repo <- "google/gemma-3-12b-it" - config_path <- tryCatch({ - hfhub::hub_download(gemma_repo, "config.json", local_files_only = TRUE) - }, error = function(e) NULL) - - if (is.null(config_path)) { - stop("Gemma3 model not found in HuggingFace cache.\n", - "Download with: huggingface-cli download ", gemma_repo) - } - resolved_model_path <- dirname(config_path) - } - } - - # Encode prompt - text_result <- encode_text_ltx2( - prompt = prompt, - backend = text_backend, - model_path = resolved_model_path, - tokenizer_path = resolved_model_path, - api_url = text_api_url, - max_sequence_length = 128L, - caption_channels = 3840L, - device = "cpu", # Text encoder on CPU (GPU-poor) - dtype = torch::torch_float32() # CPU always float32 - ) - prompt_embeds <- text_result$prompt_embeds - prompt_attention_mask <- text_result$prompt_attention_mask - - # Encode negative prompt - if (is.null(negative_prompt)) { - negative_prompt <- "" - } - neg_result <- encode_text_ltx2( - prompt = negative_prompt, - backend = text_backend, - model_path = resolved_model_path, - tokenizer_path = resolved_model_path, - api_url = text_api_url, - max_sequence_length = 128L, - caption_channels = 3840L, - device = "cpu", - dtype = torch::torch_float32() - ) - negative_prompt_embeds <- neg_result$prompt_embeds - negative_attention_mask <- neg_result$prompt_attention_mask - - # ---- Step 1b: Apply Connectors ---- - # Connectors transform packed text embeddings for video/audio cross-attention - if (is.null(connectors)) { - # Load connectors from HuggingFace using hfhub - connector_path <- tryCatch({ - if (!requireNamespace("hfhub", quietly = TRUE)) NULL - else hfhub::hub_download( - "Lightricks/LTX-2", - "connectors/diffusion_pytorch_model.safetensors", - local_files_only = TRUE - ) - }, error = function(e) NULL) - - if (!is.null(connector_path) && file.exists(connector_path)) { - if (verbose) message("Loading text connectors...") - connectors <- load_ltx2_connectors( - weights_path = connector_path, - device = "cpu", - dtype = "float32", - verbose = verbose - ) - } else { - if (verbose) message("Text connectors not found - using embeddings directly") - } - } - - if (!is.null(connectors)) { - # Run connectors to get video/audio conditioning - if (verbose) message("Applying text connectors...") - connector_result <- connectors(prompt_embeds, prompt_attention_mask) - video_embeds <- connector_result[[1]] - audio_embeds <- connector_result[[2]] - - neg_connector_result <- connectors(negative_prompt_embeds, negative_attention_mask) - neg_video_embeds <- neg_connector_result[[1]] - neg_audio_embeds <- neg_connector_result[[2]] - } else { - # Fallback: use packed embeddings directly (may not match DiT dimensions) - video_embeds <- prompt_embeds - audio_embeds <- prompt_embeds - neg_video_embeds <- negative_prompt_embeds - neg_audio_embeds <- negative_prompt_embeds - } - - # Move to GPU with correct dtype - video_embeds <- video_embeds$to(device = dit_device, dtype = latent_dtype) - audio_embeds <- audio_embeds$to(device = dit_device, dtype = latent_dtype) - neg_video_embeds <- neg_video_embeds$to(device = dit_device, dtype = latent_dtype) - neg_audio_embeds <- neg_audio_embeds$to(device = dit_device, dtype = latent_dtype) - - # ---- Step 2: Initialize Latents ---- - if (verbose) message("Initializing latents...") - - # LTX-2 has 128 latent channels - latent_channels <- 128L - batch_size <- 1L - - # Random noise - latents <- torch::torch_randn( - c(batch_size, latent_channels, latent_frames, latent_height, latent_width), - device = dit_device, - dtype = latent_dtype - ) - - # Flatten spatial dims for transformer: [B, C, T, H, W] -> [B, T*H*W, C] - num_patches <- latent_frames * latent_height * latent_width - latents <- latents$permute(c(1, 3, 4, 5, 2)) # [B, T, H, W, C] - latents <- latents$reshape(c(batch_size, num_patches, latent_channels)) - - # ---- Step 3: Create Scheduler ---- - if (verbose) message("Setting up FlowMatch scheduler...") - - schedule <- flowmatch_set_timesteps( - flowmatch_scheduler_create(shift = 9.0), - num_inference_steps = num_inference_steps, - device = dit_device - ) - - # ---- Step 4: Load/Create DiT if needed ---- - if (is.null(dit)) { - # Try to load INT4 quantized weights from R_user_dir cache - cache_dir <- tools::R_user_dir("diffuseR", "cache") - int4_path <- file.path(cache_dir, "ltx2_transformer_int4.safetensors") - # Check for single file or sharded files - int4_shards <- list.files(cache_dir, pattern = "ltx2_transformer_int4.*\\.safetensors$") - if (file.exists(int4_path) || length(int4_shards) > 0) { - if (verbose) message("Loading INT4 quantized DiT...") - - # Enable INT4-native model creation - old_use_int4 <- getOption("diffuseR.use_int4", FALSE) - old_int4_device <- getOption("diffuseR.int4_device", "cuda") - old_int4_dtype <- getOption("diffuseR.int4_dtype", torch::torch_float16()) - - options(diffuseR.use_int4 = TRUE) - options(diffuseR.int4_device = dit_device) - options(diffuseR.int4_dtype = latent_dtype) - - # Create model with int4_linear layers via make_linear() - dit <- ltx2_video_transformer_3d_model( - in_channels = latent_channels, - out_channels = latent_channels, - num_attention_heads = 32L, - attention_head_dim = 128L, - cross_attention_dim = 4096L, - audio_in_channels = 128L, - audio_out_channels = 128L, - audio_num_attention_heads = 32L, - audio_attention_head_dim = 64L, - audio_cross_attention_dim = 2048L, - num_layers = 48L, - caption_channels = 3840L, - vae_scale_factors = c(temporal_ratio, spatial_ratio, spatial_ratio) - ) - - # Load INT4 weights (keeps them compressed, dequantizes during forward) - int4_weights <- load_int4_weights(int4_path, verbose = verbose) - load_int4_weights_into_model(dit, int4_weights, verbose = verbose) - - # Move model to GPU (non-INT4 params like biases, norms) - dit <- dit$to(device = dit_device, dtype = latent_dtype) - - # Restore options - options(diffuseR.use_int4 = old_use_int4) - options(diffuseR.int4_device = old_int4_device) - options(diffuseR.int4_dtype = old_int4_dtype) - } else { - if (verbose) message("NOTE: INT4 weights not found - run quantize_ltx2_transformer() first") - stop("DiT model required. Run: quantize_ltx2_transformer()") - } - } - - # ---- Step 5: Denoising Loop ---- - if (verbose) message(sprintf("Denoising (%d steps)...", num_inference_steps)) - - # Audio placeholder (zeros for video-only generation) - audio_latents <- torch::torch_zeros( - c(batch_size, 50L, 128L), # Placeholder audio: [B, seq, audio_channels=128] - device = dit_device, - dtype = latent_dtype - ) - - timesteps_vec <- schedule$timesteps - sigmas <- schedule$sigmas - - for (i in seq_len(num_inference_steps)) { - t_idx <- i - t <- timesteps_vec[t_idx] - sigma <- sigmas[t_idx] - if (i < num_inference_steps) { - sigma_next <- sigmas[t_idx + 1L] - } else { - sigma_next <- 0 - } - - if (verbose && i %% max(1, num_inference_steps %/% 4) == 1) { - message(sprintf(" Step %d/%d (sigma=%.3f)", i, num_inference_steps, as.numeric(sigma))) - } - - # Prepare timestep tensor - timestep <- torch::torch_tensor(c(as.numeric(t)))$unsqueeze(2L) - timestep <- timestep$to(device = dit_device, dtype = latent_dtype) - - # CFG: conditional and unconditional pass - if (memory_profile$cfg_mode == "sequential") { - # Sequential CFG (memory efficient) - noise_pred <- sequential_cfg_forward( - model = dit, - latents = latents, - timestep = timestep, - prompt_embeds = video_embeds, - negative_prompt_embeds = neg_video_embeds, - guidance_scale = guidance_scale, - audio_hidden_states = audio_latents, - audio_encoder_hidden_states = audio_embeds, - num_frames = latent_frames, - height = latent_height, - width = latent_width, - fps = fps, - audio_num_frames = 50L - ) - } else { - # Batched CFG - latents_input <- torch::torch_cat(list(latents, latents), dim = 1L) - video_input <- torch::torch_cat(list(neg_video_embeds, video_embeds), dim = 1L) - audio_input <- torch::torch_cat(list(neg_audio_embeds, audio_embeds), dim = 1L) - timestep_input <- torch::torch_cat(list(timestep, timestep), dim = 1L) - - output <- dit( - hidden_states = latents_input, - audio_hidden_states = torch::torch_cat(list(audio_latents, audio_latents), dim = 1L), - encoder_hidden_states = video_input, - audio_encoder_hidden_states = audio_input, - timestep = timestep_input, - num_frames = latent_frames, - height = latent_height, - width = latent_width, - fps = fps, - audio_num_frames = 50L - ) - - noise_pred_all <- output$sample - noise_pred_uncond <- noise_pred_all[1,,]$unsqueeze(1L) - noise_pred_cond <- noise_pred_all[2,,]$unsqueeze(1L) - # CFG: use tensor method to preserve dtype - noise_pred <- noise_pred_uncond + (noise_pred_cond - noise_pred_uncond)$mul(guidance_scale) - } - - # FlowMatch step - dt <- torch::torch_tensor(sigma_next - sigma, dtype = latent_dtype, device = dit_device) - latents <- latents + dt * noise_pred - - # Cleanup for low memory - if (memory_profile$name %in% c("low", "very_low") && i %% 2 == 0) { - clear_vram() - } - } - - # ---- Step 6: Decode Latents ---- - if (verbose) message("Decoding video...") - - # Reshape latents back to spatial: [B, T*H*W, C] -> [B, C, T, H, W] - latents <- latents$reshape(c(batch_size, latent_frames, latent_height, latent_width, latent_channels)) - latents <- latents$permute(c(1, 5, 2, 3, 4)) # [B, C, T, H, W] - - # Load/create VAE if needed - if (is.null(vae)) { - if (verbose) message("Loading VAE...") - - # Try to find VAE in HuggingFace cache - vae_path <- tryCatch({ - if (requireNamespace("hfhub", quietly = TRUE)) { - config_path <- hfhub::hub_download("Lightricks/LTX-2", "vae/config.json", - local_files_only = TRUE) - dirname(config_path) - } else { - NULL - } - }, error = function(e) NULL) - - if (is.null(vae_path)) { - if (verbose) message("NOTE: VAE not found in cache - skipping decode") - video_tensor <- latents - } else { - # Determine VAE dtype based on latent dtype - vae_dtype <- if (identical(latent_dtype, torch::torch_bfloat16()) || - identical(latent_dtype, torch::torch_float16())) { - "float16" - } else { - "float32" - } - - vae <- load_ltx2_vae( - weights_path = vae_path, - device = vae_device, - dtype = vae_dtype, - verbose = verbose - ) - } - } - - if (!is.null(vae)) { - # Configure VAE for memory profile - configure_vae_for_profile(vae, memory_profile) - - # Move VAE to device - vae <- vae$to(device = vae_device) - - # Decode - video_tensor <- vae$decode(latents) - } - - # Prepare tensor for conversion to R array - # NOTE: Must use as.array() instead of $numpy() due to R torch bug where - - # tensors returned from with_no_grad() have corrupted method references - # (error: "could not find function 'fn'"). See cornyverse CLAUDE.md. - video_cpu <- video_tensor$squeeze(1L)$permute(c(2, 3, 4, 1))$cpu() - - }) # end with_no_grad - - # Convert to R array (as.array works, $numpy() fails on tensors from with_no_grad) - video_array <- as.array(video_cpu) - - # Clamp to [0, 1] - video_array <- pmax(pmin(video_array, 1), 0) - - # ---- Step 7: Save Output ---- - if (!is.null(output_file)) { - if (verbose) message(sprintf("Saving to %s...", output_file)) - save_video_frames(video_array, output_file, fps = fps, verbose = verbose) - } - - # Build result - elapsed <- as.numeric(difftime(Sys.time(), start_time, units = "secs")) - if (verbose) { - message(sprintf("Generation complete in %.1f seconds", elapsed)) - } - - result <- list( - video = video_array, - metadata = list( - prompt = prompt, - negative_prompt = negative_prompt, - width = width, - height = height, - num_frames = num_frames, - fps = fps, - num_inference_steps = num_inference_steps, - guidance_scale = guidance_scale, - seed = seed, - memory_profile = memory_profile$name, - elapsed_seconds = elapsed - ) - ) - - if (return_latents) { - result$latents <- latents$cpu() - } - - result -} - -#' Save video frames to file -#' -#' Saves an array of video frames to an MP4 file using ffmpeg via the av package. -#' -#' @param video_array Numeric array with dimensions [frames, height, width, channels]. -#' Values should be in range [0, 1]. -#' @param output_file Character. Path to output video file. -#' @param fps Numeric. Frames per second. Default: 24. -#' @param verbose Logical. Print progress messages. Default: TRUE. -#' @return Invisibly returns the output file path. -#' @keywords internal -save_video_frames <- function( - video_array, - output_file, - fps = 24, - verbose = TRUE -) { - if (!requireNamespace("av", quietly = TRUE)) { - stop("Package 'av' is required for video saving. Install with: install.packages('av')") - } - - # video_array should be [frames, height, width, channels] - dims <- dim(video_array) - if (length(dims) != 4) { - stop("video_array must have 4 dimensions: [frames, height, width, channels]") - } - - num_frames <- dims[1] - height <- dims[2] - width <- dims[3] - channels <- dims[4] - - if (channels != 3) { - stop("Expected 3 color channels, got ", channels) - } - - # Create temporary directory for frames - - temp_dir <- tempfile("video_frames_") - dir.create(temp_dir) - on.exit(unlink(temp_dir, recursive = TRUE), add = TRUE) - - # Save each frame as PNG - if (verbose) message(sprintf(" Writing %d frames...", num_frames)) - - for (i in seq_len(num_frames)) { - # Extract frame and convert to [0, 255] uint8 - frame <- video_array[i,,,] - frame <- pmax(pmin(frame, 1), 0) * 255 - - # Convert to integer matrix for png - # frame is [height, width, channels] - frame_int <- array(as.integer(round(frame)), dim = dim(frame)) - - # Save as PNG - frame_path <- file.path(temp_dir, sprintf("frame_%05d.png", i)) - png::writePNG(frame_int / 255, frame_path) - } - - # Encode video using av - if (verbose) message(" Encoding video...") - - # Get list of frame files - frame_files <- list.files(temp_dir, pattern = "frame_.*\\.png$", full.names = TRUE) - frame_files <- sort(frame_files) - - # Use av to encode - av::av_encode_video( - input = frame_files, - output = output_file, - framerate = fps, - codec = "libx264", - verbose = FALSE - ) - - if (verbose) message(sprintf(" Saved: %s", output_file)) - - invisible(output_file) -} - diff --git a/R/vae_ltx2.R b/R/vae_ltx2.R deleted file mode 100644 index 9b9deeb..0000000 --- a/R/vae_ltx2.R +++ /dev/null @@ -1,913 +0,0 @@ -#' LTX2 Video VAE -#' -#' 3D causal VAE for video encoding/decoding as used in LTX-2. -#' Supports GPU-poor tiling for memory-efficient processing. -#' -#' @name vae_ltx2 -NULL - -#' LTX2 Video Encoder -#' -#' Encodes video frames into latent space with 3D causal convolutions. -#' -#' @param in_channels Integer. Input channels (typically 3 for RGB). -#' @param out_channels Integer. Latent channels. -#' @param block_out_channels Integer vector. Output channels per block. -#' @param spatio_temporal_scaling Logical vector. Whether each block downscales. -#' @param layers_per_block Integer vector. Number of layers per block. -#' @param downsample_type Character vector. Type of downsampling per block. -#' @param patch_size Integer. Spatial patch size. -#' @param patch_size_t Integer. Temporal patch size. -#' @param resnet_norm_eps Numeric. Epsilon for normalization. -#' @param is_causal Logical. Whether to use causal convolutions. -#' @param spatial_padding_mode Character. Padding mode. -#' @export -ltx2_video_encoder3d <- torch::nn_module( - "LTX2VideoEncoder3d", - - initialize = function( - in_channels = 3L, - out_channels = 128L, - block_out_channels = c(256L, 512L, 1024L, 2048L), - spatio_temporal_scaling = c(TRUE, TRUE, TRUE, TRUE), - layers_per_block = c(4L, 6L, 6L, 2L, 2L), - downsample_type = c("spatial", "temporal", "spatiotemporal", "spatiotemporal"), - patch_size = 4L, - patch_size_t = 1L, - resnet_norm_eps = 1e-6, - is_causal = TRUE, - spatial_padding_mode = "zeros" - ) { - self$patch_size <- patch_size - self$patch_size_t <- patch_size_t - self$in_channels <- in_channels * patch_size ^ 2 - self$is_causal <- is_causal - - output_channel <- out_channels - - # Input convolution - self$conv_in <- ltx2_video_causal_conv3d( - in_channels = self$in_channels, - out_channels = output_channel, - kernel_size = 3L, - stride = 1L, - spatial_padding_mode = spatial_padding_mode - ) - - # Down blocks - num_blocks <- length(block_out_channels) - down_blocks <- list() - for (i in seq_len(num_blocks)) { - input_channel <- output_channel - output_channel <- block_out_channels[i] - - down_blocks[[i]] <- ltx2_video_down_block3d( - in_channels = input_channel, - out_channels = output_channel, - num_layers = layers_per_block[i], - resnet_eps = resnet_norm_eps, - spatio_temporal_scale = spatio_temporal_scaling[i], - downsample_type = downsample_type[i], - spatial_padding_mode = spatial_padding_mode - ) - } - self$down_blocks <- torch::nn_module_list(down_blocks) - - # Mid block - self$mid_block <- ltx2_video_mid_block3d( - in_channels = output_channel, - num_layers = layers_per_block[length(layers_per_block)], - resnet_eps = resnet_norm_eps, - spatial_padding_mode = spatial_padding_mode - ) - - # Output - self$norm_out <- per_channel_rms_norm() - self$conv_act <- torch::nn_silu() - self$conv_out <- ltx2_video_causal_conv3d( - in_channels = output_channel, - out_channels = out_channels + 1L, # +1 for log variance - kernel_size = 3L, - stride = 1L, - spatial_padding_mode = spatial_padding_mode - ) - }, - - forward = function( - hidden_states, - causal = NULL - ) { - p <- self$patch_size - p_t <- self$patch_size_t - - batch_size <- hidden_states$shape[1] - num_channels <- hidden_states$shape[2] - num_frames <- hidden_states$shape[3] - height <- hidden_states$shape[4] - width <- hidden_states$shape[5] - - post_patch_num_frames <- num_frames %/% p_t - post_patch_height <- height %/% p - post_patch_width <- width %/% p - - if (is.null(causal)) causal <- self$is_causal - - # Patchify: reshape to separate patches - hidden_states <- hidden_states$reshape(c( - batch_size, num_channels, - post_patch_num_frames, p_t, - post_patch_height, p, - post_patch_width, p - )) - # Permute to channel-first for patches - # [B, C, F', p_t, H', p, W', p] -> [B, C, p_t, p, p, F', H', W'] - hidden_states <- hidden_states$permute(c(1, 2, 4, 8, 6, 3, 5, 7)) - hidden_states <- hidden_states$flatten(start_dim = 2, end_dim = 5) - - hidden_states <- self$conv_in(hidden_states, causal = causal) - - for (i in seq_along(self$down_blocks)) { - hidden_states <- self$down_blocks[[i]](hidden_states, causal = causal) - } - - hidden_states <- self$mid_block(hidden_states, causal = causal) - - hidden_states <- self$norm_out(hidden_states) - hidden_states <- self$conv_act(hidden_states) - hidden_states <- self$conv_out(hidden_states, causal = causal) - - # Duplicate last channel for mean/logvar split - last_channel <- hidden_states[, - 1,,,]$unsqueeze(2) - last_channel <- last_channel$`repeat`(c(1L, hidden_states$shape[2] - 2L, 1L, 1L, 1L)) - hidden_states <- torch::torch_cat(list(hidden_states, last_channel), dim = 2) - - hidden_states - } -) - -#' LTX2 Video Decoder -#' -#' Decodes latent representations back to video frames. -#' -#' @param in_channels Integer. Latent channels. -#' @param out_channels Integer. Output channels (typically 3 for RGB). -#' @param block_out_channels Integer vector. Output channels per block. -#' @param spatio_temporal_scaling Logical vector. Whether each block upscales. -#' @param layers_per_block Integer vector. Number of layers per block. -#' @param patch_size Integer. Spatial patch size. -#' @param patch_size_t Integer. Temporal patch size. -#' @param resnet_norm_eps Numeric. Epsilon for normalization. -#' @param is_causal Logical. Whether to use causal convolutions. -#' @param inject_noise Logical vector. Whether to inject noise per block. -#' @param timestep_conditioning Logical. Whether to use timestep conditioning. -#' @param upsample_residual Logical vector. Whether upsamplers use residual. -#' @param upsample_factor Integer vector. Channel upscale factors. -#' @param spatial_padding_mode Character. Padding mode. -#' @export -ltx2_video_decoder3d <- torch::nn_module( - "LTX2VideoDecoder3d", - - initialize = function( - in_channels = 128L, - out_channels = 3L, - block_out_channels = c(256L, 512L, 1024L), - spatio_temporal_scaling = c(TRUE, TRUE, TRUE), - layers_per_block = c(5L, 5L, 5L, 5L), - patch_size = 4L, - patch_size_t = 1L, - resnet_norm_eps = 1e-6, - is_causal = FALSE, - inject_noise = c(FALSE, FALSE, FALSE, FALSE), - timestep_conditioning = FALSE, - upsample_residual = c(TRUE, TRUE, TRUE), - upsample_factor = c(2L, 2L, 2L), - spatial_padding_mode = "reflect" - ) { - self$patch_size <- patch_size - self$patch_size_t <- patch_size_t - self$out_channels <- out_channels * patch_size ^ 2 - self$is_causal <- is_causal - - # Reverse orders for decoder - block_out_channels <- rev(block_out_channels) - spatio_temporal_scaling <- rev(spatio_temporal_scaling) - layers_per_block <- rev(layers_per_block) - inject_noise <- rev(inject_noise) - upsample_residual <- rev(upsample_residual) - upsample_factor <- rev(upsample_factor) - - output_channel <- block_out_channels[1] - - # Input convolution - self$conv_in <- ltx2_video_causal_conv3d( - in_channels = in_channels, - out_channels = output_channel, - kernel_size = 3L, - stride = 1L, - spatial_padding_mode = spatial_padding_mode - ) - - # Mid block - self$mid_block <- ltx2_video_mid_block3d( - in_channels = output_channel, - num_layers = layers_per_block[1], - resnet_eps = resnet_norm_eps, - inject_noise = inject_noise[1], - timestep_conditioning = timestep_conditioning, - spatial_padding_mode = spatial_padding_mode - ) - - # Up blocks - num_blocks <- length(block_out_channels) - up_blocks <- list() - for (i in seq_len(num_blocks)) { - input_channel <- output_channel %/% upsample_factor[i] - output_channel <- block_out_channels[i] %/% upsample_factor[i] - - up_blocks[[i]] <- ltx2_video_up_block3d( - in_channels = input_channel, - out_channels = output_channel, - num_layers = layers_per_block[i + 1], - resnet_eps = resnet_norm_eps, - spatio_temporal_scale = spatio_temporal_scaling[i], - inject_noise = if (i + 1 <= length(inject_noise)) inject_noise[i + 1] else FALSE, - timestep_conditioning = timestep_conditioning, - upsample_residual = upsample_residual[i], - upscale_factor = upsample_factor[i], - spatial_padding_mode = spatial_padding_mode - ) - } - self$up_blocks <- torch::nn_module_list(up_blocks) - - # Output - self$norm_out <- per_channel_rms_norm() - self$conv_act <- torch::nn_silu() - self$conv_out <- ltx2_video_causal_conv3d( - in_channels = output_channel, - out_channels = self$out_channels, - kernel_size = 3L, - stride = 1L, - spatial_padding_mode = spatial_padding_mode - ) - - # Timestep embedding (optional) - self$time_embedder <- NULL - self$scale_shift_table <- NULL - self$timestep_scale_multiplier <- NULL - }, - - forward = function( - hidden_states, - temb = NULL, - causal = NULL - ) { - if (is.null(causal)) causal <- self$is_causal - - hidden_states <- self$conv_in(hidden_states, causal = causal) - - if (!is.null(self$timestep_scale_multiplier) && !is.null(temb)) { - temb <- temb * self$timestep_scale_multiplier - } - - hidden_states <- self$mid_block(hidden_states, temb, causal = causal) - - for (i in seq_along(self$up_blocks)) { - hidden_states <- self$up_blocks[[i]](hidden_states, temb, causal = causal) - } - - hidden_states <- self$norm_out(hidden_states) - hidden_states <- self$conv_act(hidden_states) - hidden_states <- self$conv_out(hidden_states, causal = causal) - - # Unpatchify: reshape to original spatial dims - p <- self$patch_size - p_t <- self$patch_size_t - - batch_size <- hidden_states$shape[1] - num_channels <- hidden_states$shape[2] - num_frames <- hidden_states$shape[3] - height <- hidden_states$shape[4] - width <- hidden_states$shape[5] - - # [B, C*p_t*p*p, F, H, W] -> [B, C, p_t, p, p, F, H, W] - hidden_states <- hidden_states$reshape(c( - batch_size, - 1, p_t, p, p, num_frames, height, width - )) - # Permute: [B, C, p_t, p, p, F, H, W] -> [B, C, F, p_t, H, p, W, p] - hidden_states <- hidden_states$permute(c(1, 2, 6, 3, 7, 5, 8, 4)) - # Flatten to full resolution - hidden_states <- hidden_states$flatten(start_dim = 7, end_dim = 8) # W*p - hidden_states <- hidden_states$flatten(start_dim = 5, end_dim = 6) # H*p - hidden_states <- hidden_states$flatten(start_dim = 3, end_dim = 4) # F*p_t - - hidden_states - } -) - -#' Diagonal Gaussian Distribution -#' -#' Represents a diagonal Gaussian distribution for VAE latents. -#' -#' @param parameters Tensor of concatenated mean and log variance. -#' @export -diagonal_gaussian_distribution <- function(parameters) { - # Split parameters into mean and logvar - chunk_dim <- 2# Channel dimension - mean_logvar <- parameters$chunk(2, dim = chunk_dim) - mean <- mean_logvar[[1]] - logvar <- mean_logvar[[2]] - - # Clamp logvar for numerical stability - logvar <- logvar$clamp(min = - 30.0, max = 20.0) - std <- torch::torch_exp(0.5 * logvar) - var <- torch::torch_exp(logvar) - - list( - mean = mean, - logvar = logvar, - std = std, - var = var, - sample = function(generator = NULL) { - sample <- torch::torch_randn_like(mean) - mean + std * sample - }, - mode = function() mean - ) -} - -#' LTX2 Video VAE -#' -#' Full VAE with encoder and decoder, supporting tiled encoding/decoding -#' for GPU-poor memory management. -#' -#' @param in_channels Integer. Input channels. -#' @param out_channels Integer. Output channels. -#' @param latent_channels Integer. Latent space channels. -#' @param block_out_channels Integer vector. Encoder block channels. -#' @param decoder_block_out_channels Integer vector. Decoder block channels. -#' @param layers_per_block Integer vector. Encoder layers per block. -#' @param decoder_layers_per_block Integer vector. Decoder layers per block. -#' @param spatio_temporal_scaling Logical vector. Encoder scaling. -#' @param decoder_spatio_temporal_scaling Logical vector. Decoder scaling. -#' @param decoder_inject_noise Logical vector. Noise injection per decoder block. -#' @param downsample_type Character vector. Downsampling types. -#' @param upsample_residual Logical vector. Upsampler residual flags. -#' @param upsample_factor Integer vector. Upsampler factors. -#' @param timestep_conditioning Logical. Whether to use timestep conditioning. -#' @param patch_size Integer. Spatial patch size. -#' @param patch_size_t Integer. Temporal patch size. -#' @param resnet_norm_eps Numeric. Normalization epsilon. -#' @param scaling_factor Numeric. Latent scaling factor. -#' @param encoder_causal Logical. Encoder causality. -#' @param decoder_causal Logical. Decoder causality. -#' @param encoder_spatial_padding_mode Character. Encoder padding mode. -#' @param decoder_spatial_padding_mode Character. Decoder padding mode. -#' @export -ltx2_video_vae <- torch::nn_module( - "AutoencoderKLLTX2Video", - - initialize = function( - in_channels = 3L, - out_channels = 3L, - latent_channels = 128L, - block_out_channels = c(256L, 512L, 1024L, 2048L), - decoder_block_out_channels = c(256L, 512L, 1024L), - layers_per_block = c(4L, 6L, 6L, 2L, 2L), - decoder_layers_per_block = c(5L, 5L, 5L, 5L), - spatio_temporal_scaling = c(TRUE, TRUE, TRUE, TRUE), - decoder_spatio_temporal_scaling = c(TRUE, TRUE, TRUE), - decoder_inject_noise = c(FALSE, FALSE, FALSE, FALSE), - downsample_type = c("spatial", "temporal", "spatiotemporal", "spatiotemporal"), - upsample_residual = c(TRUE, TRUE, TRUE), - upsample_factor = c(2L, 2L, 2L), - timestep_conditioning = FALSE, - patch_size = 4L, - patch_size_t = 1L, - resnet_norm_eps = 1e-6, - scaling_factor = 1.0, - encoder_causal = TRUE, - decoder_causal = TRUE, - encoder_spatial_padding_mode = "zeros", - decoder_spatial_padding_mode = "reflect" - ) { - self$encoder <- ltx2_video_encoder3d( - in_channels = in_channels, - out_channels = latent_channels, - block_out_channels = block_out_channels, - spatio_temporal_scaling = spatio_temporal_scaling, - layers_per_block = layers_per_block, - downsample_type = downsample_type, - patch_size = patch_size, - patch_size_t = patch_size_t, - resnet_norm_eps = resnet_norm_eps, - is_causal = encoder_causal, - spatial_padding_mode = encoder_spatial_padding_mode - ) - - self$decoder <- ltx2_video_decoder3d( - in_channels = latent_channels, - out_channels = out_channels, - block_out_channels = decoder_block_out_channels, - spatio_temporal_scaling = decoder_spatio_temporal_scaling, - layers_per_block = decoder_layers_per_block, - patch_size = patch_size, - patch_size_t = patch_size_t, - resnet_norm_eps = resnet_norm_eps, - is_causal = decoder_causal, - inject_noise = decoder_inject_noise, - timestep_conditioning = timestep_conditioning, - upsample_residual = upsample_residual, - upsample_factor = upsample_factor, - spatial_padding_mode = decoder_spatial_padding_mode - ) - - # Latent normalization buffers - self$latents_mean <- torch::nn_buffer(torch::torch_zeros(latent_channels)) - self$latents_std <- torch::nn_buffer(torch::torch_ones(latent_channels)) - - # Compression ratios - self$spatial_compression_ratio <- patch_size * 2 ^ sum(spatio_temporal_scaling) - self$temporal_compression_ratio <- patch_size_t * 2 ^ sum(spatio_temporal_scaling) - - self$scaling_factor <- scaling_factor - - # Tiling configuration (GPU-poor support) - self$use_slicing <- FALSE - self$use_tiling <- FALSE - self$use_framewise_encoding <- FALSE - self$use_framewise_decoding <- FALSE - - self$num_sample_frames_batch_size <- 16L - self$num_latent_frames_batch_size <- 2L - - self$tile_sample_min_height <- 512L - self$tile_sample_min_width <- 512L - self$tile_sample_min_num_frames <- 16L - - self$tile_sample_stride_height <- 448L - self$tile_sample_stride_width <- 448L - self$tile_sample_stride_num_frames <- 8L - }, - -#' Enable tiled encoding/decoding for memory efficiency - enable_tiling = function( - tile_sample_min_height = NULL, - tile_sample_min_width = NULL, - tile_sample_min_num_frames = NULL, - tile_sample_stride_height = NULL, - tile_sample_stride_width = NULL, - tile_sample_stride_num_frames = NULL - ) { - self$use_tiling <- TRUE - if (!is.null(tile_sample_min_height)) self$tile_sample_min_height <- tile_sample_min_height - if (!is.null(tile_sample_min_width)) self$tile_sample_min_width <- tile_sample_min_width - if (!is.null(tile_sample_min_num_frames)) self$tile_sample_min_num_frames <- tile_sample_min_num_frames - if (!is.null(tile_sample_stride_height)) self$tile_sample_stride_height <- tile_sample_stride_height - if (!is.null(tile_sample_stride_width)) self$tile_sample_stride_width <- tile_sample_stride_width - if (!is.null(tile_sample_stride_num_frames)) self$tile_sample_stride_num_frames <- tile_sample_stride_num_frames - invisible(self) - }, - -#' Disable tiling - disable_tiling = function() { - self$use_tiling <- FALSE - invisible(self) - }, - -#' Enable framewise decoding for long videos - enable_framewise_decoding = function() { - self$use_framewise_decoding <- TRUE - invisible(self) - }, - -#' Blend tiles vertically - blend_v = function( - a, - b, - blend_extent - ) { - blend_extent <- min(a$shape[4], b$shape[4], blend_extent) - for (y in seq_len(blend_extent)) { - weight <- (y - 1) / blend_extent - idx <- as.integer(a$shape[4] - blend_extent + y) - b[,,, y,] <- a[,,, idx,] * (1 - weight) + b[,,, y,] * weight - } - b - }, - -#' Blend tiles horizontally - blend_h = function( - a, - b, - blend_extent - ) { - blend_extent <- min(a$shape[5], b$shape[5], blend_extent) - for (x in seq_len(blend_extent)) { - weight <- (x - 1) / blend_extent - idx <- as.integer(a$shape[5] - blend_extent + x) - b[,,,, x] <- a[,,,, idx] * (1 - weight) + b[,,,, x] * weight - } - b - }, - -#' Blend tiles temporally - blend_t = function( - a, - b, - blend_extent - ) { - blend_extent <- min(a$shape[3], b$shape[3], blend_extent) - for (t in seq_len(blend_extent)) { - weight <- (t - 1) / blend_extent - idx <- as.integer(a$shape[3] - blend_extent + t) - b[,, t,,] <- a[,, idx,,] * (1 - weight) + b[,, t,,] * weight - } - b - }, - -#' Tiled encoding for large spatial dimensions - tiled_encode = function( - x, - causal = NULL - ) { - batch_size <- x$shape[1] - num_channels <- x$shape[2] - num_frames <- x$shape[3] - height <- x$shape[4] - width <- x$shape[5] - - latent_height <- height %/% self$spatial_compression_ratio - latent_width <- width %/% self$spatial_compression_ratio - - tile_latent_min_height <- self$tile_sample_min_height %/% self$spatial_compression_ratio - tile_latent_min_width <- self$tile_sample_min_width %/% self$spatial_compression_ratio - tile_latent_stride_height <- self$tile_sample_stride_height %/% self$spatial_compression_ratio - tile_latent_stride_width <- self$tile_sample_stride_width %/% self$spatial_compression_ratio - - blend_height <- tile_latent_min_height - tile_latent_stride_height - blend_width <- tile_latent_min_width - tile_latent_stride_width - - # Encode tiles - rows <- list() - for (i in seq(1, height, by = self$tile_sample_stride_height)) { - row <- list() - for (j in seq(1, width, by = self$tile_sample_stride_width)) { - i_end <- min(i + self$tile_sample_min_height - 1, height) - j_end <- min(j + self$tile_sample_min_width - 1, width) - tile <- self$encoder(x[,,, i:i_end, j:j_end], causal = causal) - row[[length(row) + 1]] <- tile - } - rows[[length(rows) + 1]] <- row - } - - # Blend tiles - result_rows <- list() - for (i in seq_along(rows)) { - result_row <- list() - for (j in seq_along(rows[[i]])) { - tile <- rows[[i]][[j]] - if (i > 1) { - tile <- self$blend_v(rows[[i - 1]][[j]], tile, blend_height) - } - if (j > 1) { - tile <- self$blend_h(rows[[i]][[j - 1]], tile, blend_width) - } - result_row[[length(result_row) + 1]] <- tile[,,, 1:tile_latent_stride_height, 1:tile_latent_stride_width] - } - result_rows[[length(result_rows) + 1]] <- torch::torch_cat(result_row, dim = 5) - } - - enc <- torch::torch_cat(result_rows, dim = 4)[,,, 1:latent_height, 1:latent_width] - enc - }, - -#' Tiled decoding for large spatial dimensions - tiled_decode = function( - z, - temb = NULL, - causal = NULL - ) { - batch_size <- z$shape[1] - num_channels <- z$shape[2] - num_frames <- z$shape[3] - height <- z$shape[4] - width <- z$shape[5] - - sample_height <- height * self$spatial_compression_ratio - sample_width <- width * self$spatial_compression_ratio - - tile_latent_min_height <- self$tile_sample_min_height %/% self$spatial_compression_ratio - tile_latent_min_width <- self$tile_sample_min_width %/% self$spatial_compression_ratio - tile_latent_stride_height <- self$tile_sample_stride_height %/% self$spatial_compression_ratio - tile_latent_stride_width <- self$tile_sample_stride_width %/% self$spatial_compression_ratio - - blend_height <- self$tile_sample_min_height - self$tile_sample_stride_height - blend_width <- self$tile_sample_min_width - self$tile_sample_stride_width - - # Decode tiles - rows <- list() - for (i in seq(1, height, by = tile_latent_stride_height)) { - row <- list() - for (j in seq(1, width, by = tile_latent_stride_width)) { - i_end <- min(i + tile_latent_min_height - 1, height) - j_end <- min(j + tile_latent_min_width - 1, width) - tile <- self$decoder(z[,,, i:i_end, j:j_end], temb, causal = causal) - row[[length(row) + 1]] <- tile - } - rows[[length(rows) + 1]] <- row - } - - # Blend tiles - result_rows <- list() - for (i in seq_along(rows)) { - result_row <- list() - for (j in seq_along(rows[[i]])) { - tile <- rows[[i]][[j]] - if (i > 1) { - tile <- self$blend_v(rows[[i - 1]][[j]], tile, blend_height) - } - if (j > 1) { - tile <- self$blend_h(rows[[i]][[j - 1]], tile, blend_width) - } - result_row[[length(result_row) + 1]] <- tile[,,, 1:self$tile_sample_stride_height, 1:self$tile_sample_stride_width] - } - result_rows[[length(result_rows) + 1]] <- torch::torch_cat(result_row, dim = 5) - } - - dec <- torch::torch_cat(result_rows, dim = 4)[,,, 1:sample_height, 1:sample_width] - dec - }, - -#' Internal encode with tiling support - .encode = function( - x, - causal = NULL - ) { - batch_size <- x$shape[1] - num_channels <- x$shape[2] - num_frames <- x$shape[3] - height <- x$shape[4] - width <- x$shape[5] - - if (self$use_tiling && (width > self$tile_sample_min_width || height > self$tile_sample_min_height)) { - return(self$tiled_encode(x, causal = causal)) - } - - self$encoder(x, causal = causal) - }, - -#' Encode video to latent space - encode = function( - x, - causal = NULL - ) { - if (self$use_slicing && x$shape[1] > 1) { - encoded_slices <- lapply(seq_len(x$shape[1]), function(i) { - self$.encode(x[i:i,,,,, drop = FALSE], causal = causal) - }) - h <- torch::torch_cat(encoded_slices, dim = 1) - } else { - h <- self$.encode(x, causal = causal) - } - diagonal_gaussian_distribution(h) - }, - -#' Internal decode with tiling support - .decode = function( - z, - temb = NULL, - causal = NULL - ) { - batch_size <- z$shape[1] - num_channels <- z$shape[2] - num_frames <- z$shape[3] - height <- z$shape[4] - width <- z$shape[5] - - tile_latent_min_height <- self$tile_sample_min_height %/% self$spatial_compression_ratio - tile_latent_min_width <- self$tile_sample_min_width %/% self$spatial_compression_ratio - - if (self$use_tiling && (width > tile_latent_min_width || height > tile_latent_min_height)) { - return(self$tiled_decode(z, temb, causal = causal)) - } - - self$decoder(z, temb, causal = causal) - }, - -#' Decode latent to video - decode = function( - z, - temb = NULL, - causal = NULL - ) { - if (self$use_slicing && z$shape[1] > 1) { - if (!is.null(temb)) { - decoded_slices <- lapply(seq_len(z$shape[1]), function(i) { - self$.decode(z[i:i,,,,, drop = FALSE], temb[i:i, drop = FALSE], causal = causal) - }) - } else { - decoded_slices <- lapply(seq_len(z$shape[1]), function(i) { - self$.decode(z[i:i,,,,, drop = FALSE], causal = causal) - }) - } - torch::torch_cat(decoded_slices, dim = 1) - } else { - self$.decode(z, temb, causal = causal) - } - }, - -#' Full forward pass (encode -> sample/mode -> decode) - forward = function( - sample, - temb = NULL, - sample_posterior = FALSE, - encoder_causal = NULL, - decoder_causal = NULL, - generator = NULL - ) { - posterior <- self$encode(sample, causal = encoder_causal) - if (sample_posterior) { - z <- posterior$sample(generator) - } else { - z <- posterior$mode() - } - self$decode(z, temb, causal = decoder_causal) - } -) - -#' Load LTX2 Video VAE from safetensors -#' -#' Load pre-trained LTX2 VAE weights from a HuggingFace safetensors file. -#' -#' @param weights_path Character. Path to safetensors file or directory containing weights. -#' @param config_path Character. Optional path to config.json. If NULL and weights_path -#' is a directory, looks for config.json in that directory. Otherwise uses default config. -#' @param device Character. Device to load weights to. Default: "cpu" -#' @param dtype Character or torch dtype. Data type. Default: "float32" -#' @param verbose Logical. Print loading progress. Default: TRUE -#' @return Initialized ltx2_video_vae module -#' @export -load_ltx2_vae <- function( - weights_path, - config_path = NULL, - device = "cpu", - dtype = "float32", - verbose = TRUE -) { - if (!file.exists(weights_path)) { - stop("Weights path not found: ", weights_path) - } - - # Handle directory vs file - if (dir.exists(weights_path)) { - vae_dir <- weights_path - # Look for config.json - if (is.null(config_path)) { - config_path <- file.path(vae_dir, "config.json") - if (!file.exists(config_path)) config_path <- NULL - } - # Look for weights file - weights_file <- file.path(vae_dir, "diffusion_pytorch_model.safetensors") - if (!file.exists(weights_file)) { - stop("Could not find diffusion_pytorch_model.safetensors in: ", vae_dir) - } - weights_path <- weights_file - } - - # Load config if provided - config <- NULL - if (!is.null(config_path) && file.exists(config_path)) { - config <- jsonlite::fromJSON(config_path) - if (verbose) message("Loaded config from: ", config_path) - } - - # Create VAE with config or defaults (matching HuggingFace LTX-2) - if (!is.null(config)) { - vae <- ltx2_video_vae( - in_channels = config$in_channels %||% 3L, - out_channels = config$out_channels %||% 3L, - latent_channels = config$latent_channels %||% 128L, - block_out_channels = as.integer(config$block_out_channels %||% c(256, 512, 1024, 2048)), - decoder_block_out_channels = as.integer(config$decoder_block_out_channels %||% c(256, 512, 1024)), - layers_per_block = as.integer(config$layers_per_block %||% c(4, 6, 6, 2, 2)), - decoder_layers_per_block = as.integer(config$decoder_layers_per_block %||% c(5, 5, 5, 5)), - spatio_temporal_scaling = as.logical(config$spatio_temporal_scaling %||% c(TRUE, TRUE, TRUE, TRUE)), - decoder_spatio_temporal_scaling = as.logical(config$decoder_spatio_temporal_scaling %||% c(TRUE, TRUE, TRUE)), - decoder_inject_noise = as.logical(config$decoder_inject_noise %||% c(FALSE, FALSE, FALSE, FALSE)), - downsample_type = config$downsample_type %||% c("spatial", "temporal", "spatiotemporal", "spatiotemporal"), - upsample_residual = as.logical(config$upsample_residual %||% c(TRUE, TRUE, TRUE)), - upsample_factor = as.integer(config$upsample_factor %||% c(2, 2, 2)), - timestep_conditioning = config$timestep_conditioning %||% FALSE, - patch_size = config$patch_size %||% 4L, - patch_size_t = config$patch_size_t %||% 1L, - resnet_norm_eps = config$resnet_norm_eps %||% 1e-6, - scaling_factor = config$scaling_factor %||% 1.0, - encoder_causal = config$encoder_causal %||% TRUE, - decoder_causal = config$decoder_causal %||% FALSE, - encoder_spatial_padding_mode = config$encoder_spatial_padding_mode %||% "zeros", - decoder_spatial_padding_mode = config$decoder_spatial_padding_mode %||% "reflect" - ) - } else { - vae <- ltx2_video_vae() - } - - # Load weights - if (verbose) message("Loading weights from: ", weights_path) - weights <- safetensors::safe_load_file(weights_path, framework = "torch") - - # Load weights into VAE - load_ltx2_vae_weights(vae, weights, verbose = verbose) - - # Move to device with dtype - if (dtype == "float16") { - torch_dtype <- torch::torch_float16() - } else { - torch_dtype <- torch::torch_float32() - } - vae$to(device = device, dtype = torch_dtype) - - if (verbose) message("VAE loaded successfully on device: ", device) - vae -} - -#' Load weights into LTX2 VAE module -#' -#' Maps HuggingFace safetensors parameter names to R module parameters. -#' -#' @param vae LTX2 VAE module -#' @param weights Named list of weight tensors from safetensors -#' @param verbose Logical. Print loading progress -#' @return The VAE with loaded weights (invisibly) -#' @keywords internal -load_ltx2_vae_weights <- function( - vae, - weights, - verbose = TRUE -) { - # Get native parameter names - native_params <- names(vae$parameters) - - # Build mapping from HF names to R names - remap_vae_key <- function(key) { - # HuggingFace VAE naming: - # encoder.conv_in.conv.weight -> encoder.conv_in.conv.weight - # encoder.down_blocks.0.resnets.0.norm.weight -> encoder.down_blocks.0.resnets.0.norm.weight - # The naming should be mostly 1:1 with our R module structure - - # No remapping needed for most keys - HF uses same structure - key - } - - loaded <- 0L - skipped <- 0L - unmapped <- character(0) - - torch::with_no_grad({ - for (hf_name in names(weights)) { - native_name <- remap_vae_key(hf_name) - - if (native_name %in% native_params) { - hf_tensor <- weights[[hf_name]] - native_tensor <- vae$parameters[[native_name]] - - if (all(as.integer(hf_tensor$shape) == as.integer(native_tensor$shape))) { - native_tensor$copy_(hf_tensor) - loaded <- loaded + 1L - } else { - if (verbose) { - message("Shape mismatch: ", native_name, - " (HF: ", paste(as.integer(hf_tensor$shape), collapse = "x"), - " vs R: ", paste(as.integer(native_tensor$shape), collapse = "x"), ")") - } - skipped <- skipped + 1L - } - } else { - skipped <- skipped + 1L - unmapped <- c(unmapped, paste0(hf_name, " -> ", native_name)) - } - } - }) - - if (verbose) { - message(sprintf("VAE weights: %d loaded, %d skipped", loaded, skipped)) - if (length(unmapped) > 0 && length(unmapped) <= 20) { - message("Unmapped parameters:") - for (u in unmapped[1:min(20, length(unmapped))]) { - message(" ", u) - } - } - if (length(unmapped) > 20) { - message(" ... and ", length(unmapped) - 20, " more") - } - } - - invisible(vae) -} - -#' Null-coalescing operator -#' @keywords internal -`%||%` <- function( - x, - y -) if (is.null(x)) y else x - diff --git a/R/vae_ltx2_modules.R b/R/vae_ltx2_modules.R deleted file mode 100644 index 6706595..0000000 --- a/R/vae_ltx2_modules.R +++ /dev/null @@ -1,746 +0,0 @@ -#' LTX2 Video VAE Module Building Blocks -#' -#' Low-level nn_module components for the LTX2 3D causal VAE. -#' Used by vae_ltx2.R encoder/decoder. -#' -#' @name vae_ltx2_modules -NULL - -#' Per-channel RMS normalization -#' -#' Normalizes tensor by root-mean-square along the channel dimension: -#' y = x / sqrt(mean(x^2, dim=channel_dim, keepdim=TRUE) + eps) -#' -#' @param channel_dim Integer. Dimension for RMS computation (1-indexed). Default: 2 (channels) -#' @param eps Numeric. Small constant for numerical stability. Default: 1e-8 -#' @export -per_channel_rms_norm <- torch::nn_module( - - "PerChannelRMSNorm", - - initialize = function( - channel_dim = 2L, - eps = 1e-8 - ) { - self$channel_dim <- channel_dim - self$eps <- eps - }, - - forward = function( - x, - channel_dim = NULL - ) { - if (is.null(channel_dim)) { - channel_dim <- self$channel_dim - } - mean_sq <- torch::torch_mean(x ^ 2, dim = channel_dim, keepdim = TRUE) - rms <- torch::torch_sqrt(mean_sq + self$eps) - x / rms - } -) - -#' LTX2 Video Causal 3D Convolution -#' -#' 3D convolution with runtime-selectable causal or non-causal padding. -#' Causal mode pads temporally by repeating first frame. -#' Non-causal mode pads temporally by repeating first and last frames. -#' -#' @param in_channels Integer. Input channels. -#' @param out_channels Integer. Output channels. -#' @param kernel_size Integer or vector of 3. Convolution kernel size. -#' @param stride Integer or vector of 3. Stride. -#' @param dilation Integer or vector of 3. Dilation. -#' @param groups Integer. Convolution groups. Default: 1 -#' @param spatial_padding_mode Character. Padding mode for spatial dims. Default: "zeros" -#' @export -ltx2_video_causal_conv3d <- torch::nn_module( - "LTX2VideoCausalConv3d", - - initialize = function( - in_channels, - out_channels, - kernel_size = 3L, - stride = 1L, - dilation = 1L, - groups = 1L, - spatial_padding_mode = "zeros" - ) { - self$in_channels <- in_channels - self$out_channels <- out_channels - - # Normalize kernel_size to tuple of 3 - if (length(kernel_size) == 1) { - self$kernel_size <- rep(as.integer(kernel_size), 3) - } else { - self$kernel_size <- as.integer(kernel_size) - } - - # Normalize dilation (default temporal dilation only) - if (length(dilation) == 1) { - dilation <- c(as.integer(dilation), 1L, 1L) - } - - # Normalize stride - if (length(stride) == 1) { - stride <- rep(as.integer(stride), 3) - } - - # Spatial padding (no temporal padding in the conv itself) - height_pad <- self$kernel_size[2] %/% 2L - width_pad <- self$kernel_size[3] %/% 2L - padding <- c(0L, height_pad, width_pad) - - self$conv <- torch::nn_conv3d( - in_channels = in_channels, - out_channels = out_channels, - kernel_size = self$kernel_size, - stride = stride, - dilation = dilation, - groups = groups, - padding = padding, - padding_mode = spatial_padding_mode - ) - }, - - forward = function( - hidden_states, - causal = TRUE - ) { - time_kernel_size <- self$kernel_size[1] - - if (causal) { - # Causal: pad by repeating first frame on left - pad_left <- hidden_states[,, 1:1,,]$`repeat`(c(1L, 1L, time_kernel_size - 1L, 1L, 1L)) - hidden_states <- torch::torch_cat(list(pad_left, hidden_states), dim = 3) - } else { - # Non-causal: pad by repeating first/last frames symmetrically - pad_amount <- (time_kernel_size - 1L) %/% 2L - if (pad_amount > 0) { - pad_left <- hidden_states[,, 1:1,,]$`repeat`(c(1L, 1L, pad_amount, 1L, 1L)) - n_frames <- hidden_states$shape[3] - pad_right <- hidden_states[,, n_frames:n_frames,,]$`repeat`(c(1L, 1L, pad_amount, 1L, 1L)) - hidden_states <- torch::torch_cat(list(pad_left, hidden_states, pad_right), dim = 3) - } - } - - self$conv(hidden_states) - } -) - -#' LTX2 Video ResNet Block 3D -#' -#' 3D ResNet block with per-channel RMS normalization and optional -#' noise injection and timestep conditioning. -#' -#' @param in_channels Integer. Input channels. -#' @param out_channels Integer or NULL. Output channels (defaults to in_channels). -#' @param dropout Numeric. Dropout rate. Default: 0.0 -#' @param eps Numeric. Epsilon for normalization. Default: 1e-6 -#' @param non_linearity Character. Activation function. Default: "silu" -#' @param inject_noise Logical. Whether to inject noise. Default: FALSE -#' @param timestep_conditioning Logical. Whether to use timestep conditioning. Default: FALSE -#' @param spatial_padding_mode Character. Padding mode. Default: "zeros" -#' @export -ltx2_video_resnet_block3d <- torch::nn_module( - "LTX2VideoResnetBlock3d", - - initialize = function( - in_channels, - out_channels = NULL, - dropout = 0.0, - eps = 1e-6, - non_linearity = "silu", - inject_noise = FALSE, - timestep_conditioning = FALSE, - spatial_padding_mode = "zeros" - ) { - if (is.null(out_channels)) out_channels <- in_channels - - # Activation - self$nonlinearity <- if (non_linearity == "silu" || non_linearity == "swish") { - torch::nn_silu() - } else if (non_linearity == "gelu") { - torch::nn_gelu() - } else { - torch::nn_relu() - } - - self$norm1 <- per_channel_rms_norm() - self$conv1 <- ltx2_video_causal_conv3d( - in_channels = in_channels, - out_channels = out_channels, - kernel_size = 3L, - spatial_padding_mode = spatial_padding_mode - ) - - self$norm2 <- per_channel_rms_norm() - self$dropout <- torch::nn_dropout(p = dropout) - self$conv2 <- ltx2_video_causal_conv3d( - in_channels = out_channels, - out_channels = out_channels, - kernel_size = 3L, - spatial_padding_mode = spatial_padding_mode - ) - - # Shortcut connection if channels differ - self$norm3 <- NULL - self$conv_shortcut <- NULL - if (in_channels != out_channels) { - self$norm3 <- torch::nn_layer_norm(in_channels, eps = eps, elementwise_affine = TRUE) - # Regular Conv3d for shortcut (not causal) - self$conv_shortcut <- torch::nn_conv3d( - in_channels = in_channels, - out_channels = out_channels, - kernel_size = 1L, - stride = 1L - ) - } - - # Noise injection (optional) - self$per_channel_scale1 <- NULL - self$per_channel_scale2 <- NULL - if (inject_noise) { - self$per_channel_scale1 <- torch::nn_parameter(torch::torch_zeros(c(in_channels, 1, 1))) - self$per_channel_scale2 <- torch::nn_parameter(torch::torch_zeros(c(in_channels, 1, 1))) - } - - # Timestep conditioning (optional) - self$scale_shift_table <- NULL - if (timestep_conditioning) { - init_table <- torch::torch_randn(c(4, in_channels)) / sqrt(in_channels) - self$scale_shift_table <- torch::nn_parameter(init_table) - } - - self$in_channels <- in_channels - self$out_channels <- out_channels - }, - - forward = function( - inputs, - temb = NULL, - generator = NULL, - causal = TRUE - ) { - hidden_states <- inputs - - hidden_states <- self$norm1(hidden_states) - - # Timestep conditioning (shift/scale) - shift_1 <- NULL - scale_1 <- NULL - shift_2 <- NULL - scale_2 <- NULL - if (!is.null(self$scale_shift_table) && !is.null(temb)) { - # temb shape: [B, C*4, 1, 1, 1] -> unflatten to [B, 4, C, 1, 1, 1] - temb <- temb$unflatten(2, c(4, - 1)) + self$scale_shift_table[NULL, .., NULL, NULL, NULL] - splits <- temb$unbind(dim = 2) - shift_1 <- splits[[1]] - scale_1 <- splits[[2]] - shift_2 <- splits[[3]] - scale_2 <- splits[[4]] - hidden_states <- hidden_states * (1 + scale_1) + shift_1 - } - - hidden_states <- self$nonlinearity(hidden_states) - hidden_states <- self$conv1(hidden_states, causal = causal) - - # Noise injection after conv1 - if (!is.null(self$per_channel_scale1)) { - h <- hidden_states$shape[4] - w <- hidden_states$shape[5] - spatial_noise <- torch::torch_randn(c(h, w), device = hidden_states$device, - dtype = hidden_states$dtype)$unsqueeze(1) - hidden_states <- hidden_states + (spatial_noise * self$per_channel_scale1)[NULL,, NULL,,] - } - - hidden_states <- self$norm2(hidden_states) - - # Second timestep conditioning - if (!is.null(self$scale_shift_table) && !is.null(temb)) { - hidden_states <- hidden_states * (1 + scale_2) + shift_2 - } - - hidden_states <- self$nonlinearity(hidden_states) - hidden_states <- self$dropout(hidden_states) - hidden_states <- self$conv2(hidden_states, causal = causal) - - # Noise injection after conv2 - if (!is.null(self$per_channel_scale2)) { - h <- hidden_states$shape[4] - w <- hidden_states$shape[5] - spatial_noise <- torch::torch_randn(c(h, w), device = hidden_states$device, - dtype = hidden_states$dtype)$unsqueeze(1) - hidden_states <- hidden_states + (spatial_noise * self$per_channel_scale2)[NULL,, NULL,,] - } - - # Shortcut connection - if (!is.null(self$norm3)) { - # LayerNorm expects last dim to be features, so move channels to last - inputs <- inputs$movedim(2, - 1) - inputs <- self$norm3(inputs) - inputs <- inputs$movedim(- 1, 2) - } - - if (!is.null(self$conv_shortcut)) { - inputs <- self$conv_shortcut(inputs) - } - - hidden_states <- hidden_states + inputs - hidden_states - } -) - -#' LTX Video Downsampler 3D -#' -#' Spatiotemporal downsampling with strided pixel unshuffle + convolution. -#' -#' @param in_channels Integer. Input channels. -#' @param out_channels Integer. Output channels. -#' @param stride Integer or vector of 3. Downsampling stride. -#' @param spatial_padding_mode Character. Padding mode. -#' @export -ltx_video_downsampler3d <- torch::nn_module( - "LTXVideoDownsampler3d", - - initialize = function( - in_channels, - out_channels, - stride = 1L, - spatial_padding_mode = "zeros" - ) { - if (length(stride) == 1) { - self$stride <- rep(as.integer(stride), 3) - } else { - self$stride <- as.integer(stride) - } - - # Calculate group size for averaging residual - stride_prod <- self$stride[1] * self$stride[2] * self$stride[3] - self$group_size <- (in_channels * stride_prod) %/% out_channels - - # Output channels after pixel unshuffle - conv_out_channels <- out_channels %/% stride_prod - - self$conv <- ltx2_video_causal_conv3d( - in_channels = in_channels, - out_channels = conv_out_channels, - kernel_size = 3L, - stride = 1L, - spatial_padding_mode = spatial_padding_mode - ) - }, - - forward = function( - hidden_states, - causal = TRUE - ) { - # Pad temporal dimension to handle stride - # cat with first (stride[1]-1) frames repeated - if (self$stride[1] > 1) { - pad_frames <- hidden_states[,, 1:(self$stride[1] - 1),,] - hidden_states <- torch::torch_cat(list(pad_frames, hidden_states), dim = 3) - } - - # Compute residual via pixel unshuffle pattern - # unflatten spatial dims, permute, flatten to channel-like - residual <- hidden_states$unflatten(5, c(- 1, self$stride[3])) # width - residual <- residual$unflatten(4, c(- 1, self$stride[2])) # height - residual <- residual$unflatten(3, c(- 1, self$stride[1])) # frames - - # Permute: [B, C, F//s, s, H//s, s, W//s, s] -> [B, C, s, s, s, F//s, H//s, W//s] - residual <- residual$permute(c(1, 2, 4, 6, 8, 3, 5, 7)) - residual <- residual$flatten(start_dim = 2, end_dim = 5) # [B, C*s^3, F', H', W'] - residual <- residual$unflatten(2, c(- 1, self$group_size)) # [B, out_c, group_size, F', H', W'] - residual <- residual$mean(dim = 3) # Average over groups - - # Convolution path - hidden_states <- self$conv(hidden_states, causal = causal) - - # Same pixel unshuffle for conv output - hidden_states <- hidden_states$unflatten(5, c(- 1, self$stride[3])) - hidden_states <- hidden_states$unflatten(4, c(- 1, self$stride[2])) - hidden_states <- hidden_states$unflatten(3, c(- 1, self$stride[1])) - hidden_states <- hidden_states$permute(c(1, 2, 4, 6, 8, 3, 5, 7)) - hidden_states <- hidden_states$flatten(start_dim = 2, end_dim = 5) - - hidden_states <- hidden_states + residual - hidden_states - } -) - -#' LTX Video Upsampler 3D -#' -#' Spatiotemporal upsampling with pixel shuffle + optional residual. -#' -#' @param in_channels Integer. Input channels. -#' @param stride Integer or vector of 3. Upsampling stride. -#' @param residual Logical. Whether to use residual connection. -#' @param upscale_factor Integer. Channel upscale factor. -#' @param spatial_padding_mode Character. Padding mode. -#' @export -ltx_video_upsampler3d <- torch::nn_module( - "LTXVideoUpsampler3d", - - initialize = function( - in_channels, - stride = 1L, - residual = FALSE, - upscale_factor = 1L, - spatial_padding_mode = "zeros" - ) { - if (length(stride) == 1) { - self$stride <- rep(as.integer(stride), 3) - } else { - self$stride <- as.integer(stride) - } - self$residual <- residual - self$upscale_factor <- upscale_factor - - stride_prod <- self$stride[1] * self$stride[2] * self$stride[3] - out_channels <- (in_channels * stride_prod) %/% upscale_factor - - self$conv <- ltx2_video_causal_conv3d( - in_channels = in_channels, - out_channels = out_channels, - kernel_size = 3L, - stride = 1L, - spatial_padding_mode = spatial_padding_mode - ) - }, - - forward = function( - hidden_states, - causal = TRUE - ) { - batch_size <- hidden_states$shape[1] - num_channels <- hidden_states$shape[2] - num_frames <- hidden_states$shape[3] - height <- hidden_states$shape[4] - width <- hidden_states$shape[5] - - stride_prod <- self$stride[1] * self$stride[2] * self$stride[3] - - # Residual path - residual_out <- NULL - if (self$residual) { - # Reshape for pixel shuffle: [B, C, F, H, W] -> [B, C//s^3, s, s, s, F, H, W] - residual_out <- hidden_states$reshape(c( - batch_size, - - 1, - self$stride[1], self$stride[2], self$stride[3], - num_frames, height, width - )) - # Permute: [B, C', s_t, s_h, s_w, F, H, W] -> [B, C', F, s_t, H, s_h, W, s_w] - residual_out <- residual_out$permute(c(1, 2, 6, 3, 7, 4, 8, 5)) - # Flatten spatial dims - residual_out <- residual_out$flatten(start_dim = 7, end_dim = 8) # W * s_w - residual_out <- residual_out$flatten(start_dim = 5, end_dim = 6) # H * s_h - residual_out <- residual_out$flatten(start_dim = 3, end_dim = 4) # F * s_t - - # Repeat channels for upscale factor - repeats <- stride_prod %/% self$upscale_factor - residual_out <- residual_out$`repeat`(c(1L, repeats, 1L, 1L, 1L)) - # Remove first (stride[1]-1) frames - residual_out <- residual_out[,, self$stride[1]:N,,] - } - - # Convolution path - hidden_states <- self$conv(hidden_states, causal = causal) - - # Pixel shuffle - hidden_states <- hidden_states$reshape(c( - batch_size, - - 1, - self$stride[1], self$stride[2], self$stride[3], - num_frames, height, width - )) - hidden_states <- hidden_states$permute(c(1, 2, 6, 3, 7, 4, 8, 5)) - hidden_states <- hidden_states$flatten(start_dim = 7, end_dim = 8) - hidden_states <- hidden_states$flatten(start_dim = 5, end_dim = 6) - hidden_states <- hidden_states$flatten(start_dim = 3, end_dim = 4) - # Remove first (stride[1]-1) frames - hidden_states <- hidden_states[,, self$stride[1]:N,,] - - if (self$residual && !is.null(residual_out)) { - hidden_states <- hidden_states + residual_out - } - - hidden_states - } -) - -#' LTX2 Video Down Block 3D -#' -#' Encoder down block with multiple ResNet layers and optional downsampling. -#' -#' @param in_channels Integer. Input channels. -#' @param out_channels Integer or NULL. Output channels. -#' @param num_layers Integer. Number of ResNet layers. -#' @param dropout Numeric. Dropout rate. -#' @param resnet_eps Numeric. Epsilon for normalization. -#' @param resnet_act_fn Character. Activation function. -#' @param spatio_temporal_scale Logical. Whether to use downsampling. -#' @param downsample_type Character. Type of downsampling. -#' @param spatial_padding_mode Character. Padding mode. -#' @export -ltx2_video_down_block3d <- torch::nn_module( - "LTX2VideoDownBlock3D", - - initialize = function( - in_channels, - out_channels = NULL, - num_layers = 1L, - dropout = 0.0, - resnet_eps = 1e-6, - resnet_act_fn = "swish", - spatio_temporal_scale = TRUE, - downsample_type = "conv", - spatial_padding_mode = "zeros" - ) { - if (is.null(out_channels)) out_channels <- in_channels - - # ResNet layers - resnets <- list() - for (i in seq_len(num_layers)) { - resnets[[i]] <- ltx2_video_resnet_block3d( - in_channels = in_channels, - out_channels = in_channels, # Note: same channels within block - dropout = dropout, - eps = resnet_eps, - non_linearity = resnet_act_fn, - spatial_padding_mode = spatial_padding_mode - ) - } - self$resnets <- torch::nn_module_list(resnets) - - # Downsampler - self$downsamplers <- NULL - if (spatio_temporal_scale) { - if (downsample_type == "conv") { - self$downsamplers <- torch::nn_module_list(list( - ltx2_video_causal_conv3d( - in_channels = in_channels, - out_channels = in_channels, - kernel_size = 3L, - stride = c(2L, 2L, 2L), - spatial_padding_mode = spatial_padding_mode - ) - )) - } else if (downsample_type == "spatial") { - self$downsamplers <- torch::nn_module_list(list( - ltx_video_downsampler3d( - in_channels = in_channels, - out_channels = out_channels, - stride = c(1L, 2L, 2L), - spatial_padding_mode = spatial_padding_mode - ) - )) - } else if (downsample_type == "temporal") { - self$downsamplers <- torch::nn_module_list(list( - ltx_video_downsampler3d( - in_channels = in_channels, - out_channels = out_channels, - stride = c(2L, 1L, 1L), - spatial_padding_mode = spatial_padding_mode - ) - )) - } else if (downsample_type == "spatiotemporal") { - self$downsamplers <- torch::nn_module_list(list( - ltx_video_downsampler3d( - in_channels = in_channels, - out_channels = out_channels, - stride = c(2L, 2L, 2L), - spatial_padding_mode = spatial_padding_mode - ) - )) - } - } - }, - - forward = function( - hidden_states, - temb = NULL, - generator = NULL, - causal = TRUE - ) { - for (i in seq_along(self$resnets)) { - hidden_states <- self$resnets[[i]](hidden_states, temb, generator, causal = causal) - } - - if (!is.null(self$downsamplers)) { - for (i in seq_along(self$downsamplers)) { - hidden_states <- self$downsamplers[[i]](hidden_states, causal = causal) - } - } - - hidden_states - } -) - -#' LTX2 Video Mid Block 3D -#' -#' Middle block with ResNet layers and optional timestep conditioning. -#' -#' @param in_channels Integer. Input channels. -#' @param num_layers Integer. Number of ResNet layers. -#' @param dropout Numeric. Dropout rate. -#' @param resnet_eps Numeric. Epsilon for normalization. -#' @param resnet_act_fn Character. Activation function. -#' @param inject_noise Logical. Whether to inject noise. -#' @param timestep_conditioning Logical. Whether to use timestep conditioning. -#' @param spatial_padding_mode Character. Padding mode. -#' @export -ltx2_video_mid_block3d <- torch::nn_module( - "LTX2VideoMidBlock3d", - - initialize = function( - in_channels, - num_layers = 1L, - dropout = 0.0, - resnet_eps = 1e-6, - resnet_act_fn = "swish", - inject_noise = FALSE, - timestep_conditioning = FALSE, - spatial_padding_mode = "zeros" - ) { - # Time embedder for timestep conditioning (TODO: implement if needed) - self$time_embedder <- NULL - - resnets <- list() - for (i in seq_len(num_layers)) { - resnets[[i]] <- ltx2_video_resnet_block3d( - in_channels = in_channels, - out_channels = in_channels, - dropout = dropout, - eps = resnet_eps, - non_linearity = resnet_act_fn, - inject_noise = inject_noise, - timestep_conditioning = timestep_conditioning, - spatial_padding_mode = spatial_padding_mode - ) - } - self$resnets <- torch::nn_module_list(resnets) - }, - - forward = function( - hidden_states, - temb = NULL, - generator = NULL, - causal = TRUE - ) { - for (i in seq_along(self$resnets)) { - hidden_states <- self$resnets[[i]](hidden_states, temb, generator, causal = causal) - } - hidden_states - } -) - -#' LTX2 Video Up Block 3D -#' -#' Decoder up block with upsampling and ResNet layers. -#' -#' @param in_channels Integer. Input channels. -#' @param out_channels Integer or NULL. Output channels. -#' @param num_layers Integer. Number of ResNet layers. -#' @param dropout Numeric. Dropout rate. -#' @param resnet_eps Numeric. Epsilon for normalization. -#' @param resnet_act_fn Character. Activation function. -#' @param spatio_temporal_scale Logical. Whether to use upsampling. -#' @param inject_noise Logical. Whether to inject noise. -#' @param timestep_conditioning Logical. Whether to use timestep conditioning. -#' @param upsample_residual Logical. Whether upsampler uses residual. -#' @param upscale_factor Integer. Channel upscale factor. -#' @param spatial_padding_mode Character. Padding mode. -#' @export -ltx2_video_up_block3d <- torch::nn_module( - "LTX2VideoUpBlock3d", - - initialize = function( - in_channels, - out_channels = NULL, - num_layers = 1L, - dropout = 0.0, - resnet_eps = 1e-6, - resnet_act_fn = "swish", - spatio_temporal_scale = TRUE, - inject_noise = FALSE, - timestep_conditioning = FALSE, - upsample_residual = FALSE, - upscale_factor = 1L, - spatial_padding_mode = "zeros" - ) { - if (is.null(out_channels)) out_channels <- in_channels - - # Time embedder (TODO: implement if needed) - self$time_embedder <- NULL - - # Input conv if channels differ - self$conv_in <- NULL - if (in_channels != out_channels) { - self$conv_in <- ltx2_video_resnet_block3d( - in_channels = in_channels, - out_channels = out_channels, - dropout = dropout, - eps = resnet_eps, - non_linearity = resnet_act_fn, - inject_noise = inject_noise, - timestep_conditioning = timestep_conditioning, - spatial_padding_mode = spatial_padding_mode - ) - } - - # Upsampler - self$upsamplers <- NULL - if (spatio_temporal_scale) { - self$upsamplers <- torch::nn_module_list(list( - ltx_video_upsampler3d( - in_channels = out_channels * upscale_factor, - stride = c(2L, 2L, 2L), - residual = upsample_residual, - upscale_factor = upscale_factor, - spatial_padding_mode = spatial_padding_mode - ) - )) - } - - # ResNet layers - resnets <- list() - for (i in seq_len(num_layers)) { - resnets[[i]] <- ltx2_video_resnet_block3d( - in_channels = out_channels, - out_channels = out_channels, - dropout = dropout, - eps = resnet_eps, - non_linearity = resnet_act_fn, - inject_noise = inject_noise, - timestep_conditioning = timestep_conditioning, - spatial_padding_mode = spatial_padding_mode - ) - } - self$resnets <- torch::nn_module_list(resnets) - }, - - forward = function( - hidden_states, - temb = NULL, - generator = NULL, - causal = TRUE - ) { - if (!is.null(self$conv_in)) { - hidden_states <- self$conv_in(hidden_states, temb, generator, causal = causal) - } - - if (!is.null(self$upsamplers)) { - for (i in seq_along(self$upsamplers)) { - hidden_states <- self$upsamplers[[i]](hidden_states, causal = causal) - } - } - - for (i in seq_along(self$resnets)) { - hidden_states <- self$resnets[[i]](hidden_states, temb, generator, causal = causal) - } - - hidden_states - } -) - diff --git a/R/vram.R b/R/vram.R new file mode 100644 index 0000000..0dcbce1 --- /dev/null +++ b/R/vram.R @@ -0,0 +1,202 @@ +#' VRAM Detection and Management Utilities +#' +#' Device detection, VRAM reporting, and module offloading helpers shared +#' by the image and video pipelines. +#' +#' @name vram +NULL + +#' Check if GPU is Blackwell Architecture +#' +#' Blackwell GPUs (RTX 50xx) may need special handling. +#' +#' @return Logical. TRUE if Blackwell GPU detected. +#' +#' @export +#' +#' @examples +#' \dontrun{ +#' if (is_blackwell_gpu()) { +#' message("Using Blackwell-compatible settings") +#' } +#' } +is_blackwell_gpu <- function() { + # Use gpuctl if available + if (requireNamespace("gpu.ctl", quietly = TRUE)) { + return(gpu.ctl::gpu_is_blackwell()) + } + + # Fallback: check compute capability via torch + if (torch::cuda_is_available()) { + props <- tryCatch( + torch::cuda_get_device_properties(0L), + error = function(e) NULL + ) + if (!is.null(props)) { + # Blackwell is compute 12.x + major <- props$major + return(major >= 12) + } + } + + FALSE +} + +#' Detect Available VRAM +#' +#' Uses gpuctl if available. +#' +#' @param use_free Logical. If TRUE, return free VRAM. If FALSE, return total. +#' +#' @return Numeric. VRAM in GB, or 0 if no GPU detected. +#' @keywords internal +.detect_vram <- function(use_free = FALSE) { + # Try gpuctl (preferred - uses nvidia-smi) + if (requireNamespace("gpu.ctl", quietly = TRUE)) { + info <- gpu.ctl::gpu_detect() + if (!is.null(info)) { + if (use_free && !is.null(info$vram_free_gb)) { + return(info$vram_free_gb) + } + if (!is.null(info$vram_total_gb)) { + return(info$vram_total_gb) + } + } + } + + # Fallback: check if CUDA available but can't determine VRAM + if (torch::cuda_is_available()) { + # Conservative estimate - assume 8GB if we can't detect + message("Could not detect VRAM. Install gpuctl for accurate detection.") + return(8) + } + + # No GPU detected + 0 +} + +#' Offload Module to CPU +#' +#' Moves a torch module and all its parameters to CPU. +#' +#' @param module A torch nn_module. +#' @param gc Logical. Run garbage collection after offload. +#' +#' @return The module (modified in place). +#' +#' @export +#' +#' @examples +#' \dontrun{ +#' model$to(device = "cuda") +#' output <- model(x) +#' offload_to_cpu(model) +#' } +offload_to_cpu <- function( + module, + gc = TRUE +) { + module$to(device = "cpu") + if (gc && torch::cuda_is_available()) { + gc() + torch::cuda_empty_cache() + } + invisible(module) +} + +#' Load Module to GPU +#' +#' Moves a torch module and all its parameters to CUDA. +#' +#' @param module A torch nn_module. +#' @param device Character. Target device (default "cuda"). +#' +#' @return The module (modified in place). +#' +#' @export +#' +#' @examples +#' \dontrun{ +#' load_to_gpu(model) +#' output <- model(x) +#' offload_to_cpu(model) +#' } +load_to_gpu <- function( + module, + device = "cuda" +) { + module$to(device = device) + invisible(module) +} + +#' Report VRAM Usage +#' +#' Prints current VRAM usage using gpuctl. +#' +#' @param label Character. Label for the report. +#' +#' @return Invisibly returns a list with used and free VRAM in GB. +#' +#' @export +#' +#' @examples +#' \dontrun{ +#' vram_report("After model load") +#' } +vram_report <- function(label = "") { + if (!torch::cuda_is_available()) { + message("[", label, "] No CUDA available") + return(invisible(list(used = 0, free = 0))) + } + + # Use gpuctl for accurate reporting + if (requireNamespace("gpu.ctl", quietly = TRUE)) { + info <- gpu.ctl::gpu_detect() + if (!is.null(info)) { + used <- info$vram_used_gb + free <- info$vram_free_gb + message(sprintf("[%s] VRAM: %.2f GB used, %.2f GB free", + label, used, free)) + return(invisible(list(used = used, free = free))) + } + } + + message("[", label, "] VRAM: (install gpuctl for detailed stats)") + invisible(list(used = NA, free = NA)) +} + +#' Clear VRAM Cache +#' +#' Forces garbage collection and clears CUDA memory cache. +#' +#' @param verbose Logical. Print memory status before/after. +#' +#' @return Invisibly returns NULL. +#' +#' @export +#' +#' @examples +#' \dontrun{ +#' clear_vram() +#' } +clear_vram <- function(verbose = FALSE) { + if (!torch::cuda_is_available()) { + return(invisible(NULL)) + } + + if (verbose) { + vram_report("Before clear") + } + + gc() + tryCatch( + torch::cuda_empty_cache(), + error = function(e) NULL + ) + + if (verbose) { + vram_report("After clear") + } + + invisible(NULL) +} diff --git a/inst/tinytest/test_dit_ltx2.R b/inst/tinytest/test_dit_ltx2.R deleted file mode 100644 index 0f9fd68..0000000 --- a/inst/tinytest/test_dit_ltx2.R +++ /dev/null @@ -1,256 +0,0 @@ -# Tests for LTX2 DiT Transformer modules - -# Test 1: Timesteps module -cat("Test 1: Timesteps module\n") -ts <- diffuseR:::timesteps_module(num_channels = 256L, flip_sin_to_cos = TRUE, downscale_freq_shift = 0) -timesteps <- torch::torch_tensor(c(0.5, 0.8)) -emb <- ts(timesteps) -expect_equal(as.numeric(emb$shape[2]), 256L, info = "Timestep embedding should have 256 channels") - -# Test 2: TimestepEmbedding MLP -cat("Test 2: TimestepEmbedding MLP\n") -te <- diffuseR:::timestep_embedding_module(in_channels = 256L, time_embed_dim = 512L) -x <- torch::torch_randn(c(2, 256)) -y <- te(x) -expect_equal(as.numeric(y$shape[2]), 512L, info = "Output should have 512 channels") - -# Test 3: PixArtAlphaCombinedTimestepSizeEmbeddings -cat("Test 3: PixArtAlphaCombinedTimestepSizeEmbeddings\n") -emb_module <- diffuseR:::pixart_alpha_combined_timestep_size_embeddings( - embedding_dim = 512L, - size_emb_dim = 170L, - use_additional_conditions = FALSE -) -timesteps <- torch::torch_tensor(c(500.0, 750.0)) -cond <- emb_module(timesteps, batch_size = 2L, hidden_dtype = torch::torch_float32()) -expect_true(!is.null(cond), info = "Should return conditioning tensor") -cat(sprintf(" Conditioning shape: [%s]\n", paste(as.numeric(cond$shape), collapse = ", "))) - -# Test 4: PixArtAlphaTextProjection -cat("Test 4: PixArtAlphaTextProjection\n") -text_proj <- diffuseR:::pixart_alpha_text_projection( - in_features = 3840L, - hidden_size = 4096L -) -caption <- torch::torch_randn(c(2, 77, 3840)) -proj <- text_proj(caption) -expect_equal(as.numeric(proj$shape[3]), 4096L, info = "Projected caption should have 4096 channels") - -# Test 5: RMSNorm -cat("Test 5: RMSNorm\n") -norm <- diffuseR:::rms_norm(dim = 512L, eps = 1e-6) -x <- torch::torch_randn(c(2, 10, 512)) -y <- norm(x) -expect_equal(as.numeric(y$shape), as.numeric(x$shape), info = "RMS norm should preserve shape") - -# Test 6: FeedForward -cat("Test 6: FeedForward\n") -ff <- diffuseR:::feed_forward(dim = 512L, mult = 4L, activation_fn = "gelu-approximate") -x <- torch::torch_randn(c(2, 10, 512)) -y <- ff(x) -expect_equal(as.numeric(y$shape), as.numeric(x$shape), info = "FFN should preserve shape") - -# Test 7: LTX2AdaLayerNormSingle -cat("Test 7: LTX2AdaLayerNormSingle\n") -ada_ln <- diffuseR:::ltx2_ada_layer_norm_single(embedding_dim = 512L, num_mod_params = 6L) -timesteps <- torch::torch_tensor(c(500.0, 750.0)) -result <- ada_ln(timesteps, batch_size = 2L, hidden_dtype = torch::torch_float32()) -expect_equal(length(result), 2L, info = "Should return mod_params and embedded_timestep") -cat(sprintf(" Mod params shape: [%s]\n", paste(as.numeric(result[[1]]$shape), collapse = ", "))) - -# Test 8: LTX2Attention -cat("Test 8: LTX2Attention\n") -attn <- diffuseR:::ltx2_attention( - query_dim = 512L, - heads = 8L, - kv_heads = 8L, - dim_head = 64L, - rope_type = "interleaved" -) -x <- torch::torch_randn(c(2, 100, 512)) -y <- attn(x) -expect_equal(as.numeric(y$shape), as.numeric(x$shape), info = "Self-attention should preserve shape") - -# Test 9: LTX2Attention with RoPE -cat("Test 9: LTX2Attention with RoPE\n") -# Create mock RoPE frequencies -cos_freqs <- torch::torch_ones(c(2, 100, 512)) -sin_freqs <- torch::torch_zeros(c(2, 100, 512)) -rope_freqs <- list(cos_freqs, sin_freqs) - -y_rope <- attn(x, query_rotary_emb = rope_freqs) -expect_equal(as.numeric(y_rope$shape), as.numeric(x$shape), info = "Attention with RoPE should preserve shape") - -# Test 10: LTX2Attention cross-attention -cat("Test 10: LTX2Attention cross-attention\n") -cross_attn <- diffuseR:::ltx2_attention( - query_dim = 512L, - heads = 8L, - kv_heads = 8L, - dim_head = 64L, - cross_attention_dim = 2048L -) -q <- torch::torch_randn(c(2, 100, 512)) -kv <- torch::torch_randn(c(2, 77, 2048)) -y_cross <- cross_attn(q, encoder_hidden_states = kv) -expect_equal(as.numeric(y_cross$shape), as.numeric(q$shape), info = "Cross-attention should output query shape") - -# Test 11: LTX2AudioVideoRotaryPosEmbed (video) -cat("Test 11: LTX2AudioVideoRotaryPosEmbed (video)\n") -rope <- diffuseR:::ltx2_audio_video_rotary_pos_embed( - dim = 4096L, - patch_size = 1L, - patch_size_t = 1L, - base_num_frames = 20L, - base_height = 2048L, - base_width = 2048L, - scale_factors = c(8L, 32L, 32L), - modality = "video", - rope_type = "interleaved" -) - -video_coords <- rope$prepare_video_coords( - batch_size = 2L, - num_frames = 4L, - height = 16L, - width = 16L, - device = "cpu", - fps = 24.0 -) -expect_equal(video_coords$shape[2], 3L, info = "Video coords should have 3 dims (T, H, W)") - -freqs <- rope(video_coords) -expect_equal(length(freqs), 2L, info = "Should return cos and sin freqs") -cat(sprintf(" Video RoPE cos shape: [%s]\n", paste(as.numeric(freqs[[1]]$shape), collapse = ", "))) - -# Test 12: LTX2AudioVideoRotaryPosEmbed (audio) -cat("Test 12: LTX2AudioVideoRotaryPosEmbed (audio)\n") -audio_rope <- diffuseR:::ltx2_audio_video_rotary_pos_embed( - dim = 2048L, - patch_size_t = 1L, - base_num_frames = 20L, - scale_factors = c(4L), - modality = "audio", - rope_type = "interleaved" -) - -audio_coords <- audio_rope$prepare_audio_coords( - batch_size = 2L, - num_frames = 100L, - device = "cpu" -) -expect_equal(audio_coords$shape[2], 1L, info = "Audio coords should have 1 dim (T)") - -audio_freqs <- audio_rope(audio_coords) -expect_equal(length(audio_freqs), 2L, info = "Should return cos and sin freqs") -cat(sprintf(" Audio RoPE cos shape: [%s]\n", paste(as.numeric(audio_freqs[[1]]$shape), collapse = ", "))) - -# Test 13: LTX2VideoTransformerBlock -cat("Test 13: LTX2VideoTransformerBlock\n") -block <- diffuseR:::ltx2_video_transformer_block( - dim = 512L, - num_attention_heads = 8L, - attention_head_dim = 64L, - cross_attention_dim = 2048L, - audio_dim = 256L, - audio_num_attention_heads = 4L, - audio_attention_head_dim = 64L, - audio_cross_attention_dim = 1024L -) - -# Prepare inputs -hidden_states <- torch::torch_randn(c(2, 100, 512)) -audio_hidden_states <- torch::torch_randn(c(2, 50, 256)) -encoder_hidden_states <- torch::torch_randn(c(2, 77, 2048)) -audio_encoder_hidden_states <- torch::torch_randn(c(2, 77, 1024)) -temb <- torch::torch_randn(c(2, 1, 3072)) # 6 * 512 -temb_audio <- torch::torch_randn(c(2, 1, 1536)) # 6 * 256 -temb_ca_scale_shift <- torch::torch_randn(c(2, 1, 2048)) # 4 * 512 -temb_ca_audio_scale_shift <- torch::torch_randn(c(2, 1, 1024)) # 4 * 256 -temb_ca_gate <- torch::torch_randn(c(2, 1, 512)) # 1 * 512 -temb_ca_audio_gate <- torch::torch_randn(c(2, 1, 256)) # 1 * 256 - -# Mock RoPE (ones for cos, zeros for sin = identity) -video_rope_freqs <- list(torch::torch_ones(c(2, 100, 512)), torch::torch_zeros(c(2, 100, 512))) -audio_rope_freqs <- list(torch::torch_ones(c(2, 50, 256)), torch::torch_zeros(c(2, 50, 256))) -ca_video_rope <- list(torch::torch_ones(c(2, 100, 256)), torch::torch_zeros(c(2, 100, 256))) -ca_audio_rope <- list(torch::torch_ones(c(2, 50, 256)), torch::torch_zeros(c(2, 50, 256))) - -result <- block( - hidden_states = hidden_states, - audio_hidden_states = audio_hidden_states, - encoder_hidden_states = encoder_hidden_states, - audio_encoder_hidden_states = audio_encoder_hidden_states, - temb = temb, - temb_audio = temb_audio, - temb_ca_scale_shift = temb_ca_scale_shift, - temb_ca_audio_scale_shift = temb_ca_audio_scale_shift, - temb_ca_gate = temb_ca_gate, - temb_ca_audio_gate = temb_ca_audio_gate, - video_rotary_emb = video_rope_freqs, - audio_rotary_emb = audio_rope_freqs, - ca_video_rotary_emb = ca_video_rope, - ca_audio_rotary_emb = ca_audio_rope -) - -expect_equal(length(result), 2L, info = "Block should return video and audio hidden states") -expect_equal(as.numeric(result[[1]]$shape), as.numeric(hidden_states$shape), info = "Video shape preserved") -expect_equal(as.numeric(result[[2]]$shape), as.numeric(audio_hidden_states$shape), info = "Audio shape preserved") -cat(" Block forward pass successful\n") - -# Test 14: Full LTX2VideoTransformer3DModel (small config) -cat("Test 14: LTX2VideoTransformer3DModel (small config)\n") -model <- ltx2_video_transformer_3d_model( - in_channels = 32L, - out_channels = 32L, - num_attention_heads = 4L, - attention_head_dim = 32L, - cross_attention_dim = 128L, # Must equal inner_dim = 4*32 = 128 (after caption projection) - audio_in_channels = 16L, - audio_out_channels = 16L, - audio_num_attention_heads = 2L, - audio_attention_head_dim = 32L, - audio_cross_attention_dim = 64L, # Must equal audio_inner_dim = 2*32 = 64 - num_layers = 2L, - caption_channels = 512L, - vae_scale_factors = c(8L, 32L, 32L) -) -expect_true(!is.null(model), info = "Model should instantiate") -cat(" Model instantiated with 2 layers\n") - -# Test 15: Full model forward pass -cat("Test 15: Full model forward pass\n") -torch::with_no_grad({ - batch_size <- 1L - num_frames <- 4L - height <- 8L - width <- 8L - num_patches <- num_frames * height * width - - hidden_states <- torch::torch_randn(c(batch_size, num_patches, 32L)) - audio_hidden_states <- torch::torch_randn(c(batch_size, 50L, 16L)) - encoder_hidden_states <- torch::torch_randn(c(batch_size, 77L, 512L)) - audio_encoder_hidden_states <- torch::torch_randn(c(batch_size, 77L, 512L)) # Must match caption_channels - timestep <- torch::torch_tensor(c(500.0))$unsqueeze(2) - - output <- model( - hidden_states = hidden_states, - audio_hidden_states = audio_hidden_states, - encoder_hidden_states = encoder_hidden_states, - audio_encoder_hidden_states = audio_encoder_hidden_states, - timestep = timestep, - num_frames = num_frames, - height = height, - width = width, - fps = 24.0, - audio_num_frames = 50L - ) -}) - -expect_equal(length(output), 2L, info = "Model should return sample and audio_sample") -expect_equal(as.numeric(output$sample$shape), c(1L, num_patches, 32L), info = "Video output shape correct") -expect_equal(as.numeric(output$audio_sample$shape), c(1L, 50L, 16L), info = "Audio output shape correct") -cat(sprintf(" Video output shape: [%s]\n", paste(as.numeric(output$sample$shape), collapse = ", "))) -cat(sprintf(" Audio output shape: [%s]\n", paste(as.numeric(output$audio_sample$shape), collapse = ", "))) - -cat("\nAll LTX2 DiT transformer tests completed\n") diff --git a/inst/tinytest/test_gpu_poor.R b/inst/tinytest/test_gpu_poor.R deleted file mode 100644 index a80030d..0000000 --- a/inst/tinytest/test_gpu_poor.R +++ /dev/null @@ -1,82 +0,0 @@ -# Tests for GPU-Poor Memory Management - -# Test 1: Memory profile detection -cat("Test 1: ltx2_memory_profile with explicit VRAM\n") -profile_high <- ltx2_memory_profile(vram_gb = 20) -expect_equal(profile_high$name, "high", info = "20GB should be high profile") - -profile_medium <- ltx2_memory_profile(vram_gb = 12) -expect_equal(profile_medium$name, "medium", info = "12GB should be medium profile") - -profile_low <- ltx2_memory_profile(vram_gb = 8) -expect_equal(profile_low$name, "low", info = "8GB should be low profile") - -profile_very_low <- ltx2_memory_profile(vram_gb = 6) -expect_equal(profile_very_low$name, "very_low", info = "6GB should be very_low profile") - -profile_cpu <- ltx2_memory_profile(vram_gb = 2) -expect_equal(profile_cpu$name, "cpu_only", info = "2GB should be cpu_only profile") - -# Test 2: Profile contains required fields -cat("Test 2: Profile structure\n") -profile <- ltx2_memory_profile(vram_gb = 8) -expect_true("dit_device" %in% names(profile), info = "Profile should have dit_device") -expect_true("vae_device" %in% names(profile), info = "Profile should have vae_device") -expect_true("vae_tiling" %in% names(profile), info = "Profile should have vae_tiling") -expect_true("vae_tile_size" %in% names(profile), info = "Profile should have vae_tile_size") -expect_true("max_resolution" %in% names(profile), info = "Profile should have max_resolution") -expect_true("max_frames" %in% names(profile), info = "Profile should have max_frames") -expect_true("cfg_mode" %in% names(profile), info = "Profile should have cfg_mode") - -# Test 3: Low profile enables tiling and offloading -cat("Test 3: Low profile settings\n") -low <- ltx2_memory_profile(vram_gb = 8) -expect_true(low$dit_offload, info = "Low profile should enable DiT offload") -expect_true(low$vae_tiling, info = "Low profile should enable VAE tiling") -expect_equal(low$cfg_mode, "sequential", info = "Low profile should use sequential CFG") - -# Test 4: High profile disables aggressive optimizations -cat("Test 4: High profile settings\n") -high <- ltx2_memory_profile(vram_gb = 20) -expect_false(high$dit_offload, info = "High profile should not offload DiT") -expect_false(high$vae_tiling, info = "High profile should not tile VAE") -expect_equal(high$cfg_mode, "batched", info = "High profile should use batched CFG") - -# Test 5: Resolution validation -cat("Test 5: validate_resolution\n") -profile <- ltx2_memory_profile(vram_gb = 8) -result <- validate_resolution(1080, 1920, 121, profile) -expect_true(result$adjusted, info = "Should adjust 1080p for low profile") -expect_true(result$height <= profile$max_resolution[1], info = "Height should be capped") -expect_true(result$width <= profile$max_resolution[2], info = "Width should be capped") -expect_true(result$num_frames <= profile$max_frames, info = "Frames should be capped") - -# Test 6: Resolution within limits -cat("Test 6: validate_resolution within limits\n") -result_ok <- validate_resolution(480, 640, 30, profile) -expect_false(result_ok$adjusted, info = "Should not adjust if within limits") - -# Test 7: VRAM report (just test it runs) -cat("Test 7: vram_report\n") -result <- vram_report("test") -expect_true(is.list(result), info = "vram_report should return a list") -expect_true("used" %in% names(result), info = "Result should have used") -expect_true("free" %in% names(result), info = "Result should have free") - -# Test 8: clear_vram (just test it runs) -cat("Test 8: clear_vram\n") -clear_vram() -expect_true(TRUE, info = "clear_vram should run without error") - -# Test 9: is_blackwell_gpu returns logical -cat("Test 9: is_blackwell_gpu\n") -is_bb <- is_blackwell_gpu() -expect_true(is.logical(is_bb), info = "is_blackwell_gpu should return logical") - -# Test 10: .detect_vram returns numeric -cat("Test 10: .detect_vram\n") -vram <- diffuseR:::.detect_vram() -expect_true(is.numeric(vram), info = ".detect_vram should return numeric") -expect_true(vram >= 0, info = "VRAM should be non-negative") - -cat("\nAll GPU-poor tests completed\n") diff --git a/inst/tinytest/test_ltx2_weights.R b/inst/tinytest/test_ltx2_weights.R deleted file mode 100644 index d63e579..0000000 --- a/inst/tinytest/test_ltx2_weights.R +++ /dev/null @@ -1,101 +0,0 @@ -# Tests for LTX2 weight loading functions - -# Test 1: load_ltx2_vae function exists -cat("Test 1: load_ltx2_vae function exists\n") -expect_true(is.function(load_ltx2_vae), info = "load_ltx2_vae should be a function") - -# Test 2: load_ltx2_transformer function exists -cat("Test 2: load_ltx2_transformer function exists\n") -expect_true(is.function(load_ltx2_transformer), info = "load_ltx2_transformer should be a function") - -# Test 3: load_ltx2_connectors function exists -cat("Test 3: load_ltx2_connectors function exists\n") -expect_true(is.function(load_ltx2_connectors), info = "load_ltx2_connectors should be a function") - -# Test 4: VAE module can be created with defaults -cat("Test 4: VAE module creation\n") -vae <- ltx2_video_vae() -expect_true(inherits(vae, "nn_module"), info = "VAE should be nn_module") - -# Test 5: VAE has expected structure -cat("Test 5: VAE structure\n") -expect_true("encoder" %in% names(vae$children), info = "VAE should have encoder") -expect_true("decoder" %in% names(vae$children), info = "VAE should have decoder") - -# Test 6: VAE parameter count is reasonable -cat("Test 6: VAE parameter count\n") -vae_params <- names(vae$parameters) -expect_true(length(vae_params) > 100, - info = sprintf("VAE should have many parameters (got %d)", length(vae_params))) - -# Test 7: Transformer module can be created -cat("Test 7: Transformer module creation\n") -# Create with smaller config for testing (less memory) -transformer <- ltx2_video_transformer_3d_model( - num_layers = 2L, # Minimal layers for testing - num_attention_heads = 4L, - attention_head_dim = 32L, - audio_num_attention_heads = 4L, - audio_attention_head_dim = 32L -) -expect_true(inherits(transformer, "nn_module"), info = "Transformer should be nn_module") - -# Test 8: Transformer has expected structure -cat("Test 8: Transformer structure\n") -expect_true("transformer_blocks" %in% names(transformer$children), - info = "Transformer should have transformer_blocks") -expect_true("proj_in" %in% names(transformer$children), - info = "Transformer should have proj_in") -expect_true("proj_out" %in% names(transformer$children), - info = "Transformer should have proj_out") - -# Test 9: Connectors module can be created -cat("Test 9: Connectors module creation\n") -connectors <- ltx2_text_connectors( - video_connector_num_attention_heads = 4L, - video_connector_attention_head_dim = 32L, - audio_connector_num_attention_heads = 4L, - audio_connector_attention_head_dim = 32L -) -expect_true(inherits(connectors, "nn_module"), info = "Connectors should be nn_module") - -# Test 10: Connectors has expected structure -cat("Test 10: Connectors structure\n") -expect_true("video_connector" %in% names(connectors$children), - info = "Connectors should have video_connector") -expect_true("audio_connector" %in% names(connectors$children), - info = "Connectors should have audio_connector") -expect_true("text_proj_in" %in% names(connectors$children), - info = "Connectors should have text_proj_in") - -# Test 11: load_ltx2_vae errors on missing file -cat("Test 11: load_ltx2_vae error handling\n") -expect_error(load_ltx2_vae("/nonexistent/path.safetensors"), - info = "Should error on missing file") - -# Test 12: load_ltx2_transformer errors on missing directory -cat("Test 12: load_ltx2_transformer error handling\n") -expect_error(load_ltx2_transformer("/nonexistent/dir"), - info = "Should error on missing directory") - -# Test 13: load_ltx2_connectors errors on missing file -cat("Test 13: load_ltx2_connectors error handling\n") -expect_error(load_ltx2_connectors("/nonexistent/path.safetensors"), - info = "Should error on missing file") - -# Test 14: Internal weight loading function exists for VAE -cat("Test 14: Internal VAE weight loading function\n") -expect_true(exists("load_ltx2_vae_weights", envir = asNamespace("diffuseR")), - info = "load_ltx2_vae_weights should exist") - -# Test 15: Internal weight loading function exists for transformer -cat("Test 15: Internal transformer weight loading function\n") -expect_true(exists("load_ltx2_transformer_weights", envir = asNamespace("diffuseR")), - info = "load_ltx2_transformer_weights should exist") - -# Test 16: Internal weight loading function exists for connectors -cat("Test 16: Internal connector weight loading function\n") -expect_true(exists("load_ltx2_connector_weights", envir = asNamespace("diffuseR")), - info = "load_ltx2_connector_weights should exist") - -cat("\nLTX2 weight loading tests completed\n") diff --git a/inst/tinytest/test_rope.R b/inst/tinytest/test_rope.R deleted file mode 100644 index 76b0b57..0000000 --- a/inst/tinytest/test_rope.R +++ /dev/null @@ -1,140 +0,0 @@ -# Tests for RoPE (Rotary Position Embeddings) - -# Test embedder creation -expect_silent({ - embedder <- rope_embedder_create( - dim = 2048, - patch_size = 1, - patch_size_t = 1 - ) -}) - -expect_equal(embedder$dim, 2048) -expect_equal(embedder$rope_type, "interleaved") -expect_equal(embedder$theta, 10000.0) - -# Test video coordinate preparation -embedder <- rope_embedder_create( - dim = 2048, - patch_size = 1, - patch_size_t = 1, - scale_factors = c(8, 32, 32) -) - -coords <- rope_prepare_video_coords( - embedder = embedder, - batch_size = 2, - num_frames = 4, - height = 16, - width = 16, - device = "cpu", - fps = 24.0 -) - -expect_true(inherits(coords, "torch_tensor")) - -# Check shape: (batch_size, 3, num_patches, 2) -# num_patches = num_frames * height * width = 4 * 16 * 16 = 1024 -expect_equal(as.numeric(coords$shape[1]), 2) # batch_size -expect_equal(as.numeric(coords$shape[2]), 3) # 3 dimensions (f, h, w) -expect_equal(as.numeric(coords$shape[3]), 4 * 16 * 16) # num_patches -expect_equal(as.numeric(coords$shape[4]), 2) # start, end - -# Test RoPE forward (frequency computation) -freqs <- rope_forward(embedder, coords, device = "cpu") - -expect_true(is.list(freqs)) -expect_true(inherits(freqs$cos_freqs, "torch_tensor")) -expect_true(inherits(freqs$sin_freqs, "torch_tensor")) - -# cos and sin should have same shape -expect_equal(as.numeric(freqs$cos_freqs$shape), as.numeric(freqs$sin_freqs$shape)) - -# cos values should be in [-1, 1] -expect_true(as.numeric(freqs$cos_freqs$min()) >= -1.0) -expect_true(as.numeric(freqs$cos_freqs$max()) <= 1.0) - -# sin values should be in [-1, 1] -expect_true(as.numeric(freqs$sin_freqs$min()) >= -1.0) -expect_true(as.numeric(freqs$sin_freqs$max()) <= 1.0) - -# Test apply_interleaved_rotary_emb -batch_size <- 2 -seq_len <- 1024 -channels <- 2048 - -x <- torch::torch_randn(c(batch_size, seq_len, channels)) - -# Need to ensure freqs match x's sequence length -embedder_small <- rope_embedder_create(dim = channels) -coords_small <- rope_prepare_video_coords( - embedder = embedder_small, - batch_size = batch_size, - num_frames = 4, - height = 16, - width = 16, - device = "cpu" -) -freqs_small <- rope_forward(embedder_small, coords_small, device = "cpu") - -# Apply rotation -rotated <- apply_interleaved_rotary_emb(x, freqs_small) - -expect_true(inherits(rotated, "torch_tensor")) -expect_equal(as.numeric(rotated$shape), c(batch_size, seq_len, channels)) - -# Rotated should differ from original -expect_false(torch::torch_allclose(x, rotated)) - -# Test that applying rotation twice with negated freqs returns original -# (RoPE is a rotation, so -rotation undoes it) -freqs_neg <- list( - cos_freqs = freqs_small$cos_freqs, - sin_freqs = -freqs_small$sin_freqs -) -double_rotated <- apply_interleaved_rotary_emb(rotated, freqs_neg) - -# Should be close to original (numerical precision) -diff <- (x - double_rotated)$abs()$max()$item() -expect_true(diff < 1e-5, info = sprintf("Double rotation diff: %f", diff)) - -# Test with different patch sizes -embedder_patched <- rope_embedder_create( - dim = 2048, - patch_size = 2, - patch_size_t = 2 -) - -coords_patched <- rope_prepare_video_coords( - embedder = embedder_patched, - batch_size = 1, - num_frames = 8, - height = 32, - width = 32, - device = "cpu" -) - -# num_patches = (8/2) * (32/2) * (32/2) = 4 * 16 * 16 = 1024 -expect_equal(as.numeric(coords_patched$shape[3]), 4 * 16 * 16) - -# Test GPU if available -if (torch::cuda_is_available()) { - coords_gpu <- rope_prepare_video_coords( - embedder = embedder, - batch_size = 1, - num_frames = 4, - height = 16, - width = 16, - device = "cuda" - ) - - freqs_gpu <- rope_forward(embedder, coords_gpu, device = "cuda") - - expect_equal(as.character(freqs_gpu$cos_freqs$device), "cuda:0") - - x_gpu <- torch::torch_randn(c(1, 1024, 2048), device = "cuda") - rotated_gpu <- apply_interleaved_rotary_emb(x_gpu, freqs_gpu) - expect_equal(as.character(rotated_gpu$device), "cuda:0") -} - -cat("All RoPE tests passed\n") diff --git a/inst/tinytest/test_rope_python_validation.R b/inst/tinytest/test_rope_python_validation.R deleted file mode 100644 index b57b2c9..0000000 --- a/inst/tinytest/test_rope_python_validation.R +++ /dev/null @@ -1,132 +0,0 @@ -# Validation tests for RoPE against Python diffusers -# These tests ensure numerical equivalence with HuggingFace implementation - -# Load Python test cases -test_cases_path <- system.file("validation/rope_test_cases.json", package = "diffuseR") - -if (!file.exists(test_cases_path)) { - cat("Skipping Python validation tests - test cases file not found\n") -} else { - test_cases <- jsonlite::fromJSON(test_cases_path) - tol <- 1e-4 - - # Test 1: Video coordinate shape and bounds - cat("Test 1: Video coordinate shape and bounds\n") - embedder <- rope_embedder_create( - dim = 2048, - patch_size = 1, - patch_size_t = 1, - scale_factors = c(8, 32, 32) - ) - - coords <- rope_prepare_video_coords( - embedder = embedder, - batch_size = 2, - num_frames = 4, - height = 16, - width = 16, - device = "cpu", - fps = 24.0 - ) - - python_coords <- test_cases$video_coords - - # Check shape - r_shape <- as.numeric(coords$shape) - py_shape <- python_coords$coords_shape - expect_equal(r_shape, py_shape, info = "Coordinate shapes should match") - - # Check bounds - r_min <- as.numeric(coords$min()) - r_max <- as.numeric(coords$max()) - expect_true(abs(r_min - python_coords$coords_min) < tol, - info = sprintf("Min diff: %f", abs(r_min - python_coords$coords_min))) - expect_true(abs(r_max - python_coords$coords_max) < tol, - info = sprintf("Max diff: %f", abs(r_max - python_coords$coords_max))) - - cat(sprintf(" Shape: R=%s, Python=%s\n", - paste(r_shape, collapse=","), paste(py_shape, collapse=","))) - cat(sprintf(" Min: R=%.4f, Python=%.4f\n", r_min, python_coords$coords_min)) - cat(sprintf(" Max: R=%.4f, Python=%.4f\n", r_max, python_coords$coords_max)) - - # Test 2: RoPE frequency shapes and bounds - cat("Test 2: RoPE frequency computation\n") - freqs <- rope_forward(embedder, coords, device = "cpu") - - python_freqs <- test_cases$rope_freqs - - # Check shapes - cos_shape <- as.numeric(freqs$cos_freqs$shape) - sin_shape <- as.numeric(freqs$sin_freqs$shape) - expect_equal(cos_shape, python_freqs$cos_shape, info = "Cos shape mismatch") - expect_equal(sin_shape, python_freqs$sin_shape, info = "Sin shape mismatch") - - # Check bounds - cos_min <- as.numeric(freqs$cos_freqs$min()) - cos_max <- as.numeric(freqs$cos_freqs$max()) - sin_min <- as.numeric(freqs$sin_freqs$min()) - sin_max <- as.numeric(freqs$sin_freqs$max()) - - expect_true(abs(cos_min - python_freqs$cos_min) < tol) - expect_true(abs(cos_max - python_freqs$cos_max) < tol) - expect_true(abs(sin_min - python_freqs$sin_min) < tol) - expect_true(abs(sin_max - python_freqs$sin_max) < tol) - - cat(sprintf(" Cos shape: %s\n", paste(cos_shape, collapse=","))) - cat(sprintf(" Cos range: [%.4f, %.4f]\n", cos_min, cos_max)) - cat(sprintf(" Sin range: [%.4f, %.4f]\n", sin_min, sin_max)) - - # Check means (important for numerical equivalence) - cos_mean <- as.numeric(freqs$cos_freqs$mean()) - sin_mean <- as.numeric(freqs$sin_freqs$mean()) - - cat(sprintf(" Cos mean: R=%.6f, Python=%.6f, diff=%.2e\n", - cos_mean, python_freqs$cos_mean, abs(cos_mean - python_freqs$cos_mean))) - cat(sprintf(" Sin mean: R=%.6f, Python=%.6f, diff=%.2e\n", - sin_mean, python_freqs$sin_mean, abs(sin_mean - python_freqs$sin_mean))) - - # Test 3: Apply rotary embedding - cat("Test 3: Apply interleaved rotary embedding\n") - torch::torch_manual_seed(42) - x <- torch::torch_randn(c(2, 1024, 2048)) - - rotated <- apply_interleaved_rotary_emb(x, freqs) - - python_apply <- test_cases$apply_rope - r_mean <- as.numeric(rotated$mean()) - r_std <- as.numeric(rotated$std()) - - cat(sprintf(" Output mean: R=%.6f, Python=%.6f\n", r_mean, python_apply$output_mean)) - cat(sprintf(" Output std: R=%.6f, Python=%.6f\n", r_std, python_apply$output_std)) - - # Shape should match exactly - expect_equal(as.numeric(rotated$shape), python_apply$output_shape) - - # Test 4: Patched coordinates - cat("Test 4: Patched coordinates\n") - embedder_patched <- rope_embedder_create( - dim = 2048, - patch_size = 2, - patch_size_t = 2 - ) - - coords_patched <- rope_prepare_video_coords( - embedder = embedder_patched, - batch_size = 1, - num_frames = 8, - height = 32, - width = 32, - device = "cpu" - ) - - python_patched <- test_cases$patched_coords - r_num_patches <- as.numeric(coords_patched$shape[3]) - - expect_equal(r_num_patches, python_patched$expected_num_patches, - info = sprintf("Num patches: R=%d, expected=%d", - r_num_patches, python_patched$expected_num_patches)) - cat(sprintf(" Num patches: %d (expected: %d)\n", - r_num_patches, python_patched$expected_num_patches)) - - cat("\nAll RoPE Python validation tests completed\n") -} diff --git a/inst/tinytest/test_text_encoder_ltx2.R b/inst/tinytest/test_text_encoder_ltx2.R deleted file mode 100644 index 0eeac12..0000000 --- a/inst/tinytest/test_text_encoder_ltx2.R +++ /dev/null @@ -1,155 +0,0 @@ -# Tests for LTX2 Text Encoder and Connectors - -# Test 1: 1D RoPE for connectors -cat("Test 1: LTX2RotaryPosEmbed1d\n") -rope <- diffuseR:::ltx2_rotary_pos_embed_1d( - dim = 512L, - base_seq_len = 4096L, - theta = 10000.0, - rope_type = "interleaved" -) -freqs <- rope(batch_size = 2L, seq_len = 128L, device = "cpu") -expect_equal(length(freqs), 2L, info = "Should return cos and sin") -expect_equal(as.numeric(freqs[[1]]$shape), c(2, 128, 512), info = "Cos shape correct") -expect_equal(as.numeric(freqs[[2]]$shape), c(2, 128, 512), info = "Sin shape correct") - -# Test 2: 1D Transformer Block -cat("Test 2: LTX2TransformerBlock1d\n") -block <- diffuseR:::ltx2_transformer_block_1d( - dim = 512L, - num_attention_heads = 8L, - attention_head_dim = 64L -) -x <- torch::torch_randn(c(2, 100, 512)) -y <- block(x) -expect_equal(as.numeric(y$shape), as.numeric(x$shape), info = "Block preserves shape") - -# Test 3: 1D Transformer Block with RoPE -cat("Test 3: LTX2TransformerBlock1d with RoPE\n") -rope_freqs <- rope(batch_size = 2L, seq_len = 100L, device = "cpu") -y_rope <- block(x, rotary_emb = rope_freqs) -expect_equal(as.numeric(y_rope$shape), as.numeric(x$shape), info = "Block with RoPE preserves shape") - -# Test 4: Connector Transformer 1D (without learnable registers) -cat("Test 4: LTX2ConnectorTransformer1d (no registers)\n") -connector <- diffuseR:::ltx2_connector_transformer_1d( - num_attention_heads = 8L, - attention_head_dim = 64L, - num_layers = 2L, - num_learnable_registers = NULL -) -x <- torch::torch_randn(c(2, 128, 512)) -attn_mask <- torch::torch_zeros(c(2, 128)) -result <- connector(x, attn_mask) -expect_equal(length(result), 2L, info = "Returns hidden_states and attention_mask") -expect_equal(as.numeric(result[[1]]$shape), c(2, 128, 512), info = "Output shape correct") - -# Test 5: Connector Transformer 1D with attention mask (no registers) -cat("Test 5: LTX2ConnectorTransformer1d with mask (no registers)\n") -connector_mask <- diffuseR:::ltx2_connector_transformer_1d( - num_attention_heads = 8L, - attention_head_dim = 64L, - num_layers = 2L, - num_learnable_registers = NULL # No registers - simpler for testing -) -x <- torch::torch_randn(c(2, 128, 512)) -# Additive attention mask (0 = valid, negative large = masked) -attn_mask <- torch::torch_zeros(c(2, 128)) -attn_mask[1, 65:128] <- -10000.0 # Mask second half of first batch -attn_mask[2, 100:128] <- -10000.0 # Mask last 28 of second batch -result <- connector_mask(x, attn_mask) -expect_equal(as.numeric(result[[1]]$shape), c(2, 128, 512), info = "Output shape with mask correct") - -# Test 6: Full Text Connectors (small config) -cat("Test 6: LTX2TextConnectors (small config)\n") -connectors <- ltx2_text_connectors( - caption_channels = 256L, - text_proj_in_factor = 1L, - video_connector_num_attention_heads = 4L, - video_connector_attention_head_dim = 64L, - video_connector_num_layers = 1L, - video_connector_num_learnable_registers = NULL, - audio_connector_num_attention_heads = 4L, - audio_connector_attention_head_dim = 64L, - audio_connector_num_layers = 1L, - audio_connector_num_learnable_registers = NULL -) -expect_true(!is.null(connectors), info = "Connectors instantiate") -cat(" Connectors instantiated\n") - -# Test 7: Text Connectors forward pass -cat("Test 7: Text Connectors forward\n") -text_embeds <- torch::torch_randn(c(2, 128, 256)) -attn_mask <- torch::torch_ones(c(2, 128)) # All valid -result <- connectors(text_embeds, attn_mask, additive_mask = FALSE) -expect_equal(length(result), 3L, info = "Returns video, audio, and attention mask") -cat(sprintf(" Video embedding shape: [%s]\n", paste(as.numeric(result[[1]]$shape), collapse = ", "))) -cat(sprintf(" Audio embedding shape: [%s]\n", paste(as.numeric(result[[2]]$shape), collapse = ", "))) - -# Test 8: encode_text_ltx2 with random backend -cat("Test 8: encode_text_ltx2 (random backend)\n") -result <- encode_text_ltx2( - prompt = c("A cat sitting on a mat", "A dog running in a field"), - backend = "random", - max_sequence_length = 128L, - caption_channels = 256L -) -expect_true(!is.null(result$prompt_embeds), info = "Returns prompt_embeds") -expect_true(!is.null(result$prompt_attention_mask), info = "Returns attention_mask") -expect_equal(as.numeric(result$prompt_embeds$shape), c(2, 128, 256), info = "Embeddings shape correct") -expect_equal(as.numeric(result$prompt_attention_mask$shape), c(2, 128), info = "Mask shape correct") - -# Test 9: pack_text_embeds -cat("Test 9: pack_text_embeds\n") -# Simulate Gemma output: [batch, seq_len, hidden_dim, num_layers] -hidden_states <- torch::torch_randn(c(2, 64, 128, 4)) # 4 layers -sequence_lengths <- c(50L, 60L) # Valid lengths -packed <- pack_text_embeds( - hidden_states, - sequence_lengths, - padding_side = "left" -) -expect_equal(as.numeric(packed$shape), c(2, 64, 512), info = "Packed shape correct (128 * 4 = 512)") - -# Test 10: Full integration - connectors with encoded text -cat("Test 10: Full integration test\n") -torch::with_no_grad({ - # 1. Get text embeddings (random for testing) - text_result <- encode_text_ltx2( - prompt = "A beautiful sunset over the ocean", - backend = "random", - max_sequence_length = 128L, - caption_channels = 256L - ) - - # 2. Create connectors - connectors <- ltx2_text_connectors( - caption_channels = 256L, - text_proj_in_factor = 1L, - video_connector_num_attention_heads = 4L, - video_connector_attention_head_dim = 64L, - video_connector_num_layers = 1L, - video_connector_num_learnable_registers = NULL, - audio_connector_num_attention_heads = 4L, - audio_connector_attention_head_dim = 64L, - audio_connector_num_layers = 1L, - audio_connector_num_learnable_registers = NULL - ) - - # 3. Process through connectors - connector_result <- connectors( - text_result$prompt_embeds, - text_result$prompt_attention_mask, - additive_mask = FALSE - ) - - video_embeds <- connector_result[[1]] - audio_embeds <- connector_result[[2]] -}) - -expect_true(!is.null(video_embeds), info = "Video embeddings produced") -expect_true(!is.null(audio_embeds), info = "Audio embeddings produced") -cat(sprintf(" Final video embeddings: [%s]\n", paste(as.numeric(video_embeds$shape), collapse = ", "))) -cat(sprintf(" Final audio embeddings: [%s]\n", paste(as.numeric(audio_embeds$shape), collapse = ", "))) - -cat("\nAll LTX2 Text Encoder tests completed\n") diff --git a/inst/tinytest/test_txt2vid_ltx2.R b/inst/tinytest/test_txt2vid_ltx2.R deleted file mode 100644 index bea23ed..0000000 --- a/inst/tinytest/test_txt2vid_ltx2.R +++ /dev/null @@ -1,46 +0,0 @@ -# Tests for LTX-2 Video Generation Pipeline -# Note: Full integration tests require model weights - -# Test 1: Pipeline function exists and has expected signature -cat("Test 1: txt2vid_ltx2 function signature\n") -expect_true(is.function(txt2vid_ltx2), info = "txt2vid_ltx2 should be a function") - -# Check key parameters exist -params <- names(formals(txt2vid_ltx2)) -expect_true("prompt" %in% params, info = "Should have prompt param") -expect_true("width" %in% params, info = "Should have width param") -expect_true("height" %in% params, info = "Should have height param") -expect_true("num_frames" %in% params, info = "Should have num_frames param") -expect_true("memory_profile" %in% params, info = "Should have memory_profile param") -expect_true("text_backend" %in% params, info = "Should have text_backend param") - -# Test 2: Default parameters are sensible -cat("Test 2: Default parameters\n") -defaults <- formals(txt2vid_ltx2) -expect_equal(defaults$width, 768L, info = "Default width should be 768") -expect_equal(defaults$height, 512L, info = "Default height should be 512") -expect_equal(defaults$num_frames, 121L, info = "Default frames should be 121") -expect_equal(defaults$num_inference_steps, 8L, info = "Default steps should be 8 (distilled)") -expect_equal(defaults$guidance_scale, 4.0, info = "Default CFG should be 4.0") - -# Test 3: Memory profile resolution -cat("Test 3: Memory profile parameter\n") -profile_str <- ltx2_memory_profile(vram_gb = 8) -expect_equal(profile_str$name, "low", info = "8GB should resolve to low profile") - -# Test 4: Latent dimension calculation -cat("Test 4: Latent dimensions\n") -# LTX-2 uses 32x spatial and 8x temporal compression -width <- 768L -height <- 512L -num_frames <- 121L - -latent_width <- width %/% 32L -latent_height <- height %/% 32L -latent_frames <- (num_frames - 1L) %/% 8L + 1L - -expect_equal(latent_width, 24L, info = "Latent width correct") -expect_equal(latent_height, 16L, info = "Latent height correct") -expect_equal(latent_frames, 16L, info = "Latent frames correct (121 -> 16)") - -cat("\nLTX-2 pipeline tests completed\n") diff --git a/inst/tinytest/test_vae_ltx2.R b/inst/tinytest/test_vae_ltx2.R deleted file mode 100644 index 7937478..0000000 --- a/inst/tinytest/test_vae_ltx2.R +++ /dev/null @@ -1,242 +0,0 @@ -# Tests for LTX2 Video VAE modules - -# Test 1: PerChannelRMSNorm -cat("Test 1: PerChannelRMSNorm initialization and forward\n") -norm <- per_channel_rms_norm() -expect_true(!is.null(norm), info = "RMS norm should initialize") - -x <- torch::torch_randn(c(2, 4, 3, 8, 8)) -y <- norm(x) -expect_equal(as.numeric(x$shape), as.numeric(y$shape), info = "Shape should be preserved") - -# RMS along channel dim should be approximately 1 -rms_out <- torch::torch_sqrt(torch::torch_mean(y^2, dim = 2, keepdim = TRUE) + 1e-8) -expect_true(abs(rms_out$mean()$item() - 1.0) < 0.1, info = "Output RMS should be ~1") - -# Test 2: LTX2VideoCausalConv3d -cat("Test 2: LTX2VideoCausalConv3d initialization\n") -conv <- ltx2_video_causal_conv3d( - in_channels = 4L, - out_channels = 8L, - kernel_size = 3L -) -expect_true(!is.null(conv), info = "Causal conv should initialize") -expect_equal(conv$kernel_size, c(3L, 3L, 3L), info = "Kernel size should be 3x3x3") - -# Test 3: Causal conv forward (causal mode) -cat("Test 3: LTX2VideoCausalConv3d forward (causal)\n") -x <- torch::torch_randn(c(2, 4, 5, 8, 8)) # B, C, T, H, W -y <- conv(x, causal = TRUE) -expect_equal(as.numeric(y$shape[1]), 2, info = "Batch dim preserved") -expect_equal(as.numeric(y$shape[2]), 8, info = "Output channels correct") -expect_equal(as.numeric(y$shape[3]), 5, info = "Temporal dim preserved (stride=1)") -expect_equal(as.numeric(y$shape[4]), 8, info = "Height preserved") -expect_equal(as.numeric(y$shape[5]), 8, info = "Width preserved") - -# Test 4: Causal conv forward (non-causal mode) -cat("Test 4: LTX2VideoCausalConv3d forward (non-causal)\n") -y_nc <- conv(x, causal = FALSE) -expect_equal(as.numeric(y_nc$shape), as.numeric(y$shape), info = "Non-causal shape matches causal") - -# Test 5: LTX2VideoResnetBlock3d -cat("Test 5: LTX2VideoResnetBlock3d initialization and forward\n") -resnet <- ltx2_video_resnet_block3d( - in_channels = 8L, - out_channels = 8L -) -expect_true(!is.null(resnet), info = "ResNet block should initialize") - -x <- torch::torch_randn(c(2, 8, 4, 8, 8)) -y <- resnet(x, causal = TRUE) -expect_equal(as.numeric(y$shape), as.numeric(x$shape), info = "ResNet should preserve shape") - -# Test 6: ResNet block with channel change -cat("Test 6: LTX2VideoResnetBlock3d with channel change\n") -resnet_change <- ltx2_video_resnet_block3d( - in_channels = 8L, - out_channels = 16L -) -y <- resnet_change(x, causal = TRUE) -expect_equal(as.numeric(y$shape[2]), 16, info = "Output channels should change") - -# Test 7: LTXVideoDownsampler3d -# Note: For stride (2,2,2), temporal dim T must satisfy (T + stride-1) % stride == 0 -# So T % 2 == 1 (T must be odd for temporal stride 2) -cat("Test 7: LTXVideoDownsampler3d\n") -downsampler <- ltx_video_downsampler3d( - in_channels = 8L, - out_channels = 16L, - stride = c(2L, 2L, 2L) -) -x <- torch::torch_randn(c(2, 8, 5, 16, 16)) # T=5 (odd for stride 2 compatibility) -y <- downsampler(x, causal = TRUE) -expect_equal(as.numeric(y$shape[1]), 2, info = "Batch preserved") -expect_equal(as.numeric(y$shape[2]), 16, info = "Channels increased") -# Output T = (T_in + stride - 1) / stride = (5 + 1) / 2 = 3 -expect_equal(as.numeric(y$shape[3]), 3, info = "Temporal: (5+1)/2=3") -expect_equal(as.numeric(y$shape[4]), 8, info = "Height halved") -expect_equal(as.numeric(y$shape[5]), 8, info = "Width halved") - -# Test 8: LTXVideoUpsampler3d -cat("Test 8: LTXVideoUpsampler3d\n") -upsampler <- ltx_video_upsampler3d( - in_channels = 16L, - stride = c(2L, 2L, 2L), - residual = TRUE, - upscale_factor = 1L -) -x <- torch::torch_randn(c(2, 16, 2, 8, 8)) -y <- upsampler(x, causal = TRUE) -expect_equal(as.numeric(y$shape[1]), 2, info = "Batch preserved") -# Output channels = in_channels * stride_prod / upscale_factor = 16 * 8 / 1 = 128 -expect_equal(as.numeric(y$shape[3]), 3, info = "Temporal doubled (minus 1 for causal)") -expect_equal(as.numeric(y$shape[4]), 16, info = "Height doubled") -expect_equal(as.numeric(y$shape[5]), 16, info = "Width doubled") - -# Test 9: LTX2VideoDownBlock3D -cat("Test 9: LTX2VideoDownBlock3D\n") -down_block <- ltx2_video_down_block3d( - in_channels = 8L, - out_channels = 16L, - num_layers = 2L, - spatio_temporal_scale = TRUE, - downsample_type = "conv" -) -x <- torch::torch_randn(c(2, 8, 4, 16, 16)) -y <- down_block(x, causal = TRUE) -expect_equal(as.numeric(y$shape[1]), 2, info = "Batch preserved") -expect_equal(as.numeric(y$shape[3]), 2, info = "Temporal halved") -expect_equal(as.numeric(y$shape[4]), 8, info = "Height halved") -expect_equal(as.numeric(y$shape[5]), 8, info = "Width halved") - -# Test 10: LTX2VideoMidBlock3d -cat("Test 10: LTX2VideoMidBlock3d\n") -mid_block <- ltx2_video_mid_block3d( - in_channels = 8L, - num_layers = 2L -) -x <- torch::torch_randn(c(2, 8, 4, 8, 8)) -y <- mid_block(x, causal = TRUE) -expect_equal(as.numeric(y$shape), as.numeric(x$shape), info = "Mid block preserves shape") - -# Test 11: LTX2VideoUpBlock3d -# Note: In LTX2 decoder, in_channels always equals out_channels. -# The upsampler expects out_channels * upscale_factor as input. -# So input tensor channels = out_channels * upscale_factor, output = out_channels -cat("Test 11: LTX2VideoUpBlock3d\n") -up_block <- ltx2_video_up_block3d( - in_channels = 8L, # Same as out_channels (normal LTX2 usage) - out_channels = 8L, - num_layers = 2L, - spatio_temporal_scale = TRUE, - upsample_residual = TRUE, - upscale_factor = 2L -) -# Input has out_channels * upscale_factor = 8 * 2 = 16 channels -x <- torch::torch_randn(c(2, 16, 2, 8, 8)) -y <- up_block(x, causal = TRUE) -expect_equal(as.numeric(y$shape[1]), 2, info = "Batch preserved") -# Output has out_channels = 8 channels -expect_equal(as.numeric(y$shape[2]), 8, info = "Channels: 16 -> 8") - -# Test 12: LTX2VideoEncoder3d (small config) -# Note: For spatiotemporal downsampling, T must be odd so that (T+1)/2 is integer. -# For 2 spatiotemporal downs: T=5 -> (5+1)/2=3 -> (3+1)/2=2 -cat("Test 12: LTX2VideoEncoder3d\n") -encoder <- ltx2_video_encoder3d( - in_channels = 3L, - out_channels = 32L, - block_out_channels = c(32L, 64L), - spatio_temporal_scaling = c(TRUE, TRUE), - layers_per_block = c(1L, 1L, 1L), - downsample_type = c("spatiotemporal", "spatiotemporal"), - patch_size = 2L, - patch_size_t = 1L -) -# Input: T=5 (odd for spatiotemporal compat), H=32, W=32 -x <- torch::torch_randn(c(1, 3, 5, 32, 32)) -y <- encoder(x, causal = TRUE) -expect_equal(as.numeric(y$shape[1]), 1, info = "Encoder batch preserved") -cat(sprintf(" Encoder output shape: [%s]\n", paste(as.numeric(y$shape), collapse=", "))) - -# Test 13: LTX2VideoDecoder3d (small config) -cat("Test 13: LTX2VideoDecoder3d\n") -decoder <- ltx2_video_decoder3d( - in_channels = 32L, - out_channels = 3L, - block_out_channels = c(32L, 64L), - spatio_temporal_scaling = c(TRUE, TRUE), - layers_per_block = c(1L, 1L, 1L), - patch_size = 2L, - patch_size_t = 1L, - upsample_residual = c(TRUE, TRUE), - upsample_factor = c(2L, 2L) -) -# Use a small latent input -z <- torch::torch_randn(c(1, 32, 1, 4, 4)) -out <- decoder(z, causal = TRUE) -expect_equal(as.numeric(out$shape[1]), 1, info = "Decoder batch preserved") -cat(sprintf(" Decoder output shape: [%s]\n", paste(as.numeric(out$shape), collapse=", "))) - -# Test 14: DiagonalGaussianDistribution -cat("Test 14: DiagonalGaussianDistribution\n") -# Create parameters tensor (mean and logvar concatenated along channel dim) -params <- torch::torch_randn(c(2, 64, 2, 4, 4)) # 32 mean + 32 logvar -dist <- diagonal_gaussian_distribution(params) -expect_equal(as.numeric(dist$mean$shape[2]), 32, info = "Mean has half channels") -expect_equal(as.numeric(dist$logvar$shape[2]), 32, info = "Logvar has half channels") -sample <- dist$sample() -expect_equal(as.numeric(sample$shape), as.numeric(dist$mean$shape), info = "Sample shape matches mean") -mode <- dist$mode() -expect_equal(as.numeric(mode$shape), as.numeric(dist$mean$shape), info = "Mode shape matches mean") - -# Test 15: Full VAE instantiation (small config) -cat("Test 15: Full ltx2_video_vae instantiation\n") -vae <- ltx2_video_vae( - in_channels = 3L, - out_channels = 3L, - latent_channels = 32L, - block_out_channels = c(32L, 64L), - decoder_block_out_channels = c(32L, 64L), - layers_per_block = c(1L, 1L, 1L), - decoder_layers_per_block = c(1L, 1L, 1L), - spatio_temporal_scaling = c(TRUE, TRUE), - decoder_spatio_temporal_scaling = c(TRUE, TRUE), - downsample_type = c("spatiotemporal", "spatiotemporal"), - upsample_residual = c(TRUE, TRUE), - upsample_factor = c(2L, 2L), - patch_size = 2L, - patch_size_t = 1L -) -expect_true(!is.null(vae), info = "VAE should instantiate") -expect_true(!is.null(vae$encoder), info = "VAE should have encoder") -expect_true(!is.null(vae$decoder), info = "VAE should have decoder") - -# Test 16: VAE encode -cat("Test 16: VAE encode\n") -x <- torch::torch_randn(c(1, 3, 5, 32, 32)) # T=5 (odd for spatiotemporal) -torch::with_no_grad({ - posterior <- vae$encode(x, causal = TRUE) -}) -expect_true(!is.null(posterior$mean), info = "Posterior should have mean") -expect_true(!is.null(posterior$std), info = "Posterior should have std") -cat(sprintf(" Latent mean shape: [%s]\n", paste(as.numeric(posterior$mean$shape), collapse=", "))) - -# Test 17: VAE decode -cat("Test 17: VAE decode\n") -torch::with_no_grad({ - z <- posterior$sample() - decoded <- vae$decode(z, causal = TRUE) -}) -cat(sprintf(" Decoded shape: [%s]\n", paste(as.numeric(decoded$shape), collapse=", "))) - -# Test 18: VAE enable_tiling -cat("Test 18: VAE enable_tiling\n") -vae$enable_tiling(tile_sample_min_height = 64L, tile_sample_min_width = 64L) -expect_true(vae$use_tiling, info = "Tiling should be enabled") -expect_equal(vae$tile_sample_min_height, 64L, info = "Tile height should be updated") - -vae$disable_tiling() -expect_false(vae$use_tiling, info = "Tiling should be disabled") - -cat("\nAll LTX2 VAE module tests completed\n") diff --git a/man/apply_interleaved_rotary_emb.Rd b/man/apply_interleaved_rotary_emb.Rd deleted file mode 100644 index bf98173..0000000 --- a/man/apply_interleaved_rotary_emb.Rd +++ /dev/null @@ -1,19 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{apply_interleaved_rotary_emb} -\alias{apply_interleaved_rotary_emb} -\title{Apply interleaved rotary embeddings} -\usage{ -apply_interleaved_rotary_emb(x, freqs) -} -\arguments{ -\item{x}{torch tensor. Query or key tensor of shape (B, S, C).} - -\item{freqs}{List. Contains cos_freqs and sin_freqs from rope_forward().} -} -\value{ -torch tensor. Rotated tensor with same shape as input. -} -\description{ -Applies rotary position embeddings to query or key tensors using the -interleaved format where real and imaginary components alternate. -} diff --git a/man/apply_interleaved_rotary_emb_list.Rd b/man/apply_interleaved_rotary_emb_list.Rd deleted file mode 100644 index 3c41a47..0000000 --- a/man/apply_interleaved_rotary_emb_list.Rd +++ /dev/null @@ -1,11 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{apply_interleaved_rotary_emb_list} -\alias{apply_interleaved_rotary_emb_list} -\title{Apply interleaved rotary embedding} -\usage{ -apply_interleaved_rotary_emb_list(x, freqs) -} -\description{ -Apply interleaved rotary embedding -} -\keyword{internal} diff --git a/man/apply_split_rotary_emb.Rd b/man/apply_split_rotary_emb.Rd deleted file mode 100644 index 2a1bc4d..0000000 --- a/man/apply_split_rotary_emb.Rd +++ /dev/null @@ -1,19 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{apply_split_rotary_emb} -\alias{apply_split_rotary_emb} -\title{Apply split rotary embeddings} -\usage{ -apply_split_rotary_emb(x, freqs) -} -\arguments{ -\item{x}{torch tensor. Query or key tensor.} - -\item{freqs}{List. Contains cos_freqs and sin_freqs from rope_forward().} -} -\value{ -torch tensor. Rotated tensor with same shape as input. -} -\description{ -Applies rotary position embeddings to query or key tensors using the -split format where first half is real and second half is imaginary. -} diff --git a/man/configure_vae_for_profile.Rd b/man/configure_vae_for_profile.Rd deleted file mode 100644 index 92238ce..0000000 --- a/man/configure_vae_for_profile.Rd +++ /dev/null @@ -1,25 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{configure_vae_for_profile} -\alias{configure_vae_for_profile} -\title{Configure VAE for Memory Profile} -\usage{ -configure_vae_for_profile(vae, profile) -} -\arguments{ -\item{vae}{The LTX2 VAE module.} - -\item{profile}{Memory profile from `ltx2_memory_profile()`.} -} -\value{ -The VAE (modified in place). -} -\description{ -Sets VAE tiling parameters based on memory profile. -} -\examples{ -\dontrun{ -profile <- ltx2_memory_profile(vram_gb = 8) -vae <- load_ltx2_vae(...) -configure_vae_for_profile(vae, profile) -} -} diff --git a/man/ddim_scheduler_create.Rd b/man/ddim_scheduler_create.Rd index 7fff721..6cd646b 100644 --- a/man/ddim_scheduler_create.Rd +++ b/man/ddim_scheduler_create.Rd @@ -3,17 +3,13 @@ \alias{ddim_scheduler_create} \title{Create a DDIM Scheduler} \usage{ -ddim_scheduler_create( - num_train_timesteps = 1000, - num_inference_steps = 50, - eta = 0, - beta_schedule = c("linear", "scaled_linear", "cosine"), - beta_start = 0.00085, - beta_end = 0.012, - rescale_betas_zero_snr = FALSE, - dtype = torch::torch_float32(), - device = c(torch::torch_device("cpu"), torch::torch_device("cuda")) -) +ddim_scheduler_create(num_train_timesteps = 1000, num_inference_steps = 50, + eta = 0, + beta_schedule = c("linear", "scaled_linear", "cosine"), + beta_start = 0.00085, beta_end = 0.012, + rescale_betas_zero_snr = FALSE, + dtype = torch::torch_float32(), + device = c(torch::torch_device("cpu"), torch::torch_device("cuda"))) } \arguments{ \item{num_train_timesteps}{Integer. The number of diffusion steps used to diff --git a/man/ddim_scheduler_step.Rd b/man/ddim_scheduler_step.Rd index e9ac78b..9df3960 100644 --- a/man/ddim_scheduler_step.Rd +++ b/man/ddim_scheduler_step.Rd @@ -3,22 +3,12 @@ \alias{ddim_scheduler_step} \title{Perform a DDIM scheduler step} \usage{ -ddim_scheduler_step( - model_output, - timestep, - sample, - schedule, - eta = 0, - use_clipped_model_output = FALSE, - thresholding = FALSE, - generator = NULL, - variance_noise = NULL, - clip_sample = FALSE, - set_alpha_to_one = FALSE, - prediction_type = c("epsilon", "sample", "v_prediction"), - dtype = torch::torch_float32(), - device = "cpu" -) +ddim_scheduler_step(model_output, timestep, sample, schedule, eta = 0, + use_clipped_model_output = FALSE, thresholding = FALSE, + generator = NULL, variance_noise = NULL, + clip_sample = FALSE, set_alpha_to_one = FALSE, + prediction_type = c("epsilon", "sample", "v_prediction"), + dtype = torch::torch_float32(), device = "cpu") } \arguments{ \item{model_output}{Numeric array. The output from the diffusion model, typically diff --git a/man/dequantize_int4.Rd b/man/dequantize_int4.Rd deleted file mode 100644 index 15cd2a9..0000000 --- a/man/dequantize_int4.Rd +++ /dev/null @@ -1,26 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{dequantize_int4} -\alias{dequantize_int4} -\title{Dequantize INT4 Tensor} -\usage{ -dequantize_int4(q, dtype = torch::torch_float16(), device = "cpu") -} -\arguments{ -\item{q}{List. Quantized data from `quantize_int4()`.} - -\item{dtype}{Torch dtype. Output dtype (default float16).} - -\item{device}{Character. Target device.} -} -\value{ -Tensor with original shape and specified dtype. -} -\description{ -Reconstructs a float tensor from INT4-quantized data. -} -\examples{ -\dontrun{ -q <- quantize_int4(weights) -weights_approx <- dequantize_int4(q, dtype = torch_float16(), device = "cuda") -} -} diff --git a/man/diagonal_gaussian_distribution.Rd b/man/diagonal_gaussian_distribution.Rd deleted file mode 100644 index 486a788..0000000 --- a/man/diagonal_gaussian_distribution.Rd +++ /dev/null @@ -1,13 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{diagonal_gaussian_distribution} -\alias{diagonal_gaussian_distribution} -\title{Diagonal Gaussian Distribution} -\usage{ -diagonal_gaussian_distribution(parameters) -} -\arguments{ -\item{parameters}{Tensor of concatenated mean and log variance.} -} -\description{ -Represents a diagonal Gaussian distribution for VAE latents. -} diff --git a/man/dit_offloaded_forward.Rd b/man/dit_offloaded_forward.Rd deleted file mode 100644 index 09db2fd..0000000 --- a/man/dit_offloaded_forward.Rd +++ /dev/null @@ -1,53 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{dit_offloaded_forward} -\alias{dit_offloaded_forward} -\title{DiT Chunk-based Forward Pass} -\usage{ -dit_offloaded_forward( - hidden_states, - layers, - chunk_size = 1L, - device = "cuda", - verbose = FALSE, - ... -) -} -\arguments{ -\item{hidden_states}{Input tensor.} - -\item{layers}{List of transformer layers (on CPU).} - -\item{chunk_size}{Integer. Number of layers to load at once (default 1).} - -\item{device}{Target device for computation.} - -\item{verbose}{Logical. Print progress.} - -\item{...}{Additional arguments passed to each layer.} -} -\value{ -Output tensor (on CPU). -} -\description{ -Runs transformer layers in chunks, moving each chunk to GPU before -computation and back to CPU after. Balances memory usage with speed. -} -\examples{ -\dontrun{ -# Layer-by-layer for 8GB VRAM -output <- dit_offloaded_forward( - hidden_states, - model$transformer_blocks, - chunk_size = 1, - device = "cuda" -) - -# Chunk-based for 16GB VRAM -output <- dit_offloaded_forward( - hidden_states, - model$transformer_blocks, - chunk_size = 12, # 12 layers at a time - device = "cuda" -) -} -} diff --git a/man/download_component.Rd b/man/download_component.Rd index 684b0f5..6353746 100644 --- a/man/download_component.Rd +++ b/man/download_component.Rd @@ -3,13 +3,8 @@ \alias{download_component} \title{Download a single TorchScript model component} \usage{ -download_component( - model_name = "sd21", - component, - device = "cpu", - overwrite = FALSE, - show_progress = TRUE -) +download_component(model_name = "sd21", component, device = "cpu", + overwrite = FALSE, show_progress = TRUE) } \arguments{ \item{model_name}{Character string, the name of the model (e.g., \code{"sd21"}).} diff --git a/man/download_model.Rd b/man/download_model.Rd index abe915b..1dc4108 100644 --- a/man/download_model.Rd +++ b/man/download_model.Rd @@ -3,14 +3,10 @@ \alias{download_model} \title{Download TorchScript model files for Stable Diffusion} \usage{ -download_model( - model_name = "sd21", - devices = list(unet = "cpu", decoder = "cpu", text_encoder = "cpu"), - unet_dtype_str = NULL, - overwrite = FALSE, - show_progress = TRUE, - download_models = FALSE -) +download_model(model_name = "sd21", + devices = list(unet = "cpu", decoder = "cpu", text_encoder = "cpu"), + unet_dtype_str = NULL, overwrite = FALSE, show_progress = TRUE, + download_models = FALSE) } \arguments{ \item{model_name}{Name of the model (e.g., "sd21" for stable-diffusion-2-1)} @@ -31,8 +27,11 @@ A named list of full file paths, keyed by component name. \description{ Downloads the required model files (e.g., UNet, decoder, text encoder) for a given Stable Diffusion model using \code{hfhub::hub_download()}. +} +\details{ Files are cached by hfhub (typically \code{~/.cache/huggingface/hub/}). Legacy files in the old \code{R_user_dir()} location are also recognized. + } \examples{ \dontrun{ diff --git a/man/encode_bpe.Rd b/man/encode_bpe.Rd index f0ad760..d99ea9e 100644 --- a/man/encode_bpe.Rd +++ b/man/encode_bpe.Rd @@ -3,15 +3,8 @@ \alias{encode_bpe} \title{Encode text to token IDs} \usage{ -encode_bpe( - tokenizer, - text, - add_special_tokens = TRUE, - max_length = NULL, - padding = "none", - truncation = FALSE, - return_tensors = "list" -) +encode_bpe(tokenizer, text, add_special_tokens = TRUE, max_length = NULL, + padding = "none", truncation = FALSE, return_tensors = "list") } \arguments{ \item{tokenizer}{A bpe_tokenizer object.} diff --git a/man/encode_text_ltx2.Rd b/man/encode_text_ltx2.Rd deleted file mode 100644 index 3035579..0000000 --- a/man/encode_text_ltx2.Rd +++ /dev/null @@ -1,52 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{encode_text_ltx2} -\alias{encode_text_ltx2} -\title{Encode Text for LTX2} -\usage{ -encode_text_ltx2( - prompt, - backend = "random", - model_path = NULL, - tokenizer_path = NULL, - text_encoder = NULL, - embeddings_file = NULL, - api_url = NULL, - max_sequence_length = 1024L, - caption_channels = 3840L, - device = "cpu", - dtype = torch::torch_float32() -) -} -\arguments{ -\item{prompt}{Character vector of prompts.} - -\item{backend}{Character. Backend to use ("gemma3", "precomputed", "api", "random").} - -\item{model_path}{Character. Path to Gemma3 model directory (for "gemma3" backend).} - -\item{tokenizer_path}{Character. Path to tokenizer (for "gemma3" backend, defaults to model_path).} - -\item{text_encoder}{Pre-loaded Gemma3 text encoder module (for "gemma3" backend).} - -\item{embeddings_file}{Character. Path to pre-computed embeddings (for "precomputed" backend).} - -\item{api_url}{Character. URL of text encoding API (for "api" backend).} - -\item{max_sequence_length}{Integer. Maximum sequence length (default 1024).} - -\item{caption_channels}{Integer. Caption embedding dimension (default 3840).} - -\item{device}{Character. Device for tensors.} - -\item{dtype}{torch_dtype. Data type for tensors.} -} -\value{ -List with prompt_embeds and prompt_attention_mask tensors. -} -\description{ -Encodes text prompts for LTX2 video generation. Supports multiple backends: -- "gemma3": Native R torch Gemma3 text encoder -- "precomputed": Load pre-computed embeddings from file -- "api": Call an HTTP API for text encoding -- "random": Generate random embeddings (for testing only) -} diff --git a/man/encode_with_gemma3.Rd b/man/encode_with_gemma3.Rd index 1bb2e16..bd67717 100644 --- a/man/encode_with_gemma3.Rd +++ b/man/encode_with_gemma3.Rd @@ -3,16 +3,9 @@ \alias{encode_with_gemma3} \title{Encode text with Gemma3 for LTX-2} \usage{ -encode_with_gemma3( - prompts, - model = NULL, - tokenizer = NULL, - max_sequence_length = 1024L, - scale_factor = 8, - device = "cuda", - dtype = "float16", - verbose = TRUE -) +encode_with_gemma3(prompts, model = NULL, tokenizer = NULL, + max_sequence_length = 1024L, device = "cuda", + dtype = "float16", verbose = TRUE) } \arguments{ \item{prompts}{Character vector of prompts.} @@ -23,8 +16,6 @@ encode_with_gemma3( \item{max_sequence_length}{Integer. Maximum sequence length.} -\item{scale_factor}{Numeric. Scale factor for packing (default 8).} - \item{device}{Character. Device for computation.} \item{dtype}{Character. Data type.} @@ -32,9 +23,13 @@ encode_with_gemma3( \item{verbose}{Logical. Print progress.} } \value{ -List with prompt_embeds and prompt_attention_mask. +List with prompt_embeds (raw stacked hidden states, + shape \code{[batch, seq_len, hidden_size, num_layers + 1]}) and + prompt_attention_mask. } \description{ Full pipeline for encoding text prompts using Gemma3 text encoder. -Returns packed embeddings ready for LTX-2 connectors. +Returns the raw stacked per-layer hidden states (embedding layer plus all +transformer layers) for downstream connector modules, which handle +normalization and projection themselves. } diff --git a/man/feed_forward.Rd b/man/feed_forward.Rd deleted file mode 100644 index 668cb05..0000000 --- a/man/feed_forward.Rd +++ /dev/null @@ -1,19 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{feed_forward} -\alias{feed_forward} -\title{FeedForward module} -\usage{ -feed_forward( - dim, - dim_out = NULL, - mult = 4L, - dropout = 0, - activation_fn = "gelu-approximate", - inner_dim = NULL, - bias = TRUE -) -} -\description{ -FeedForward module -} -\keyword{internal} diff --git a/man/flowmatch_calculate_shift.Rd b/man/flowmatch_calculate_shift.Rd index 7433111..003673a 100644 --- a/man/flowmatch_calculate_shift.Rd +++ b/man/flowmatch_calculate_shift.Rd @@ -3,13 +3,8 @@ \alias{flowmatch_calculate_shift} \title{Calculate shift for dynamic shifting} \usage{ -flowmatch_calculate_shift( - seq_len, - base_seq_len = 256L, - max_seq_len = 4096L, - base_shift = 0.5, - max_shift = 1.15 -) +flowmatch_calculate_shift(seq_len, base_seq_len = 256L, max_seq_len = 4096L, + base_shift = 0.5, max_shift = 1.15) } \arguments{ \item{seq_len}{Integer. The sequence length (num_patches).} diff --git a/man/flowmatch_scheduler_create.Rd b/man/flowmatch_scheduler_create.Rd index e51e60d..0a2d899 100644 --- a/man/flowmatch_scheduler_create.Rd +++ b/man/flowmatch_scheduler_create.Rd @@ -3,18 +3,12 @@ \alias{flowmatch_scheduler_create} \title{Create a FlowMatch Euler Discrete Scheduler} \usage{ -flowmatch_scheduler_create( - num_train_timesteps = 1000L, - shift = 1, - use_dynamic_shifting = FALSE, - base_shift = 0.5, - max_shift = 1.15, - base_seq_len = 256L, - max_seq_len = 4096L, - invert_sigmas = FALSE, - shift_terminal = NULL, - time_shift_type = c("exponential", "linear") -) +flowmatch_scheduler_create(num_train_timesteps = 1000L, shift = 1, + use_dynamic_shifting = FALSE, base_shift = 0.5, + max_shift = 1.15, base_seq_len = 256L, + max_seq_len = 4096L, invert_sigmas = FALSE, + shift_terminal = NULL, + time_shift_type = c("exponential", "linear")) } \arguments{ \item{num_train_timesteps}{Integer. The number of diffusion steps used to diff --git a/man/flowmatch_scheduler_step.Rd b/man/flowmatch_scheduler_step.Rd index 4db2184..63f3eaa 100644 --- a/man/flowmatch_scheduler_step.Rd +++ b/man/flowmatch_scheduler_step.Rd @@ -3,13 +3,8 @@ \alias{flowmatch_scheduler_step} \title{Perform a FlowMatch scheduler step} \usage{ -flowmatch_scheduler_step( - model_output, - timestep, - sample, - schedule, - generator = NULL -) +flowmatch_scheduler_step(model_output, timestep, sample, schedule, + generator = NULL) } \arguments{ \item{model_output}{torch tensor. The output from the diffusion model diff --git a/man/flowmatch_set_timesteps.Rd b/man/flowmatch_set_timesteps.Rd index 89881cb..78c2f7f 100644 --- a/man/flowmatch_set_timesteps.Rd +++ b/man/flowmatch_set_timesteps.Rd @@ -3,14 +3,8 @@ \alias{flowmatch_set_timesteps} \title{Set timesteps for inference} \usage{ -flowmatch_set_timesteps( - schedule, - num_inference_steps = 50L, - device = "cpu", - mu = NULL, - sigmas = NULL, - timesteps = NULL -) +flowmatch_set_timesteps(schedule, num_inference_steps = 50L, device = "cpu", + mu = NULL, sigmas = NULL, timesteps = NULL) } \arguments{ \item{schedule}{List. The FlowMatch scheduler object.} diff --git a/man/gelu_activation.Rd b/man/gelu_activation.Rd deleted file mode 100644 index 7ff959a..0000000 --- a/man/gelu_activation.Rd +++ /dev/null @@ -1,11 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{gelu_activation} -\alias{gelu_activation} -\title{GELU activation with optional approximation} -\usage{ -gelu_activation(dim_in, dim_out, approximate = "none", bias = TRUE) -} -\description{ -GELU activation with optional approximation -} -\keyword{internal} diff --git a/man/gemma3_rotary_embedding.Rd b/man/gemma3_rotary_embedding.Rd index f840809..c80dea0 100644 --- a/man/gemma3_rotary_embedding.Rd +++ b/man/gemma3_rotary_embedding.Rd @@ -3,12 +3,8 @@ \alias{gemma3_rotary_embedding} \title{Gemma3 Rotary Position Embeddings} \usage{ -gemma3_rotary_embedding( - dim, - max_position_embeddings = 8192L, - base = 10000, - scaling_factor = 1 -) +gemma3_rotary_embedding(dim, max_position_embeddings = 8192L, base = 10000, + scaling_factor = 1) } \arguments{ \item{dim}{Integer. Head dimension.} diff --git a/man/get_timestep_embedding.Rd b/man/get_timestep_embedding.Rd deleted file mode 100644 index ce6b5e1..0000000 --- a/man/get_timestep_embedding.Rd +++ /dev/null @@ -1,18 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{get_timestep_embedding} -\alias{get_timestep_embedding} -\title{Get timestep embedding (sinusoidal)} -\usage{ -get_timestep_embedding( - timesteps, - embedding_dim, - flip_sin_to_cos = FALSE, - downscale_freq_shift = 1, - scale = 1, - max_period = 10000 -) -} -\description{ -Get timestep embedding (sinusoidal) -} -\keyword{internal} diff --git a/man/gpu_poor.Rd b/man/gpu_poor.Rd deleted file mode 100644 index 9d6ee20..0000000 --- a/man/gpu_poor.Rd +++ /dev/null @@ -1,8 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{gpu_poor} -\alias{gpu_poor} -\title{GPU-Poor Memory Management for LTX-2} -\description{ -wan2GP-style memory optimizations for running LTX-2 video generation -on limited VRAM (6-16GB). -} diff --git a/man/img2img.Rd b/man/img2img.Rd index f25797d..ad8cdbc 100644 --- a/man/img2img.Rd +++ b/man/img2img.Rd @@ -3,29 +3,13 @@ \alias{img2img} \title{Image-to-Image Generation with Stable Diffusion} \usage{ -img2img( - input_image, - prompt, - negative_prompt = NULL, - img_dim = 512, - model_name = c("sd21", "sdxl"), - pipeline = NULL, - devices = "auto", - unet_dtype_str = "float16", - download_models = FALSE, - scheduler = "ddim", - num_inference_steps = 50, - strength = 0.8, - guidance_scale = 7.5, - seed = NULL, - save_file = TRUE, - filename = NULL, - metadata_path = NULL, - use_native_decoder = FALSE, - use_native_text_encoder = FALSE, - use_native_unet = FALSE, - ... -) +img2img(input_image, prompt, negative_prompt = NULL, img_dim = 512, + model_name = c("sd21", "sdxl"), pipeline = NULL, devices = "auto", + unet_dtype_str = "float16", download_models = FALSE, + scheduler = "ddim", num_inference_steps = 50, strength = 0.8, + guidance_scale = 7.5, seed = NULL, save_file = TRUE, filename = NULL, + metadata_path = NULL, use_native_decoder = FALSE, + use_native_text_encoder = FALSE, use_native_unet = FALSE, ...) } \arguments{ \item{input_image}{Path to the input image or a tensor representing the image.} diff --git a/man/int4_linear.Rd b/man/int4_linear.Rd deleted file mode 100644 index 5749828..0000000 --- a/man/int4_linear.Rd +++ /dev/null @@ -1,43 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{int4_linear} -\alias{int4_linear} -\title{INT4 Linear Layer} -\usage{ -int4_linear( - in_features, - out_features, - bias = TRUE, - device = "cuda", - dtype = torch::torch_float16() -) -} -\arguments{ -\item{in_features}{Integer. Size of each input sample.} - -\item{out_features}{Integer. Size of each output sample.} - -\item{bias}{Logical. If TRUE, adds a learnable bias (stored in float16).} - -\item{device}{Character. Device for the layer.} - -\item{dtype}{torch_dtype. Data type for dequantized operations.} -} -\value{ -nn_module with INT4 weight storage. -} -\description{ -A linear layer that stores weights in INT4 format and dequantizes on-the-fly -during forward pass. This enables running large models on limited VRAM by -keeping weights compressed on GPU. -} -\details{ -The layer stores: -- `weight_packed`: uint8 tensor with packed INT4 values -- `weight_scales`: float32 tensor with per-block scales -- `weight_shape`: original weight shape -- `bias`: optional float16 bias - -During forward(), weights are dequantized to the target dtype, the matmul -is performed, and the dequantized tensor is freed. This allows ~40GB models -to run with ~10GB of VRAM for weights. -} diff --git a/man/int4_linear_from_quantized.Rd b/man/int4_linear_from_quantized.Rd deleted file mode 100644 index a30ba7e..0000000 --- a/man/int4_linear_from_quantized.Rd +++ /dev/null @@ -1,30 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{int4_linear_from_quantized} -\alias{int4_linear_from_quantized} -\title{Create INT4 Linear from Pre-quantized Weights} -\usage{ -int4_linear_from_quantized( - q_weight, - q_bias = NULL, - bias_tensor = NULL, - device = "cuda", - dtype = torch::torch_float16() -) -} -\arguments{ -\item{q_weight}{List with packed, scales, orig_shape from load_int4_weights().} - -\item{q_bias}{Optional. Quantized bias (or NULL for no bias).} - -\item{bias_tensor}{Optional. Float tensor for bias (if not quantized).} - -\item{device}{Character. Target device.} - -\item{dtype}{torch_dtype. Target dtype for operations.} -} -\value{ -int4_linear module with loaded weights. -} -\description{ -Creates an INT4 linear layer from pre-quantized weight data. -} diff --git a/man/linear_to_int4.Rd b/man/linear_to_int4.Rd deleted file mode 100644 index 183ad5d..0000000 --- a/man/linear_to_int4.Rd +++ /dev/null @@ -1,20 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{linear_to_int4} -\alias{linear_to_int4} -\title{Create INT4 Linear from Standard Linear} -\usage{ -linear_to_int4(linear_module, device = "cuda", dtype = torch::torch_float16()) -} -\arguments{ -\item{linear_module}{nn_linear module to convert.} - -\item{device}{Character. Target device for INT4 weights.} - -\item{dtype}{torch_dtype. Target dtype for dequantized operations.} -} -\value{ -int4_linear module with quantized weights. -} -\description{ -Converts a standard nn_linear layer to an INT4 linear layer. -} diff --git a/man/load_gemma3_text_encoder.Rd b/man/load_gemma3_text_encoder.Rd index e7e07e6..bffba3d 100644 --- a/man/load_gemma3_text_encoder.Rd +++ b/man/load_gemma3_text_encoder.Rd @@ -3,12 +3,8 @@ \alias{load_gemma3_text_encoder} \title{Load Gemma3 Text Model from safetensors} \usage{ -load_gemma3_text_encoder( - model_path, - device = "cpu", - dtype = "float16", - verbose = TRUE -) +load_gemma3_text_encoder(model_path, device = "cpu", dtype = "float16", + verbose = TRUE) } \arguments{ \item{model_path}{Character. Path to directory containing model files.} diff --git a/man/load_int4_into_model.Rd b/man/load_int4_into_model.Rd deleted file mode 100644 index ebfecfa..0000000 --- a/man/load_int4_into_model.Rd +++ /dev/null @@ -1,41 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{load_int4_into_model} -\alias{load_int4_into_model} -\title{Load INT4 Weights into Model} -\usage{ -load_int4_into_model( - model, - int4_weights, - device = "cuda", - dtype = torch::torch_float16(), - verbose = TRUE -) -} -\arguments{ -\item{model}{nn_module. The model to convert.} - -\item{int4_weights}{List of quantized weights from `load_int4_weights()`.} - -\item{device}{Character. Target device for INT4 weights.} - -\item{dtype}{torch_dtype. Target dtype for dequantized operations.} - -\item{verbose}{Logical. Print progress.} -} -\value{ -The model with linear layers replaced by INT4 versions. -} -\description{ -Replaces linear layers in a model with INT4 versions and loads quantized weights. -This is the main entry point for running large models with INT4 quantization. -} -\details{ -This function: -1. Identifies linear layers by matching parameter names ending in ".weight" -2. Creates INT4Linear layers with matching dimensions -3. Loads quantized weights and biases -4. Replaces the original layers in the model - -The INT4 weights stay compressed on GPU (~10GB for a 40GB model). -During forward(), each layer dequantizes on-the-fly, keeping memory usage low. -} diff --git a/man/load_int4_weights.Rd b/man/load_int4_weights.Rd deleted file mode 100644 index d74ae3a..0000000 --- a/man/load_int4_weights.Rd +++ /dev/null @@ -1,27 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{load_int4_weights} -\alias{load_int4_weights} -\title{Load INT4 Quantized Weights} -\usage{ -load_int4_weights(path, verbose = TRUE) -} -\arguments{ -\item{path}{Character. Path to safetensors file or base path for sharded files. -For sharded files, pass the base path (e.g., "model_int4.safetensors") and -the function will find all shards matching the pattern.} - -\item{verbose}{Logical. Print progress.} -} -\value{ -List of quantized parameter structures ready for `dequantize_int4()`. -} -\description{ -Loads INT4 quantized weights from safetensors file(s). -} -\examples{ -\dontrun{ -q <- load_int4_weights("model_int4.safetensors") -# Dequantize specific parameter on GPU -weight <- dequantize_int4(q[["linear.weight"]], device = "cuda") -} -} diff --git a/man/load_int4_weights_into_model.Rd b/man/load_int4_weights_into_model.Rd deleted file mode 100644 index c274f75..0000000 --- a/man/load_int4_weights_into_model.Rd +++ /dev/null @@ -1,21 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{load_int4_weights_into_model} -\alias{load_int4_weights_into_model} -\title{Load INT4 Weights into INT4 Model} -\usage{ -load_int4_weights_into_model(model, int4_weights, verbose = TRUE) -} -\arguments{ -\item{model}{nn_module created with INT4 layers.} - -\item{int4_weights}{List from `load_int4_weights()`.} - -\item{verbose}{Logical. Print progress.} -} -\value{ -Model with INT4 weights loaded (invisibly). -} -\description{ -Loads pre-quantized INT4 weights into a model created with `make_linear()` -when `diffuseR.use_int4 = TRUE`. -} diff --git a/man/load_ltx2_connector_weights.Rd b/man/load_ltx2_connector_weights.Rd deleted file mode 100644 index eb7dc35..0000000 --- a/man/load_ltx2_connector_weights.Rd +++ /dev/null @@ -1,18 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{load_ltx2_connector_weights} -\alias{load_ltx2_connector_weights} -\title{Load weights into LTX2 connectors module} -\usage{ -load_ltx2_connector_weights(connectors, weights, verbose = TRUE) -} -\arguments{ -\item{connectors}{LTX2 connectors module} - -\item{weights}{Named list of weight tensors} - -\item{verbose}{Print progress} -} -\description{ -Load weights into LTX2 connectors module -} -\keyword{internal} diff --git a/man/load_ltx2_connectors.Rd b/man/load_ltx2_connectors.Rd deleted file mode 100644 index e4f6c6b..0000000 --- a/man/load_ltx2_connectors.Rd +++ /dev/null @@ -1,30 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{load_ltx2_connectors} -\alias{load_ltx2_connectors} -\title{Load LTX2 Text Connectors from safetensors} -\usage{ -load_ltx2_connectors( - weights_path, - config_path = NULL, - device = "cpu", - dtype = "float32", - verbose = TRUE -) -} -\arguments{ -\item{weights_path}{Character. Path to safetensors file.} - -\item{config_path}{Character. Optional path to config.json.} - -\item{device}{Character. Device to load weights to. Default: "cpu"} - -\item{dtype}{Character. Data type ("float32", "float16"). Default: "float32"} - -\item{verbose}{Logical. Print loading progress. Default: TRUE} -} -\value{ -Initialized ltx2_text_connectors module -} -\description{ -Load pre-trained LTX2 connector weights from HuggingFace safetensors file. -} diff --git a/man/load_ltx2_transformer.Rd b/man/load_ltx2_transformer.Rd deleted file mode 100644 index 08ebd2d..0000000 --- a/man/load_ltx2_transformer.Rd +++ /dev/null @@ -1,31 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{load_ltx2_transformer} -\alias{load_ltx2_transformer} -\title{Load LTX2 DiT Transformer from safetensors} -\usage{ -load_ltx2_transformer( - weights_dir, - config_path = NULL, - device = "cpu", - dtype = "float16", - verbose = TRUE -) -} -\arguments{ -\item{weights_dir}{Character. Directory containing safetensors files.} - -\item{config_path}{Character. Optional path to config.json. If NULL, uses default config.} - -\item{device}{Character. Device to load weights to. Default: "cpu"} - -\item{dtype}{Character. Data type ("float32", "float16", "bfloat16"). Default: "float16"} - -\item{verbose}{Logical. Print loading progress. Default: TRUE} -} -\value{ -Initialized ltx2_video_transformer3d module -} -\description{ -Load pre-trained LTX2 transformer weights from HuggingFace safetensors files. -Supports both single file and sharded multi-file loading. -} diff --git a/man/load_ltx2_transformer_sharded.Rd b/man/load_ltx2_transformer_sharded.Rd deleted file mode 100644 index b82e7d0..0000000 --- a/man/load_ltx2_transformer_sharded.Rd +++ /dev/null @@ -1,25 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{load_ltx2_transformer_sharded} -\alias{load_ltx2_transformer_sharded} -\title{Load sharded transformer weights} -\usage{ -load_ltx2_transformer_sharded( - transformer, - weights_dir, - index_path, - verbose = TRUE -) -} -\arguments{ -\item{transformer}{LTX2 transformer module} - -\item{weights_dir}{Directory containing sharded safetensors} - -\item{index_path}{Path to index.json} - -\item{verbose}{Print progress} -} -\description{ -Load sharded transformer weights -} -\keyword{internal} diff --git a/man/load_ltx2_transformer_weights.Rd b/man/load_ltx2_transformer_weights.Rd deleted file mode 100644 index a03aec2..0000000 --- a/man/load_ltx2_transformer_weights.Rd +++ /dev/null @@ -1,18 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{load_ltx2_transformer_weights} -\alias{load_ltx2_transformer_weights} -\title{Load weights into LTX2 transformer module} -\usage{ -load_ltx2_transformer_weights(transformer, weights, verbose = TRUE) -} -\arguments{ -\item{transformer}{LTX2 transformer module} - -\item{weights}{Named list of weight tensors} - -\item{verbose}{Print progress} -} -\description{ -Load weights into LTX2 transformer module -} -\keyword{internal} diff --git a/man/load_ltx2_vae.Rd b/man/load_ltx2_vae.Rd deleted file mode 100644 index 6228561..0000000 --- a/man/load_ltx2_vae.Rd +++ /dev/null @@ -1,31 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{load_ltx2_vae} -\alias{load_ltx2_vae} -\title{Load LTX2 Video VAE from safetensors} -\usage{ -load_ltx2_vae( - weights_path, - config_path = NULL, - device = "cpu", - dtype = "float32", - verbose = TRUE -) -} -\arguments{ -\item{weights_path}{Character. Path to safetensors file or directory containing weights.} - -\item{config_path}{Character. Optional path to config.json. If NULL and weights_path -is a directory, looks for config.json in that directory. Otherwise uses default config.} - -\item{device}{Character. Device to load weights to. Default: "cpu"} - -\item{dtype}{Character or torch dtype. Data type. Default: "float32"} - -\item{verbose}{Logical. Print loading progress. Default: TRUE} -} -\value{ -Initialized ltx2_video_vae module -} -\description{ -Load pre-trained LTX2 VAE weights from a HuggingFace safetensors file. -} diff --git a/man/load_ltx2_vae_weights.Rd b/man/load_ltx2_vae_weights.Rd deleted file mode 100644 index 037c7a8..0000000 --- a/man/load_ltx2_vae_weights.Rd +++ /dev/null @@ -1,21 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{load_ltx2_vae_weights} -\alias{load_ltx2_vae_weights} -\title{Load weights into LTX2 VAE module} -\usage{ -load_ltx2_vae_weights(vae, weights, verbose = TRUE) -} -\arguments{ -\item{vae}{LTX2 VAE module} - -\item{weights}{Named list of weight tensors from safetensors} - -\item{verbose}{Logical. Print loading progress} -} -\value{ -The VAE with loaded weights (invisibly) -} -\description{ -Maps HuggingFace safetensors parameter names to R module parameters. -} -\keyword{internal} diff --git a/man/load_model_component.Rd b/man/load_model_component.Rd index edc91b6..d8efbcb 100644 --- a/man/load_model_component.Rd +++ b/man/load_model_component.Rd @@ -3,14 +3,8 @@ \alias{load_model_component} \title{Load a specific component of a diffusion model} \usage{ -load_model_component( - component, - model_name = "sd21", - device = "cpu", - unet_dtype_str = NULL, - download = TRUE, - use_native = FALSE -) +load_model_component(component, model_name = "sd21", device = "cpu", + unet_dtype_str = NULL, download = TRUE, use_native = FALSE) } \arguments{ \item{component}{Character string, the component to load: "unet", "decoder", or "text_encoder".} diff --git a/man/load_pipeline.Rd b/man/load_pipeline.Rd index 783d704..b805ca7 100644 --- a/man/load_pipeline.Rd +++ b/man/load_pipeline.Rd @@ -3,16 +3,9 @@ \alias{load_pipeline} \title{Load a diffusion model pipeline} \usage{ -load_pipeline( - model_name, - m2d, - i2i = FALSE, - unet_dtype_str, - use_native_decoder = FALSE, - use_native_text_encoder = FALSE, - use_native_unet = FALSE, - ... -) +load_pipeline(model_name, m2d, i2i = FALSE, unet_dtype_str, + use_native_decoder = FALSE, use_native_text_encoder = FALSE, + use_native_unet = FALSE, ...) } \arguments{ \item{model_name}{The name of the model to load.} diff --git a/man/ltx2_ada_layer_norm_single.Rd b/man/ltx2_ada_layer_norm_single.Rd deleted file mode 100644 index dd4e6b3..0000000 --- a/man/ltx2_ada_layer_norm_single.Rd +++ /dev/null @@ -1,15 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{ltx2_ada_layer_norm_single} -\alias{ltx2_ada_layer_norm_single} -\title{LTX2 AdaLayerNorm Single} -\usage{ -ltx2_ada_layer_norm_single( - embedding_dim, - num_mod_params = 6L, - use_additional_conditions = FALSE -) -} -\description{ -LTX2 AdaLayerNorm Single -} -\keyword{internal} diff --git a/man/ltx2_attention.Rd b/man/ltx2_attention.Rd deleted file mode 100644 index 8d62fea..0000000 --- a/man/ltx2_attention.Rd +++ /dev/null @@ -1,24 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{ltx2_attention} -\alias{ltx2_attention} -\title{LTX2 Attention module} -\usage{ -ltx2_attention( - query_dim, - heads = 8L, - kv_heads = 8L, - dim_head = 64L, - dropout = 0, - bias = TRUE, - cross_attention_dim = NULL, - out_bias = TRUE, - qk_norm = "rms_norm_across_heads", - norm_eps = 1e-06, - norm_elementwise_affine = TRUE, - rope_type = "interleaved" -) -} -\description{ -LTX2 Attention module -} -\keyword{internal} diff --git a/man/ltx2_audio_video_rotary_pos_embed.Rd b/man/ltx2_audio_video_rotary_pos_embed.Rd deleted file mode 100644 index 08c61bb..0000000 --- a/man/ltx2_audio_video_rotary_pos_embed.Rd +++ /dev/null @@ -1,27 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{ltx2_audio_video_rotary_pos_embed} -\alias{ltx2_audio_video_rotary_pos_embed} -\title{LTX2 Audio-Video Rotary Positional Embeddings} -\usage{ -ltx2_audio_video_rotary_pos_embed( - dim, - patch_size = 1L, - patch_size_t = 1L, - base_num_frames = 20L, - base_height = 2048L, - base_width = 2048L, - sampling_rate = 16000L, - hop_length = 160L, - scale_factors = c(8L, 32L, 32L), - theta = 10000, - causal_offset = 1L, - modality = "video", - double_precision = TRUE, - rope_type = "interleaved", - num_attention_heads = 32L -) -} -\description{ -LTX2 Audio-Video Rotary Positional Embeddings -} -\keyword{internal} diff --git a/man/ltx2_connector_transformer_1d.Rd b/man/ltx2_connector_transformer_1d.Rd deleted file mode 100644 index b94e9d4..0000000 --- a/man/ltx2_connector_transformer_1d.Rd +++ /dev/null @@ -1,22 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{ltx2_connector_transformer_1d} -\alias{ltx2_connector_transformer_1d} -\title{1D Connector Transformer for LTX2} -\usage{ -ltx2_connector_transformer_1d( - num_attention_heads = 30L, - attention_head_dim = 128L, - num_layers = 2L, - num_learnable_registers = 128L, - rope_base_seq_len = 4096L, - rope_theta = 10000, - rope_double_precision = TRUE, - eps = 1e-06, - causal_temporal_positioning = FALSE, - rope_type = "interleaved" -) -} -\description{ -1D Connector Transformer for LTX2 -} -\keyword{internal} diff --git a/man/ltx2_memory_profile.Rd b/man/ltx2_memory_profile.Rd deleted file mode 100644 index e183e9b..0000000 --- a/man/ltx2_memory_profile.Rd +++ /dev/null @@ -1,45 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{ltx2_memory_profile} -\alias{ltx2_memory_profile} -\title{Get LTX-2 Memory Profile} -\usage{ -ltx2_memory_profile(vram_gb = NULL, model = "ltx2-19b-fp4") -} -\arguments{ -\item{vram_gb}{Numeric. Available VRAM in GB, or NULL for auto-detection.} - -\item{model}{Character. Model variant: "ltx2-19b-fp4" (default), "ltx2-19b-fp8", -or "ltx2-19b-distilled".} -} -\value{ -A list with memory profile settings. -} -\description{ -Determines optimal memory configuration based on available VRAM. -} -\details{ -LTX-2 is a 19B parameter model. Even with FP4 quantization (~10GB weights), -it requires careful memory management. The GPU-poor approach: - -1. Text encoding runs on CPU (cached) -2. DiT loaded in chunks, processed layer-by-layer, unloaded -3. VAE loaded after DiT unload, decode with tiling, unload - -Memory profiles: -\describe{ - \item{high}{16GB+ - FP4 DiT with chunk loading, VAE on GPU} - \item{medium}{12GB - FP4 DiT chunk loading, VAE tiled} - \item{low}{8GB - FP4 DiT layer-by-layer, VAE tiled small} - \item{very_low}{6GB - FP4 layer-by-layer, VAE on CPU} - \item{cpu_only}{All on CPU} -} -} -\examples{ -\dontrun{ -# Auto-detect profile -profile <- ltx2_memory_profile() - -# Specific VRAM -profile <- ltx2_memory_profile(vram_gb = 8) -} -} diff --git a/man/ltx2_rotary_pos_embed_1d.Rd b/man/ltx2_rotary_pos_embed_1d.Rd deleted file mode 100644 index 3ece718..0000000 --- a/man/ltx2_rotary_pos_embed_1d.Rd +++ /dev/null @@ -1,18 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{ltx2_rotary_pos_embed_1d} -\alias{ltx2_rotary_pos_embed_1d} -\title{1D Rotary Position Embeddings for LTX2 Text Connectors} -\usage{ -ltx2_rotary_pos_embed_1d( - dim, - base_seq_len = 4096L, - theta = 10000, - double_precision = TRUE, - rope_type = "interleaved", - num_attention_heads = 32L -) -} -\description{ -1D Rotary Position Embeddings for LTX2 Text Connectors -} -\keyword{internal} diff --git a/man/ltx2_text_connectors.Rd b/man/ltx2_text_connectors.Rd deleted file mode 100644 index 8ae0ff4..0000000 --- a/man/ltx2_text_connectors.Rd +++ /dev/null @@ -1,60 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{ltx2_text_connectors} -\alias{ltx2_text_connectors} -\title{LTX2 Text Connectors} -\usage{ -ltx2_text_connectors( - caption_channels = 3840L, - text_proj_in_factor = 49L, - video_connector_num_attention_heads = 30L, - video_connector_attention_head_dim = 128L, - video_connector_num_layers = 2L, - video_connector_num_learnable_registers = NULL, - audio_connector_num_attention_heads = 30L, - audio_connector_attention_head_dim = 128L, - audio_connector_num_layers = 2L, - audio_connector_num_learnable_registers = NULL, - connector_rope_base_seq_len = 4096L, - rope_theta = 10000, - rope_double_precision = TRUE, - causal_temporal_positioning = FALSE, - rope_type = "split" -) -} -\arguments{ -\item{caption_channels}{Integer. Dimension of caption embeddings (default 3840).} - -\item{text_proj_in_factor}{Integer. Factor for input projection (default 1).} - -\item{video_connector_num_attention_heads}{Integer. Number of attention heads for video connector.} - -\item{video_connector_attention_head_dim}{Integer. Attention head dimension for video.} - -\item{video_connector_num_layers}{Integer. Number of transformer layers for video.} - -\item{video_connector_num_learnable_registers}{Integer. Number of learnable registers for video.} - -\item{audio_connector_num_attention_heads}{Integer. Number of attention heads for audio connector.} - -\item{audio_connector_attention_head_dim}{Integer. Attention head dimension for audio.} - -\item{audio_connector_num_layers}{Integer. Number of transformer layers for audio.} - -\item{audio_connector_num_learnable_registers}{Integer. Number of learnable registers for audio.} - -\item{connector_rope_base_seq_len}{Integer. Base sequence length for RoPE.} - -\item{rope_theta}{Numeric. RoPE theta parameter.} - -\item{rope_double_precision}{Logical. Use double precision for RoPE.} - -\item{causal_temporal_positioning}{Logical. Use causal temporal positioning.} - -\item{rope_type}{Character. RoPE type ("interleaved" or "split").} -} -\value{ -nn_module for text connectors. -} -\description{ -Transforms packed text encoder hidden states for video and audio streams. -} diff --git a/man/ltx2_transformer_block_1d.Rd b/man/ltx2_transformer_block_1d.Rd deleted file mode 100644 index 2944e1e..0000000 --- a/man/ltx2_transformer_block_1d.Rd +++ /dev/null @@ -1,18 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{ltx2_transformer_block_1d} -\alias{ltx2_transformer_block_1d} -\title{1D Transformer Block for LTX2 Text Connectors} -\usage{ -ltx2_transformer_block_1d( - dim, - num_attention_heads, - attention_head_dim, - activation_fn = "gelu-approximate", - eps = 1e-06, - rope_type = "interleaved" -) -} -\description{ -1D Transformer Block for LTX2 Text Connectors -} -\keyword{internal} diff --git a/man/ltx2_video_causal_conv3d.Rd b/man/ltx2_video_causal_conv3d.Rd deleted file mode 100644 index 5d28977..0000000 --- a/man/ltx2_video_causal_conv3d.Rd +++ /dev/null @@ -1,35 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{ltx2_video_causal_conv3d} -\alias{ltx2_video_causal_conv3d} -\title{LTX2 Video Causal 3D Convolution} -\usage{ -ltx2_video_causal_conv3d( - in_channels, - out_channels, - kernel_size = 3L, - stride = 1L, - dilation = 1L, - groups = 1L, - spatial_padding_mode = "zeros" -) -} -\arguments{ -\item{in_channels}{Integer. Input channels.} - -\item{out_channels}{Integer. Output channels.} - -\item{kernel_size}{Integer or vector of 3. Convolution kernel size.} - -\item{stride}{Integer or vector of 3. Stride.} - -\item{dilation}{Integer or vector of 3. Dilation.} - -\item{groups}{Integer. Convolution groups. Default: 1} - -\item{spatial_padding_mode}{Character. Padding mode for spatial dims. Default: "zeros"} -} -\description{ -3D convolution with runtime-selectable causal or non-causal padding. -Causal mode pads temporally by repeating first frame. -Non-causal mode pads temporally by repeating first and last frames. -} diff --git a/man/ltx2_video_decoder3d.Rd b/man/ltx2_video_decoder3d.Rd deleted file mode 100644 index 5b2c3a8..0000000 --- a/man/ltx2_video_decoder3d.Rd +++ /dev/null @@ -1,54 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{ltx2_video_decoder3d} -\alias{ltx2_video_decoder3d} -\title{LTX2 Video Decoder} -\usage{ -ltx2_video_decoder3d( - in_channels = 128L, - out_channels = 3L, - block_out_channels = c(256L, 512L, 1024L), - spatio_temporal_scaling = c(TRUE, TRUE, TRUE), - layers_per_block = c(5L, 5L, 5L, 5L), - patch_size = 4L, - patch_size_t = 1L, - resnet_norm_eps = 1e-06, - is_causal = FALSE, - inject_noise = c(FALSE, FALSE, FALSE, FALSE), - timestep_conditioning = FALSE, - upsample_residual = c(TRUE, TRUE, TRUE), - upsample_factor = c(2L, 2L, 2L), - spatial_padding_mode = "reflect" -) -} -\arguments{ -\item{in_channels}{Integer. Latent channels.} - -\item{out_channels}{Integer. Output channels (typically 3 for RGB).} - -\item{block_out_channels}{Integer vector. Output channels per block.} - -\item{spatio_temporal_scaling}{Logical vector. Whether each block upscales.} - -\item{layers_per_block}{Integer vector. Number of layers per block.} - -\item{patch_size}{Integer. Spatial patch size.} - -\item{patch_size_t}{Integer. Temporal patch size.} - -\item{resnet_norm_eps}{Numeric. Epsilon for normalization.} - -\item{is_causal}{Logical. Whether to use causal convolutions.} - -\item{inject_noise}{Logical vector. Whether to inject noise per block.} - -\item{timestep_conditioning}{Logical. Whether to use timestep conditioning.} - -\item{upsample_residual}{Logical vector. Whether upsamplers use residual.} - -\item{upsample_factor}{Integer vector. Channel upscale factors.} - -\item{spatial_padding_mode}{Character. Padding mode.} -} -\description{ -Decodes latent representations back to video frames. -} diff --git a/man/ltx2_video_down_block3d.Rd b/man/ltx2_video_down_block3d.Rd deleted file mode 100644 index 6cbb8b0..0000000 --- a/man/ltx2_video_down_block3d.Rd +++ /dev/null @@ -1,39 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{ltx2_video_down_block3d} -\alias{ltx2_video_down_block3d} -\title{LTX2 Video Down Block 3D} -\usage{ -ltx2_video_down_block3d( - in_channels, - out_channels = NULL, - num_layers = 1L, - dropout = 0, - resnet_eps = 1e-06, - resnet_act_fn = "swish", - spatio_temporal_scale = TRUE, - downsample_type = "conv", - spatial_padding_mode = "zeros" -) -} -\arguments{ -\item{in_channels}{Integer. Input channels.} - -\item{out_channels}{Integer or NULL. Output channels.} - -\item{num_layers}{Integer. Number of ResNet layers.} - -\item{dropout}{Numeric. Dropout rate.} - -\item{resnet_eps}{Numeric. Epsilon for normalization.} - -\item{resnet_act_fn}{Character. Activation function.} - -\item{spatio_temporal_scale}{Logical. Whether to use downsampling.} - -\item{downsample_type}{Character. Type of downsampling.} - -\item{spatial_padding_mode}{Character. Padding mode.} -} -\description{ -Encoder down block with multiple ResNet layers and optional downsampling. -} diff --git a/man/ltx2_video_encoder3d.Rd b/man/ltx2_video_encoder3d.Rd deleted file mode 100644 index a1eafc8..0000000 --- a/man/ltx2_video_encoder3d.Rd +++ /dev/null @@ -1,45 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{ltx2_video_encoder3d} -\alias{ltx2_video_encoder3d} -\title{LTX2 Video Encoder} -\usage{ -ltx2_video_encoder3d( - in_channels = 3L, - out_channels = 128L, - block_out_channels = c(256L, 512L, 1024L, 2048L), - spatio_temporal_scaling = c(TRUE, TRUE, TRUE, TRUE), - layers_per_block = c(4L, 6L, 6L, 2L, 2L), - downsample_type = c("spatial", "temporal", "spatiotemporal", "spatiotemporal"), - patch_size = 4L, - patch_size_t = 1L, - resnet_norm_eps = 1e-06, - is_causal = TRUE, - spatial_padding_mode = "zeros" -) -} -\arguments{ -\item{in_channels}{Integer. Input channels (typically 3 for RGB).} - -\item{out_channels}{Integer. Latent channels.} - -\item{block_out_channels}{Integer vector. Output channels per block.} - -\item{spatio_temporal_scaling}{Logical vector. Whether each block downscales.} - -\item{layers_per_block}{Integer vector. Number of layers per block.} - -\item{downsample_type}{Character vector. Type of downsampling per block.} - -\item{patch_size}{Integer. Spatial patch size.} - -\item{patch_size_t}{Integer. Temporal patch size.} - -\item{resnet_norm_eps}{Numeric. Epsilon for normalization.} - -\item{is_causal}{Logical. Whether to use causal convolutions.} - -\item{spatial_padding_mode}{Character. Padding mode.} -} -\description{ -Encodes video frames into latent space with 3D causal convolutions. -} diff --git a/man/ltx2_video_mid_block3d.Rd b/man/ltx2_video_mid_block3d.Rd deleted file mode 100644 index 1d87390..0000000 --- a/man/ltx2_video_mid_block3d.Rd +++ /dev/null @@ -1,36 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{ltx2_video_mid_block3d} -\alias{ltx2_video_mid_block3d} -\title{LTX2 Video Mid Block 3D} -\usage{ -ltx2_video_mid_block3d( - in_channels, - num_layers = 1L, - dropout = 0, - resnet_eps = 1e-06, - resnet_act_fn = "swish", - inject_noise = FALSE, - timestep_conditioning = FALSE, - spatial_padding_mode = "zeros" -) -} -\arguments{ -\item{in_channels}{Integer. Input channels.} - -\item{num_layers}{Integer. Number of ResNet layers.} - -\item{dropout}{Numeric. Dropout rate.} - -\item{resnet_eps}{Numeric. Epsilon for normalization.} - -\item{resnet_act_fn}{Character. Activation function.} - -\item{inject_noise}{Logical. Whether to inject noise.} - -\item{timestep_conditioning}{Logical. Whether to use timestep conditioning.} - -\item{spatial_padding_mode}{Character. Padding mode.} -} -\description{ -Middle block with ResNet layers and optional timestep conditioning. -} diff --git a/man/ltx2_video_resnet_block3d.Rd b/man/ltx2_video_resnet_block3d.Rd deleted file mode 100644 index 0d599e0..0000000 --- a/man/ltx2_video_resnet_block3d.Rd +++ /dev/null @@ -1,37 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{ltx2_video_resnet_block3d} -\alias{ltx2_video_resnet_block3d} -\title{LTX2 Video ResNet Block 3D} -\usage{ -ltx2_video_resnet_block3d( - in_channels, - out_channels = NULL, - dropout = 0, - eps = 1e-06, - non_linearity = "silu", - inject_noise = FALSE, - timestep_conditioning = FALSE, - spatial_padding_mode = "zeros" -) -} -\arguments{ -\item{in_channels}{Integer. Input channels.} - -\item{out_channels}{Integer or NULL. Output channels (defaults to in_channels).} - -\item{dropout}{Numeric. Dropout rate. Default: 0.0} - -\item{eps}{Numeric. Epsilon for normalization. Default: 1e-6} - -\item{non_linearity}{Character. Activation function. Default: "silu"} - -\item{inject_noise}{Logical. Whether to inject noise. Default: FALSE} - -\item{timestep_conditioning}{Logical. Whether to use timestep conditioning. Default: FALSE} - -\item{spatial_padding_mode}{Character. Padding mode. Default: "zeros"} -} -\description{ -3D ResNet block with per-channel RMS normalization and optional -noise injection and timestep conditioning. -} diff --git a/man/ltx2_video_transformer_3d_model.Rd b/man/ltx2_video_transformer_3d_model.Rd deleted file mode 100644 index 405a881..0000000 --- a/man/ltx2_video_transformer_3d_model.Rd +++ /dev/null @@ -1,107 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{ltx2_video_transformer_3d_model} -\alias{ltx2_video_transformer_3d_model} -\title{LTX2 Video Transformer 3D Model (Audio-Video)} -\usage{ -ltx2_video_transformer_3d_model( - in_channels, - out_channels, - patch_size, - patch_size_t, - num_attention_heads, - attention_head_dim, - cross_attention_dim, - vae_scale_factors, - 32L, - 32L), - pos_embed_max_pos, - base_height, - base_width, - audio_in_channels, - audio_out_channels, - audio_patch_size, - audio_patch_size_t, - audio_num_attention_heads, - audio_attention_head_dim, - audio_cross_attention_dim -) -} -\arguments{ -\item{in_channels}{Integer. Video input channels (default: 128).} - -\item{out_channels}{Integer. Video output channels (default: 128).} - -\item{patch_size}{Integer. Spatial patch size (default: 1).} - -\item{patch_size_t}{Integer. Temporal patch size (default: 1).} - -\item{num_attention_heads}{Integer. Video attention heads (default: 32).} - -\item{attention_head_dim}{Integer. Video attention head dimension (default: 128).} - -\item{cross_attention_dim}{Integer. Video cross-attention dimension (default: 4096).} - -\item{vae_scale_factors}{Integer vector. VAE scale factors (default: c(8, 32, 32)).} - -\item{pos_embed_max_pos}{Integer. Max position for RoPE (default: 20).} - -\item{base_height}{Integer. Base height for RoPE (default: 2048).} - -\item{base_width}{Integer. Base width for RoPE (default: 2048).} - -\item{audio_in_channels}{Integer. Audio input channels (default: 128).} - -\item{audio_out_channels}{Integer. Audio output channels (default: 128).} - -\item{audio_patch_size}{Integer. Audio patch size (default: 1).} - -\item{audio_patch_size_t}{Integer. Audio temporal patch size (default: 1).} - -\item{audio_num_attention_heads}{Integer. Audio attention heads (default: 32).} - -\item{audio_attention_head_dim}{Integer. Audio head dimension (default: 64).} - -\item{audio_cross_attention_dim}{Integer. Audio cross-attention dim (default: 2048).} - -\item{audio_scale_factor}{Integer. Audio scale factor (default: 4).} - -\item{audio_pos_embed_max_pos}{Integer. Audio max position (default: 20).} - -\item{audio_sampling_rate}{Integer. Audio sampling rate (default: 16000).} - -\item{audio_hop_length}{Integer. Audio hop length (default: 160).} - -\item{num_layers}{Integer. Number of transformer layers (default: 48).} - -\item{activation_fn}{Character. Activation function (default: "gelu-approximate").} - -\item{qk_norm}{Character. QK normalization type (default: "rms_norm_across_heads").} - -\item{norm_elementwise_affine}{Logical. Use elementwise affine in norms (default: FALSE).} - -\item{norm_eps}{Numeric. Epsilon for normalization (default: 1e-6).} - -\item{caption_channels}{Integer. Caption embedding channels (default: 3840).} - -\item{attention_bias}{Logical. Use bias in attention (default: TRUE).} - -\item{attention_out_bias}{Logical. Use bias in attention output (default: TRUE).} - -\item{rope_theta}{Numeric. Theta for RoPE (default: 10000).} - -\item{rope_double_precision}{Logical. Use double precision for RoPE (default: TRUE).} - -\item{causal_offset}{Integer. Causal offset for RoPE (default: 1).} - -\item{timestep_scale_multiplier}{Numeric. Timestep scale (default: 1000).} - -\item{cross_attn_timestep_scale_multiplier}{Numeric. Cross-attn timestep scale (default: 1000).} - -\item{rope_type}{Character. RoPE type: "interleaved" or "split" (default: "interleaved").} -} -\value{ -An nn_module representing the LTX2 video transformer. -} -\description{ -Full audio-video transformer matching HuggingFace diffusers implementation. -} diff --git a/man/ltx2_video_transformer_block.Rd b/man/ltx2_video_transformer_block.Rd deleted file mode 100644 index a66683e..0000000 --- a/man/ltx2_video_transformer_block.Rd +++ /dev/null @@ -1,27 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{ltx2_video_transformer_block} -\alias{ltx2_video_transformer_block} -\title{LTX2 Video Transformer Block (Audio-Video)} -\usage{ -ltx2_video_transformer_block( - dim, - num_attention_heads, - attention_head_dim, - cross_attention_dim, - audio_dim, - audio_num_attention_heads, - audio_attention_head_dim, - audio_cross_attention_dim, - qk_norm = "rms_norm_across_heads", - activation_fn = "gelu-approximate", - attention_bias = TRUE, - attention_out_bias = TRUE, - eps = 1e-06, - elementwise_affine = FALSE, - rope_type = "interleaved" -) -} -\description{ -LTX2 Video Transformer Block (Audio-Video) -} -\keyword{internal} diff --git a/man/ltx2_video_up_block3d.Rd b/man/ltx2_video_up_block3d.Rd deleted file mode 100644 index e3f7c70..0000000 --- a/man/ltx2_video_up_block3d.Rd +++ /dev/null @@ -1,48 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{ltx2_video_up_block3d} -\alias{ltx2_video_up_block3d} -\title{LTX2 Video Up Block 3D} -\usage{ -ltx2_video_up_block3d( - in_channels, - out_channels = NULL, - num_layers = 1L, - dropout = 0, - resnet_eps = 1e-06, - resnet_act_fn = "swish", - spatio_temporal_scale = TRUE, - inject_noise = FALSE, - timestep_conditioning = FALSE, - upsample_residual = FALSE, - upscale_factor = 1L, - spatial_padding_mode = "zeros" -) -} -\arguments{ -\item{in_channels}{Integer. Input channels.} - -\item{out_channels}{Integer or NULL. Output channels.} - -\item{num_layers}{Integer. Number of ResNet layers.} - -\item{dropout}{Numeric. Dropout rate.} - -\item{resnet_eps}{Numeric. Epsilon for normalization.} - -\item{resnet_act_fn}{Character. Activation function.} - -\item{spatio_temporal_scale}{Logical. Whether to use upsampling.} - -\item{inject_noise}{Logical. Whether to inject noise.} - -\item{timestep_conditioning}{Logical. Whether to use timestep conditioning.} - -\item{upsample_residual}{Logical. Whether upsampler uses residual.} - -\item{upscale_factor}{Integer. Channel upscale factor.} - -\item{spatial_padding_mode}{Character. Padding mode.} -} -\description{ -Decoder up block with upsampling and ResNet layers. -} diff --git a/man/ltx2_video_vae.Rd b/man/ltx2_video_vae.Rd deleted file mode 100644 index 0015e11..0000000 --- a/man/ltx2_video_vae.Rd +++ /dev/null @@ -1,101 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{ltx2_video_vae} -\alias{ltx2_video_vae} -\title{LTX2 Video VAE} -\usage{ -ltx2_video_vae( - in_channels, - out_channels, - latent_channels, - block_out_channels, - 512L, - 1024L, - 2048L), - decoder_block_out_channels, - 512L, - 1024L), - layers_per_block, - 6L, - 6L, - 2L, - 2L), - decoder_layers_per_block, - 5L, - 5L, - 5L), - spatio_temporal_scaling, - TRUE, - TRUE, - TRUE), - decoder_spatio_temporal_scaling, - TRUE, - TRUE), - decoder_inject_noise, - FALSE, - FALSE, - FALSE), - downsample_type, - "temporal", - "spatiotemporal", - "spatiotemporal"), - upsample_residual, - TRUE, - TRUE), - upsample_factor, - 2L, - 2L), - timestep_conditioning, - patch_size, - patch_size_t, - resnet_norm_eps -) -} -\arguments{ -\item{in_channels}{Integer. Input channels.} - -\item{out_channels}{Integer. Output channels.} - -\item{latent_channels}{Integer. Latent space channels.} - -\item{block_out_channels}{Integer vector. Encoder block channels.} - -\item{decoder_block_out_channels}{Integer vector. Decoder block channels.} - -\item{layers_per_block}{Integer vector. Encoder layers per block.} - -\item{decoder_layers_per_block}{Integer vector. Decoder layers per block.} - -\item{spatio_temporal_scaling}{Logical vector. Encoder scaling.} - -\item{decoder_spatio_temporal_scaling}{Logical vector. Decoder scaling.} - -\item{decoder_inject_noise}{Logical vector. Noise injection per decoder block.} - -\item{downsample_type}{Character vector. Downsampling types.} - -\item{upsample_residual}{Logical vector. Upsampler residual flags.} - -\item{upsample_factor}{Integer vector. Upsampler factors.} - -\item{timestep_conditioning}{Logical. Whether to use timestep conditioning.} - -\item{patch_size}{Integer. Spatial patch size.} - -\item{patch_size_t}{Integer. Temporal patch size.} - -\item{resnet_norm_eps}{Numeric. Normalization epsilon.} - -\item{scaling_factor}{Numeric. Latent scaling factor.} - -\item{encoder_causal}{Logical. Encoder causality.} - -\item{decoder_causal}{Logical. Decoder causality.} - -\item{encoder_spatial_padding_mode}{Character. Encoder padding mode.} - -\item{decoder_spatial_padding_mode}{Character. Decoder padding mode.} -} -\description{ -Full VAE with encoder and decoder, supporting tiled encoding/decoding -for GPU-poor memory management. -} diff --git a/man/ltx_video_downsampler3d.Rd b/man/ltx_video_downsampler3d.Rd deleted file mode 100644 index c908f87..0000000 --- a/man/ltx_video_downsampler3d.Rd +++ /dev/null @@ -1,24 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{ltx_video_downsampler3d} -\alias{ltx_video_downsampler3d} -\title{LTX Video Downsampler 3D} -\usage{ -ltx_video_downsampler3d( - in_channels, - out_channels, - stride = 1L, - spatial_padding_mode = "zeros" -) -} -\arguments{ -\item{in_channels}{Integer. Input channels.} - -\item{out_channels}{Integer. Output channels.} - -\item{stride}{Integer or vector of 3. Downsampling stride.} - -\item{spatial_padding_mode}{Character. Padding mode.} -} -\description{ -Spatiotemporal downsampling with strided pixel unshuffle + convolution. -} diff --git a/man/ltx_video_upsampler3d.Rd b/man/ltx_video_upsampler3d.Rd deleted file mode 100644 index 58f5176..0000000 --- a/man/ltx_video_upsampler3d.Rd +++ /dev/null @@ -1,27 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{ltx_video_upsampler3d} -\alias{ltx_video_upsampler3d} -\title{LTX Video Upsampler 3D} -\usage{ -ltx_video_upsampler3d( - in_channels, - stride = 1L, - residual = FALSE, - upscale_factor = 1L, - spatial_padding_mode = "zeros" -) -} -\arguments{ -\item{in_channels}{Integer. Input channels.} - -\item{stride}{Integer or vector of 3. Upsampling stride.} - -\item{residual}{Logical. Whether to use residual connection.} - -\item{upscale_factor}{Integer. Channel upscale factor.} - -\item{spatial_padding_mode}{Character. Padding mode.} -} -\description{ -Spatiotemporal upsampling with pixel shuffle + optional residual. -} diff --git a/man/make_linear.Rd b/man/make_linear.Rd deleted file mode 100644 index 0dc97d5..0000000 --- a/man/make_linear.Rd +++ /dev/null @@ -1,27 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{make_linear} -\alias{make_linear} -\title{Create Linear Layer (Standard or INT4)} -\usage{ -make_linear(in_features, out_features, bias = TRUE) -} -\arguments{ -\item{in_features}{Integer. Input dimension.} - -\item{out_features}{Integer. Output dimension.} - -\item{bias}{Logical. Include bias term.} -} -\value{ -nn_linear or int4_linear module. -} -\description{ -Factory function that creates either a standard nn_linear or INT4 linear layer -based on package options. Use this instead of torch::nn_linear() in model code. -} -\details{ -Behavior controlled by options: -- `diffuseR.use_int4`: If TRUE, create INT4 layer (default FALSE) -- `diffuseR.int4_device`: Device for INT4 layers (default "cuda") -- `diffuseR.int4_dtype`: Dtype for INT4 operations (default torch_float16()) -} diff --git a/man/models2devices.Rd b/man/models2devices.Rd index 4bee2f4..607e1b6 100644 --- a/man/models2devices.Rd +++ b/man/models2devices.Rd @@ -3,12 +3,8 @@ \alias{models2devices} \title{models2devices} \usage{ -models2devices( - model_name, - devices = "cpu", - unet_dtype_str = NULL, - download_models = FALSE -) +models2devices(model_name, devices = "cpu", unet_dtype_str = NULL, + download_models = FALSE) } \arguments{ \item{model_name}{A character string representing the name of the model to be used.} diff --git a/man/pack_text_embeds.Rd b/man/pack_text_embeds.Rd deleted file mode 100644 index aeacde2..0000000 --- a/man/pack_text_embeds.Rd +++ /dev/null @@ -1,34 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{pack_text_embeds} -\alias{pack_text_embeds} -\title{Pack Text Embeddings (Gemma-style)} -\usage{ -pack_text_embeds( - text_hidden_states, - sequence_lengths, - padding_side = "left", - scale_factor = 8, - eps = 1e-06, - device = "cpu" -) -} -\arguments{ -\item{text_hidden_states}{Tensor of shape [batch, seq_len, hidden_dim, num_layers].} - -\item{sequence_lengths}{Integer vector of valid sequence lengths per batch item.} - -\item{padding_side}{Character. "left" or "right".} - -\item{scale_factor}{Numeric. Scale factor for normalization (default 8).} - -\item{eps}{Numeric. Epsilon for numerical stability.} - -\item{device}{Character. Device for tensors.} -} -\value{ -Tensor of shape [batch, seq_len, hidden_dim * num_layers]. -} -\description{ -Normalizes and packs text encoder hidden states from multiple layers. -This is used when working with raw Gemma outputs. -} diff --git a/man/pack_video_latents.Rd b/man/pack_video_latents.Rd deleted file mode 100644 index 0669042..0000000 --- a/man/pack_video_latents.Rd +++ /dev/null @@ -1,21 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{pack_video_latents} -\alias{pack_video_latents} -\title{Pack Video Latents for DiT} -\usage{ -pack_video_latents(latents, patch_size = 1L, patch_size_t = 1L) -} -\arguments{ -\item{latents}{Tensor of shape \[B, C, F, H, W\].} - -\item{patch_size}{Integer. Spatial patch size (default 1).} - -\item{patch_size_t}{Integer. Temporal patch size (default 1).} -} -\value{ -Packed tensor of shape \[B, num_patches, C*patch_size_t*patch_size^2\]. -} -\description{ -Transforms 5D video latents \[B, C, F, H, W\] to 3D sequence \[B, F*H*W, C\] -for input to the DiT transformer. -} diff --git a/man/per_channel_rms_norm.Rd b/man/per_channel_rms_norm.Rd deleted file mode 100644 index 9c7b198..0000000 --- a/man/per_channel_rms_norm.Rd +++ /dev/null @@ -1,16 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{per_channel_rms_norm} -\alias{per_channel_rms_norm} -\title{Per-channel RMS normalization} -\usage{ -per_channel_rms_norm(channel_dim = 2L, eps = 1e-08) -} -\arguments{ -\item{channel_dim}{Integer. Dimension for RMS computation (1-indexed). Default: 2 (channels)} - -\item{eps}{Numeric. Small constant for numerical stability. Default: 1e-8} -} -\description{ -Normalizes tensor by root-mean-square along the channel dimension: -y = x / sqrt(mean(x^2, dim=channel_dim, keepdim=TRUE) + eps) -} diff --git a/man/pixart_alpha_combined_timestep_size_embeddings.Rd b/man/pixart_alpha_combined_timestep_size_embeddings.Rd deleted file mode 100644 index c665a34..0000000 --- a/man/pixart_alpha_combined_timestep_size_embeddings.Rd +++ /dev/null @@ -1,15 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{pixart_alpha_combined_timestep_size_embeddings} -\alias{pixart_alpha_combined_timestep_size_embeddings} -\title{PixArt Alpha Combined Timestep Size Embeddings} -\usage{ -pixart_alpha_combined_timestep_size_embeddings( - embedding_dim, - size_emb_dim, - use_additional_conditions = FALSE -) -} -\description{ -PixArt Alpha Combined Timestep Size Embeddings -} -\keyword{internal} diff --git a/man/pixart_alpha_text_projection.Rd b/man/pixart_alpha_text_projection.Rd deleted file mode 100644 index c9ec192..0000000 --- a/man/pixart_alpha_text_projection.Rd +++ /dev/null @@ -1,16 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{pixart_alpha_text_projection} -\alias{pixart_alpha_text_projection} -\title{PixArt Alpha Text Projection} -\usage{ -pixart_alpha_text_projection( - in_features, - hidden_size, - out_features = NULL, - act_fn = "gelu_tanh" -) -} -\description{ -PixArt Alpha Text Projection -} -\keyword{internal} diff --git a/man/quantize_int4.Rd b/man/quantize_int4.Rd deleted file mode 100644 index d9bbe43..0000000 --- a/man/quantize_int4.Rd +++ /dev/null @@ -1,36 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{quantize_int4} -\alias{quantize_int4} -\title{Quantize Tensor to INT4} -\usage{ -quantize_int4(x, block_size = 64L) -} -\arguments{ -\item{x}{Tensor. Input float tensor.} - -\item{block_size}{Integer. Number of values per scale factor (default 64).} -} -\value{ -A list with: - - `packed`: uint8 tensor with packed INT4 values - - `scales`: float tensor with per-block scale factors - - `orig_shape`: original tensor shape - - `orig_numel`: original number of elements - - `block_size`: block size used -} -\description{ -Quantizes a float tensor to 4-bit integers with block-wise scaling. -Two INT4 values are packed per byte for 7-8x compression. -} -\details{ -INT4 range is -8 to 7. Values are scaled per block, quantized, shifted to -unsigned (0-15), and packed two per byte. Compression is ~7x vs float32, -~3.5x vs float16. Typical reconstruction error is 10-12\% of std. -} -\examples{ -\dontrun{ -x <- torch_randn(c(4096, 4096)) * 0.02 -q <- quantize_int4(x) -x_back <- dequantize_int4(q) -} -} diff --git a/man/quantize_ltx2_transformer.Rd b/man/quantize_ltx2_transformer.Rd deleted file mode 100644 index 8ca3b4a..0000000 --- a/man/quantize_ltx2_transformer.Rd +++ /dev/null @@ -1,58 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{quantize_ltx2_transformer} -\alias{quantize_ltx2_transformer} -\title{Quantize LTX-2 Transformer to INT4} -\usage{ -quantize_ltx2_transformer( - model_id = "Lightricks/LTX-2", - output_dir = NULL, - block_size = 64L, - force = FALSE, - download = FALSE, - verbose = TRUE -) -} -\arguments{ -\item{model_id}{Character. HuggingFace model ID (default "Lightricks/LTX-2").} - -\item{output_dir}{Character. Directory to save quantized weights. -Default uses `tools::R_user_dir("diffuseR", "cache")`.} - -\item{block_size}{Integer. Block size for INT4 quantization (default 64).} - -\item{force}{Logical. Re-quantize even if cached file exists.} - -\item{download}{Logical. If TRUE, download model from HuggingFace if not cached.} - -\item{verbose}{Logical. Print progress.} -} -\value{ -Character. Path to the quantized weights file. -} -\description{ -Downloads (if needed) and quantizes the LTX-2 19B transformer to INT4 format. -The quantized weights are cached for future use. -} -\details{ -LTX-2 is a 19B parameter model (~40GB in BF16). INT4 quantization reduces -this to ~5.7GB, fitting in 16GB VRAM with room for activations. - -The function: -1. Uses hfhub to locate/download the model from HuggingFace -2. Loads each safetensor shard -3. Quantizes all weights to INT4 (block-wise, 64 values per scale) -4. Saves as safetensors file -} -\examples{ -\dontrun{ -# Quantize and cache (first run takes ~10-20 minutes) -weights_path <- quantize_ltx2_transformer() - -# Load quantized weights -q <- load_int4_weights(weights_path) - -# Dequantize specific layer on GPU -layer_weight <- dequantize_int4(q[["transformer_blocks.0.attn1.to_q.weight"]], - device = "cuda", dtype = torch_float16()) -} -} diff --git a/man/quantize_ltx2_vae.Rd b/man/quantize_ltx2_vae.Rd deleted file mode 100644 index b02eb48..0000000 --- a/man/quantize_ltx2_vae.Rd +++ /dev/null @@ -1,34 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{quantize_ltx2_vae} -\alias{quantize_ltx2_vae} -\title{Quantize LTX-2 VAE to INT4} -\usage{ -quantize_ltx2_vae( - model_id = "Lightricks/LTX-2", - output_dir = NULL, - block_size = 64L, - force = FALSE, - download = FALSE, - verbose = TRUE -) -} -\arguments{ -\item{model_id}{Character. HuggingFace model ID (default "Lightricks/LTX-2").} - -\item{output_dir}{Character. Directory to save quantized weights. -Default uses `tools::R_user_dir("diffuseR", "cache")`.} - -\item{block_size}{Integer. Block size for INT4 quantization.} - -\item{force}{Logical. Re-quantize even if cached file exists.} - -\item{download}{Logical. If TRUE, download model from HuggingFace if not cached.} - -\item{verbose}{Logical. Print progress.} -} -\value{ -Character. Path to the quantized weights file. -} -\description{ -Quantizes the LTX-2 video VAE to INT4 format. -} diff --git a/man/quantize_model_int4.Rd b/man/quantize_model_int4.Rd deleted file mode 100644 index 92c4c34..0000000 --- a/man/quantize_model_int4.Rd +++ /dev/null @@ -1,20 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{quantize_model_int4} -\alias{quantize_model_int4} -\title{Quantize Model Weights to INT4} -\usage{ -quantize_model_int4(module, block_size = 64L, verbose = TRUE) -} -\arguments{ -\item{module}{nn_module. The model to quantize.} - -\item{block_size}{Integer. Block size for quantization.} - -\item{verbose}{Logical. Print progress.} -} -\value{ -List of quantized parameters (does not modify original module). -} -\description{ -Quantizes all parameters in a torch module to INT4 format. -} diff --git a/man/quantize_safetensors_int4.Rd b/man/quantize_safetensors_int4.Rd deleted file mode 100644 index 68f842e..0000000 --- a/man/quantize_safetensors_int4.Rd +++ /dev/null @@ -1,31 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{quantize_safetensors_int4} -\alias{quantize_safetensors_int4} -\title{Quantize Safetensor Weights to INT4} -\usage{ -quantize_safetensors_int4(paths, output_path, block_size = 64L, verbose = TRUE) -} -\arguments{ -\item{paths}{Character vector. Paths to safetensor files.} - -\item{output_path}{Character. Path to save INT4 weights (.safetensors).} - -\item{block_size}{Integer. Block size for quantization (default 64).} - -\item{verbose}{Logical. Print progress.} -} -\value{ -Invisible NULL. Writes quantized weights to output_path. -} -\description{ -Loads weights from safetensors file(s) and quantizes to INT4. -Useful for quantizing large models without loading the full module. -} -\examples{ -\dontrun{ -# Quantize sharded transformer weights -paths <- list.files("~/.cache/huggingface/.../transformer", - pattern = "safetensors$", full.names = TRUE) -quantize_safetensors_int4(paths, "dit_int4.safetensors") -} -} diff --git a/man/rms_norm.Rd b/man/rms_norm.Rd deleted file mode 100644 index 13a1c29..0000000 --- a/man/rms_norm.Rd +++ /dev/null @@ -1,11 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{rms_norm} -\alias{rms_norm} -\title{RMS Normalization} -\usage{ -rms_norm(dim, eps = 1e-06, elementwise_affine = TRUE) -} -\description{ -RMS Normalization -} -\keyword{internal} diff --git a/man/rope.Rd b/man/rope.Rd deleted file mode 100644 index f9af843..0000000 --- a/man/rope.Rd +++ /dev/null @@ -1,14 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{rope} -\alias{rope} -\title{Rotary Position Embeddings (RoPE) for LTX-2 Video Models} -\description{ -Implementation of rotary positional embeddings for 3D video (spatiotemporal) -coordinates as used in LTX-2. RoPE encodes position information directly -into the attention queries and keys without adding position embeddings. -} -\references{ -Su, J., Lu, Y., Pan, S., Murtadha, A., Wen, B., & Liu, Y. (2021). -"RoFormer: Enhanced Transformer with Rotary Position Embedding." -\url{https://arxiv.org/abs/2104.09864} -} diff --git a/man/rope_embedder_create.Rd b/man/rope_embedder_create.Rd deleted file mode 100644 index 1ac953a..0000000 --- a/man/rope_embedder_create.Rd +++ /dev/null @@ -1,54 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{rope_embedder_create} -\alias{rope_embedder_create} -\title{Create RoPE position embedder for video} -\usage{ -rope_embedder_create( - dim, - patch_size = 1L, - patch_size_t = 1L, - base_num_frames = 20L, - base_height = 2048L, - base_width = 2048L, - scale_factors = c(8, 32, 32), - theta = 10000, - causal_offset = 1L, - double_precision = TRUE, - rope_type = c("interleaved", "split"), - num_attention_heads = 32L -) -} -\arguments{ -\item{dim}{Integer. Dimension for RoPE (typically attention head dim * num_heads).} - -\item{patch_size}{Integer. Spatial patch size. Default: 1} - -\item{patch_size_t}{Integer. Temporal patch size. Default: 1} - -\item{base_num_frames}{Integer. Base number of frames for normalization. Default: 20} - -\item{base_height}{Integer. Base height for normalization. Default: 2048} - -\item{base_width}{Integer. Base width for normalization. Default: 2048} - -\item{scale_factors}{Numeric vector of length 3. VAE scale factors for -(temporal, height, width). Default: c(8, 32, 32)} - -\item{theta}{Numeric. Base frequency for RoPE. Default: 10000.0} - -\item{causal_offset}{Integer. Offset for causal VAE modeling. Default: 1} - -\item{double_precision}{Logical. Whether to use float64 for frequency -computation. Default: TRUE} - -\item{rope_type}{Character. Type of RoPE: "interleaved" or "split". Default: "interleaved"} - -\item{num_attention_heads}{Integer. Number of attention heads (for split RoPE). Default: 32} -} -\value{ -A list containing RoPE configuration and methods. -} -\description{ -Creates a RoPE embedder configured for video generation, handling -spatiotemporal coordinates (frames, height, width). -} diff --git a/man/rope_forward.Rd b/man/rope_forward.Rd deleted file mode 100644 index def96b5..0000000 --- a/man/rope_forward.Rd +++ /dev/null @@ -1,25 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{rope_forward} -\alias{rope_forward} -\title{Compute RoPE frequencies from coordinates} -\usage{ -rope_forward(embedder, coords, device = NULL) -} -\arguments{ -\item{embedder}{List. RoPE embedder configuration.} - -\item{coords}{torch tensor. Coordinate tensor from rope_prepare_video_coords().} - -\item{device}{Character or torch device. Device for output tensors.} -} -\value{ -A list with: - \describe{ - \item{cos_freqs}{Cosine frequencies tensor} - \item{sin_freqs}{Sine frequencies tensor} - } -} -\description{ -Converts spatiotemporal coordinates to (cos, sin) frequency tensors -for applying rotary embeddings. -} diff --git a/man/rope_prepare_video_coords.Rd b/man/rope_prepare_video_coords.Rd deleted file mode 100644 index 7765da9..0000000 --- a/man/rope_prepare_video_coords.Rd +++ /dev/null @@ -1,38 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{rope_prepare_video_coords} -\alias{rope_prepare_video_coords} -\title{Prepare video coordinates for RoPE} -\usage{ -rope_prepare_video_coords( - embedder, - batch_size, - num_frames, - height, - width, - device = "cpu", - fps = 24 -) -} -\arguments{ -\item{embedder}{List. RoPE embedder configuration.} - -\item{batch_size}{Integer. Batch size.} - -\item{num_frames}{Integer. Number of latent frames.} - -\item{height}{Integer. Latent height.} - -\item{width}{Integer. Latent width.} - -\item{device}{Character or torch device. Device for tensors.} - -\item{fps}{Numeric. Video frames per second. Default: 24.0} -} -\value{ -torch tensor of shape (batch_size, 3, num_patches, 2). -} -\description{ -Creates per-dimension patch boundaries for video coordinates in pixel space. -Returns tensor of shape (batch_size, 3, num_patches, 2) where dimension 1 -represents (frame, height, width) and dimension 3 represents (start, end). -} diff --git a/man/save_int4_weights.Rd b/man/save_int4_weights.Rd deleted file mode 100644 index 7457673..0000000 --- a/man/save_int4_weights.Rd +++ /dev/null @@ -1,46 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{save_int4_weights} -\alias{save_int4_weights} -\title{Save INT4 Quantized Weights} -\usage{ -save_int4_weights( - quantized_params, - path, - max_shard_size = 2e+09, - verbose = TRUE -) -} -\arguments{ -\item{quantized_params}{List of quantized parameters from `quantize_model_int4()`.} - -\item{path}{Character. Base path for safetensors files. If multiple shards needed, -files will be named `path-00001-of-NNNNN.safetensors`.} - -\item{max_shard_size}{Numeric. Maximum bytes per shard (default 2GB to avoid R -integer overflow issues).} - -\item{verbose}{Logical. Print progress.} -} -\value{ -Invisible character vector of saved file paths. -} -\description{ -Saves INT4 quantized weights to disk as sharded safetensors files. -} -\details{ -Weights are saved in safetensors format with the following structure: -\itemize{ - \item `{name}::packed` - uint8 tensor with packed INT4 values - \item `{name}::scales` - float32 tensor with per-block scales - \item `{name}::shape` - int64 tensor with original shape -} - -Large models are automatically sharded to avoid R's 2GB vector limit. -The block size is fixed at 64 (standard for INT4 quantization). -} -\examples{ -\dontrun{ -q <- quantize_model_int4(model) -save_int4_weights(q, "model_int4.safetensors") -} -} diff --git a/man/save_video.Rd b/man/save_video.Rd index 63c9f3b..8608d49 100644 --- a/man/save_video.Rd +++ b/man/save_video.Rd @@ -3,15 +3,8 @@ \alias{save_video} \title{Save Video to File} \usage{ -save_video( - video, - file, - fps = 24, - format = NULL, - backend = "auto", - quality = 85, - verbose = TRUE -) +save_video(video, file, fps = 24, format = NULL, backend = "auto", + quality = 85, verbose = TRUE) } \arguments{ \item{video}{Array of video frames with shape [T, H, W, C] where C is 3 (RGB). diff --git a/man/save_video_ffmpeg.Rd b/man/save_video_ffmpeg.Rd index 71acd55..03ceca9 100644 --- a/man/save_video_ffmpeg.Rd +++ b/man/save_video_ffmpeg.Rd @@ -3,14 +3,8 @@ \alias{save_video_ffmpeg} \title{Save Video using FFmpeg} \usage{ -save_video_ffmpeg( - video, - file, - fps = 24, - format = "mp4", - quality = 85, - verbose = TRUE -) +save_video_ffmpeg(video, file, fps = 24, format = "mp4", quality = 85, + verbose = TRUE) } \arguments{ \item{video}{Array of video frames [T, H, W, C].} diff --git a/man/save_video_frames.Rd b/man/save_video_frames.Rd deleted file mode 100644 index 1ae8832..0000000 --- a/man/save_video_frames.Rd +++ /dev/null @@ -1,24 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{save_video_frames} -\alias{save_video_frames} -\title{Save video frames to file} -\usage{ -save_video_frames(video_array, output_file, fps = 24, verbose = TRUE) -} -\arguments{ -\item{video_array}{Numeric array with dimensions [frames, height, width, channels]. -Values should be in range [0, 1].} - -\item{output_file}{Character. Path to output video file.} - -\item{fps}{Numeric. Frames per second. Default: 24.} - -\item{verbose}{Logical. Print progress messages. Default: TRUE.} -} -\value{ -Invisibly returns the output file path. -} -\description{ -Saves an array of video frames to an MP4 file using ffmpeg via the av package. -} -\keyword{internal} diff --git a/man/sequential_cfg_forward.Rd b/man/sequential_cfg_forward.Rd deleted file mode 100644 index 72e2d3d..0000000 --- a/man/sequential_cfg_forward.Rd +++ /dev/null @@ -1,46 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{sequential_cfg_forward} -\alias{sequential_cfg_forward} -\title{Sequential CFG Forward Pass} -\usage{ -sequential_cfg_forward( - model, - latents, - timestep, - prompt_embeds, - negative_prompt_embeds, - guidance_scale, - ... -) -} -\arguments{ -\item{model}{The DiT model.} - -\item{latents}{Current latent tensor.} - -\item{timestep}{Current timestep tensor.} - -\item{prompt_embeds}{Conditional prompt embeddings.} - -\item{negative_prompt_embeds}{Unconditional prompt embeddings.} - -\item{guidance_scale}{CFG scale.} - -\item{...}{Additional arguments to model forward pass.} -} -\value{ -The CFG-combined noise prediction. -} -\description{ -Runs unconditional and conditional forward passes separately to halve -peak activation memory. For GPU-poor scenarios. -} -\examples{ -\dontrun{ -noise_pred <- sequential_cfg_forward( - model, latents, timestep, - prompt_embeds, negative_prompt_embeds, - guidance_scale = 4.0 -) -} -} diff --git a/man/text_encoder2_native.Rd b/man/text_encoder2_native.Rd index a07a7c7..9661799 100644 --- a/man/text_encoder2_native.Rd +++ b/man/text_encoder2_native.Rd @@ -3,14 +3,8 @@ \alias{text_encoder2_native} \title{Native CLIP Text Encoder 2 (OpenCLIP ViT-bigG for SDXL)} \usage{ -text_encoder2_native( - vocab_size = 49408, - context_length = 77, - embed_dim = 1280, - num_layers = 32, - num_heads = 20, - mlp_dim = 5120 -) +text_encoder2_native(vocab_size = 49408, context_length = 77, embed_dim = 1280, + num_layers = 32, num_heads = 20, mlp_dim = 5120) } \arguments{ \item{vocab_size}{Vocabulary size (default 49408)} diff --git a/man/text_encoder_native.Rd b/man/text_encoder_native.Rd index 0c0fec7..2279664 100644 --- a/man/text_encoder_native.Rd +++ b/man/text_encoder_native.Rd @@ -3,15 +3,9 @@ \alias{text_encoder_native} \title{Native CLIP Text Encoder} \usage{ -text_encoder_native( - vocab_size = 49408, - context_length = 77, - embed_dim = 768, - num_layers = 12, - num_heads = 12, - mlp_dim = 3072, - apply_final_ln = TRUE -) +text_encoder_native(vocab_size = 49408, context_length = 77, embed_dim = 768, + num_layers = 12, num_heads = 12, mlp_dim = 3072, + apply_final_ln = TRUE) } \arguments{ \item{vocab_size}{Vocabulary size (default 49408)} diff --git a/man/timestep_embedding.Rd b/man/timestep_embedding.Rd index cdc0bd8..f357184 100644 --- a/man/timestep_embedding.Rd +++ b/man/timestep_embedding.Rd @@ -3,12 +3,8 @@ \alias{timestep_embedding} \title{Sinusoidal Timestep Embedding} \usage{ -timestep_embedding( - timesteps, - dim, - flip_sin_to_cos = TRUE, - downscale_freq_shift = 0L -) +timestep_embedding(timesteps, dim, flip_sin_to_cos = TRUE, + downscale_freq_shift = 0L) } \arguments{ \item{timesteps}{Tensor of timesteps (batch_size,)} diff --git a/man/timestep_embedding_module.Rd b/man/timestep_embedding_module.Rd deleted file mode 100644 index 3fd7221..0000000 --- a/man/timestep_embedding_module.Rd +++ /dev/null @@ -1,16 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{timestep_embedding_module} -\alias{timestep_embedding_module} -\title{Timestep embedding MLP} -\usage{ -timestep_embedding_module( - in_channels, - time_embed_dim, - act_fn = "silu", - out_dim = NULL -) -} -\description{ -Timestep embedding MLP -} -\keyword{internal} diff --git a/man/timesteps_module.Rd b/man/timesteps_module.Rd deleted file mode 100644 index fdd9170..0000000 --- a/man/timesteps_module.Rd +++ /dev/null @@ -1,16 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{timesteps_module} -\alias{timesteps_module} -\title{Timesteps module} -\usage{ -timesteps_module( - num_channels, - flip_sin_to_cos = TRUE, - downscale_freq_shift = 0, - scale = 1L -) -} -\description{ -Timesteps module -} -\keyword{internal} diff --git a/man/tokenize_gemma3.Rd b/man/tokenize_gemma3.Rd index 98c5a34..c939dd4 100644 --- a/man/tokenize_gemma3.Rd +++ b/man/tokenize_gemma3.Rd @@ -3,13 +3,8 @@ \alias{tokenize_gemma3} \title{Tokenize text for Gemma3} \usage{ -tokenize_gemma3( - tokenizer, - text, - max_length = 1024L, - padding = "max_length", - return_tensors = "pt" -) +tokenize_gemma3(tokenizer, text, max_length = 1024L, padding = "max_length", + return_tensors = "pt") } \arguments{ \item{tokenizer}{Gemma3 tokenizer object.} diff --git a/man/txt2img_sd21.Rd b/man/txt2img_sd21.Rd index 55d3332..90362f1 100644 --- a/man/txt2img_sd21.Rd +++ b/man/txt2img_sd21.Rd @@ -3,28 +3,13 @@ \alias{txt2img_sd21} \title{Generate an image from a text prompt using a diffusion pipeline} \usage{ -txt2img_sd21( - prompt, - negative_prompt = NULL, - img_dim = 768, - pipeline = NULL, - devices = "auto", - unet_dtype_str = NULL, - download_models = FALSE, - scheduler = "ddim", - timesteps = NULL, - initial_latents = NULL, - num_inference_steps = 50, - guidance_scale = 7.5, - seed = NULL, - save_file = TRUE, - filename = NULL, - metadata_path = NULL, - use_native_decoder = FALSE, - use_native_text_encoder = FALSE, - use_native_unet = FALSE, - ... -) +txt2img_sd21(prompt, negative_prompt = NULL, img_dim = 768, pipeline = NULL, + devices = "auto", unet_dtype_str = NULL, download_models = FALSE, + scheduler = "ddim", timesteps = NULL, initial_latents = NULL, + num_inference_steps = 50, guidance_scale = 7.5, seed = NULL, + save_file = TRUE, filename = NULL, metadata_path = NULL, + use_native_decoder = FALSE, use_native_text_encoder = FALSE, + use_native_unet = FALSE, ...) } \arguments{ \item{prompt}{A character string prompt describing the image to generate.} diff --git a/man/txt2img_sdxl.Rd b/man/txt2img_sdxl.Rd index c514f10..bc133d6 100644 --- a/man/txt2img_sdxl.Rd +++ b/man/txt2img_sdxl.Rd @@ -3,30 +3,14 @@ \alias{txt2img_sdxl} \title{Generate an image from a text prompt using SDXL} \usage{ -txt2img_sdxl( - prompt, - negative_prompt = NULL, - img_dim = 1024, - pipeline = NULL, - devices = "auto", - memory_profile = NULL, - unet_dtype_str = NULL, - download_models = FALSE, - scheduler = "ddim", - timesteps = NULL, - initial_latents = NULL, - num_inference_steps = 30, - guidance_scale = 7.5, - seed = NULL, - save_file = TRUE, - filename = NULL, - metadata_path = NULL, - use_native_decoder = FALSE, - use_native_text_encoder = FALSE, - use_native_unet = FALSE, - verbose = TRUE, - ... -) +txt2img_sdxl(prompt, negative_prompt = NULL, img_dim = 1024, pipeline = NULL, + devices = "auto", memory_profile = NULL, unet_dtype_str = NULL, + download_models = FALSE, scheduler = "ddim", timesteps = NULL, + initial_latents = NULL, num_inference_steps = 30, + guidance_scale = 7.5, seed = NULL, save_file = TRUE, + filename = NULL, metadata_path = NULL, use_native_decoder = FALSE, + use_native_text_encoder = FALSE, use_native_unet = FALSE, + verbose = TRUE, ...) } \arguments{ \item{prompt}{A character string prompt describing the image to generate.} diff --git a/man/txt2vid_ltx2.Rd b/man/txt2vid_ltx2.Rd deleted file mode 100644 index 83c0387..0000000 --- a/man/txt2vid_ltx2.Rd +++ /dev/null @@ -1,96 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{txt2vid_ltx2} -\alias{txt2vid_ltx2} -\title{Generate Video from Text Prompt using LTX-2} -\usage{ -txt2vid_ltx2( - prompt, - negative_prompt = NULL, - width = 768L, - height = 512L, - num_frames = 121L, - fps = 24, - num_inference_steps = 8L, - guidance_scale = 4, - memory_profile = "auto", - text_backend = "gemma3", - text_model_path = NULL, - text_api_url = NULL, - vae = NULL, - dit = NULL, - connectors = NULL, - seed = NULL, - output_file = NULL, - output_format = "mp4", - return_latents = FALSE, - verbose = TRUE -) -} -\arguments{ -\item{prompt}{Character. Text prompt describing the video to generate.} - -\item{negative_prompt}{Character. Optional negative prompt.} - -\item{width}{Integer. Video width in pixels (default 768).} - -\item{height}{Integer. Video height in pixels (default 512).} - -\item{num_frames}{Integer. Number of frames to generate (default 121).} - -\item{fps}{Numeric. Frames per second (default 24).} - -\item{num_inference_steps}{Integer. Number of denoising steps (default 8 for distilled).} - -\item{guidance_scale}{Numeric. CFG scale (default 4.0).} - -\item{memory_profile}{Character or list. Memory profile: "auto" for auto-detection, -or a profile from `ltx2_memory_profile()`.} - -\item{text_backend}{Character. Text encoding backend: "gemma3" (native), "api", "precomputed", or "random".} - -\item{text_model_path}{Character. Path to Gemma3 model (for "gemma3" backend). Supports glob patterns.} - -\item{text_api_url}{Character. URL for text encoding API (if backend = "api").} - -\item{vae}{Optional. Pre-loaded VAE module.} - -\item{dit}{Optional. Pre-loaded DiT transformer module.} - -\item{connectors}{Optional. Pre-loaded text connectors module.} - -\item{seed}{Integer. Random seed for reproducibility.} - -\item{output_file}{Character. Path to save output video (NULL for no save).} - -\item{output_format}{Character. Output format: "mp4", "gif", or "frames".} - -\item{return_latents}{Logical. If TRUE, also return final latents.} - -\item{verbose}{Logical. Print progress messages.} -} -\value{ -A list with: - - `video`: Array of video frames [frames, height, width, channels] - - `latents`: (if return_latents=TRUE) Final latent tensor - - `metadata`: Generation metadata -} -\description{ -Generates video using the LTX-2 diffusion transformer model. -} -\examples{ -\dontrun{ -# Basic usage -result <- txt2vid_ltx2("A cat walking on a beach at sunset") - -# With specific settings -result <- txt2vid_ltx2( - prompt = "A timelapse of clouds moving over mountains", - width = 512, - height = 320, - num_frames = 61, - num_inference_steps = 8, - seed = 42, - output_file = "clouds.mp4" -) -} -} diff --git a/man/unet_native.Rd b/man/unet_native.Rd index df9d8a0..d5f70f9 100644 --- a/man/unet_native.Rd +++ b/man/unet_native.Rd @@ -3,14 +3,10 @@ \alias{unet_native} \title{Native UNet for Stable Diffusion} \usage{ -unet_native( - in_channels = 4L, - out_channels = 4L, - block_out_channels = c(320L, 640L, 1280L, 1280L), - layers_per_block = 2L, - cross_attention_dim = 1024L, - attention_head_dim = 64L -) +unet_native(in_channels = 4L, out_channels = 4L, + block_out_channels = c(320L, 640L, 1280L, 1280L), + layers_per_block = 2L, cross_attention_dim = 1024L, + attention_head_dim = 64L) } \arguments{ \item{in_channels}{Input channels (default 4 for latent space)} diff --git a/man/unet_sdxl_native.Rd b/man/unet_sdxl_native.Rd index 2f01ca8..4e6668a 100644 --- a/man/unet_sdxl_native.Rd +++ b/man/unet_sdxl_native.Rd @@ -3,17 +3,12 @@ \alias{unet_sdxl_native} \title{Native SDXL UNet} \usage{ -unet_sdxl_native( - in_channels = 4L, - out_channels = 4L, - block_out_channels = c(320L, 640L, 1280L), - layers_per_block = 2L, - transformer_layers_per_block = c(0L, 2L, 10L), - cross_attention_dim = 2048L, - attention_head_dim = 64L, - addition_embed_dim = 1280L, - addition_time_embed_dim = 256L -) +unet_sdxl_native(in_channels = 4L, out_channels = 4L, + block_out_channels = c(320L, 640L, 1280L), + layers_per_block = 2L, + transformer_layers_per_block = c(0L, 2L, 10L), + cross_attention_dim = 2048L, attention_head_dim = 64L, + addition_embed_dim = 1280L, addition_time_embed_dim = 256L) } \arguments{ \item{in_channels}{Input channels (default 4 for latent space)} diff --git a/man/unpack_video_latents.Rd b/man/unpack_video_latents.Rd deleted file mode 100644 index 526a14f..0000000 --- a/man/unpack_video_latents.Rd +++ /dev/null @@ -1,33 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{unpack_video_latents} -\alias{unpack_video_latents} -\title{Unpack Video Latents from DiT} -\usage{ -unpack_video_latents( - latents, - num_frames, - height, - width, - patch_size = 1L, - patch_size_t = 1L -) -} -\arguments{ -\item{latents}{Tensor of shape \[B, num_patches, D\].} - -\item{num_frames}{Integer. Target number of latent frames.} - -\item{height}{Integer. Target latent height.} - -\item{width}{Integer. Target latent width.} - -\item{patch_size}{Integer. Spatial patch size (default 1).} - -\item{patch_size_t}{Integer. Temporal patch size (default 1).} -} -\value{ -Unpacked tensor of shape \[B, C, F, H, W\]. -} -\description{ -Transforms 3D sequence \[B, num_patches, D\] back to 5D video latents \[B, C, F, H, W\]. -} diff --git a/man/vae_ltx2.Rd b/man/vae_ltx2.Rd deleted file mode 100644 index 1436d72..0000000 --- a/man/vae_ltx2.Rd +++ /dev/null @@ -1,8 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{vae_ltx2} -\alias{vae_ltx2} -\title{LTX2 Video VAE} -\description{ -3D causal VAE for video encoding/decoding as used in LTX-2. -Supports GPU-poor tiling for memory-efficient processing. -} diff --git a/man/vae_ltx2_modules.Rd b/man/vae_ltx2_modules.Rd deleted file mode 100644 index 0322f37..0000000 --- a/man/vae_ltx2_modules.Rd +++ /dev/null @@ -1,8 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{vae_ltx2_modules} -\alias{vae_ltx2_modules} -\title{LTX2 Video VAE Module Building Blocks} -\description{ -Low-level nn_module components for the LTX2 3D causal VAE. -Used by vae_ltx2.R encoder/decoder. -} diff --git a/man/validate_resolution.Rd b/man/validate_resolution.Rd deleted file mode 100644 index 14a2e77..0000000 --- a/man/validate_resolution.Rd +++ /dev/null @@ -1,28 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{validate_resolution} -\alias{validate_resolution} -\title{Validate Resolution Against Profile} -\usage{ -validate_resolution(height, width, num_frames, profile) -} -\arguments{ -\item{height}{Integer. Requested height.} - -\item{width}{Integer. Requested width.} - -\item{num_frames}{Integer. Requested number of frames.} - -\item{profile}{Memory profile from `ltx2_memory_profile()`.} -} -\value{ -List with adjusted height, width, num_frames and warning if adjusted. -} -\description{ -Checks if requested resolution fits within memory profile limits. -} -\examples{ -\dontrun{ -profile <- ltx2_memory_profile(vram_gb = 8) -validated <- validate_resolution(720, 1280, 60, profile) -} -} diff --git a/man/vram.Rd b/man/vram.Rd new file mode 100644 index 0000000..6165a84 --- /dev/null +++ b/man/vram.Rd @@ -0,0 +1,8 @@ +% tinyrox says don't edit this manually, but it can't stop you! +\name{vram} +\alias{vram} +\title{VRAM Detection and Management Utilities} +\description{ +Device detection, VRAM reporting, and module offloading helpers shared +by the image and video pipelines. +} From d8f25df1c3ebb3d770167c1cf84a19f3ef791a1f Mon Sep 17 00:00:00 2001 From: TroyHernandez Date: Fri, 3 Jul 2026 10:52:35 -0500 Subject: [PATCH 02/83] License hygiene: attribution, ignores, CLAUDE.md provenance rewrite - DESCRIPTION: cph entries for cornball.ai, HuggingFace (diffusers ports), and Lightricks (constants/format facts); ORCID for maintainer - inst/COPYRIGHTS: enumerates derivation sources and weight licensing - ref/ and tasks/ gitignored + Rbuildignored; upstream reference checkouts live in ref/upstream/ and are never committed - CLAUDE.md: LTX section rewritten for the 2.3 clean-room rewrite; stale 2.0 API docs removed, generic implementation facts kept - Drop tracked pyrotechnics conversions in ref/ (superseded by the real diffusers checkout in ref/upstream/) --- .Rbuildignore | 2 + .gitignore | 4 + CLAUDE.md | 118 +- DESCRIPTION | 10 +- inst/COPYRIGHTS | 36 + ref/pipeline_ltx2.R | 1147 -------------------- ref/scheduling_flow_match_euler_discrete.R | 610 ----------- 7 files changed, 79 insertions(+), 1848 deletions(-) create mode 100644 inst/COPYRIGHTS delete mode 100644 ref/pipeline_ltx2.R delete mode 100644 ref/scheduling_flow_match_euler_discrete.R diff --git a/.Rbuildignore b/.Rbuildignore index a83a93a..1d0835b 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -10,3 +10,5 @@ ^fyi\.md$ ^man-md$ ^inst/validation/\.venv$ +^ref$ +^tasks$ diff --git a/.gitignore b/.gitignore index 7397cae..9eac07f 100644 --- a/.gitignore +++ b/.gitignore @@ -48,3 +48,7 @@ po/*~ # RStudio Connect folder rsconnect/ /devman-md/ + +# Upstream reference checkouts and working notes (never committed) +ref/ +tasks/ diff --git a/CLAUDE.md b/CLAUDE.md index e822882..bfc0ea6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -51,7 +51,7 @@ The `txt2img_*` and `img2img` functions default to `devices = "auto"`, which: ## GPU-Poor Execution Plan (TODO) -Profile-based memory optimization for constrained GPUs. Inspired by mmgp/Wan2GP approach. +Profile-based memory optimization for constrained GPUs. ### API @@ -188,7 +188,7 @@ vram_report <- function(label) { } ``` -### Future Optimizations (from mmgp) +### Future Optimizations Not implemented, but worth considering: @@ -320,66 +320,32 @@ See cornyverse CLAUDE.md for safetensors package setup (use cornball-ai fork unt - [ ] Add SD3 model support - [ ] ControlNet integration -### LTX-2 Video Generation (In Progress) -- [x] FlowMatch scheduler (validated against Python) -- [x] RoPE positional embeddings (validated against Python) -- [x] LTX2 Video VAE (3D causal convolutions) - see learnings below -- [x] DiT transformer (audio-video) - see learnings below -- [x] Text encoder integration (connectors + flexible backends) -- [x] GPU-poor optimizations (wan2GP style memory profiles) -- [x] Pipeline integration (txt2vid_ltx2) -- [x] Video output utilities (save_video) -- [x] Weight loading from HuggingFace safetensors +### LTX-2.3 Video Generation (clean-room rewrite in progress) -#### LTX-2 Weight Loading +The original LTX-2.0 port was removed and is being replaced by a ground-up +LTX-2.3 implementation written from the Apache-2.0 diffusers reference only +(branch `feat/ltx23`; working checklist in `tasks/todo.md`). -Load LTX-2 model weights from HuggingFace safetensors: +**Reference checkouts** (in `ref/upstream/`, gitignored): +- `ref/upstream/diffusers` — the code reference (Apache-2.0) +- `ref/upstream/ltx2-official` — Lightricks monorepo (LTX-2 Community License; + used for numeric constants and file-format facts only, never as a code base) -```r -# Load VAE (2.44 GB) -vae <- load_ltx2_vae( - weights_path = "~/.cache/huggingface/hub/models--Lightricks--LTX-2/vae/diffusion_pytorch_model.safetensors", - config_path = "~/.cache/huggingface/hub/models--Lightricks--LTX-2/vae/config.json", - device = "cuda", - dtype = "float16" -) - -# Load transformer (37.8 GB, sharded across 8 files) -transformer <- load_ltx2_transformer( - weights_dir = "~/.cache/huggingface/hub/models--Lightricks--LTX-2/transformer", - device = "cpu", # Start on CPU, offload to GPU layer-by-layer - dtype = "float16" -) - -# Load text connectors (2.86 GB) -connectors <- load_ltx2_connectors( - weights_path = "~/.cache/huggingface/hub/models--Lightricks--LTX-2/connectors/diffusion_pytorch_model.safetensors", - config_path = "~/.cache/huggingface/hub/models--Lightricks--LTX-2/connectors/config.json" -) -``` - -**Model sizes:** -| Component | Size | Notes | -|-----------|------|-------| -| VAE | 2.44 GB | Single safetensors file | -| Transformer | 37.8 GB | Sharded across 8 files | -| Connectors | 2.86 GB | Single safetensors file | -| Total (19B) | 43.3 GB | Full precision | -| Total FP8 | 27.1 GB | Quantized | +**Rules:** +- Never derive code from Wan2GP (WanGP Community License 2.0 restricts + commercial embedding). Do not open Wan2GP source or conversions of it. +- Model weights are downloaded by the user from `Lightricks/LTX-2.3` under the + LTX-2 Community License and never redistributed with the package. +- Kept support files with independent provenance: `gemma3_text_encoder.R` + (from HF transformers), `tokenizer_bpe.R` (original), `flowmatch_scheduler.R` + (from diffusers). -#### LTX2 VAE Implementation Learnings +#### Implementation facts carried over from the 2.0 port **Temporal dimension constraint for causal downsampling:** -For LTX2's causal 3D convolutions with stride S downsampling, the temporal dimension T must satisfy: -``` -(T + S - 1) % S == 0 -``` -This means T % S == 1, so **T must be odd for stride=2**. - -Example valid sequences for 2 spatiotemporal downsampling stages: -- T=5 → (5+1)/2=3 → (3+1)/2=2 ✓ -- T=9 → (9+1)/2=5 → (5+1)/2=3 ✓ -- T=4 → (4+1)/2=2.5 ✗ (fails unflatten) +For causal 3D convolutions with stride S downsampling, the temporal dimension T +must satisfy `(T + S - 1) %% S == 0`, i.e. T %% S == 1, so **T must be odd for +stride=2** (T=4 fails the unflatten; T=5 → 3 → 2 works). **R torch `unflatten` is 1-indexed:** ```r @@ -387,21 +353,9 @@ Example valid sequences for 2 spatiotemporal downsampling stages: # R: x$unflatten(3, c(-1, stride)) # dim 3 is 1-indexed ``` -**LTX2 decoder channel flow:** -In LTX2 up blocks, `in_channels == out_channels` always. The upsampler handles channel reduction via pixel shuffle. Test inputs must match this pattern. - -#### LTX2 DiT Transformer Learnings - -**cross_attention_dim must equal inner_dim:** -In the LTX2 transformer, `caption_projection` projects text embeddings from `caption_channels` to `inner_dim`. The transformer blocks then expect `encoder_hidden_states` to have dimension `cross_attention_dim`. These must be equal: -```r -# In model config: -inner_dim = num_attention_heads * attention_head_dim # e.g., 32 * 128 = 4096 -cross_attention_dim = 4096 # Must equal inner_dim! -``` - **R torch lacks nnf_scaled_dot_product_attention:** -Manual scaled dot-product attention is required: +Manual scaled dot-product attention is required (and materializes the full +[B, H, S, S] attention matrix — chunk queries at high resolution): ```r scale <- 1.0 / sqrt(head_dim) attn_weights <- torch::torch_matmul(query, key$transpose(-2L, -1L)) * scale @@ -410,32 +364,18 @@ attn_weights <- torch::nnf_softmax(attn_weights, dim = -1L) hidden_states <- torch::torch_matmul(attn_weights, value) ``` -**Avoid function name collisions across files:** -The `apply_interleaved_rotary_emb` function in `rope.R` expects `freqs$cos_freqs`, while a similar function in `dit_ltx2_modules.R` expects `freqs[[1]]`. Name collision caused segfaults - renamed to `apply_interleaved_rotary_emb_list` in dit module. - -#### LTX2 Text Encoder Learnings - -**Architecture: Gemma3 + Connectors:** -LTX-2 uses Gemma3 as the text encoder, with separate connector transformers for video and audio streams: -``` -Gemma3 → pack_text_embeds → text_proj_in → video_connector → video_embeds - → audio_connector → audio_embeds -``` - **Attention mask broadcasting:** -When using 2D attention masks [B, S], they must be expanded to [B, 1, 1, S] for broadcasting with attention weights [B, H, S, S]: +2D attention masks [B, S] must be expanded to [B, 1, 1, S] to broadcast with +attention weights [B, H, S, S]: ```r if (attention_mask$ndim == 2L) { attention_mask <- attention_mask$unsqueeze(2L)$unsqueeze(2L) } ``` -**Flexible text encoding backends:** -The `encode_text_ltx2()` function supports multiple backends: -- `"gemma3"`: Native R torch Gemma3 encoder (no Python dependency) -- `"precomputed"`: Load from file (cached embeddings) -- `"api"`: HTTP request to external service (Gemma container) -- `"random"`: Random embeddings for testing +**Avoid function name collisions across files:** +Two same-named helpers with different calling conventions in different files +once caused segfaults. One name, one signature, package-wide. #### Native Gemma3 Text Encoder diff --git a/DESCRIPTION b/DESCRIPTION index 081925d..eb52b11 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,8 +1,14 @@ Package: diffuseR Title: Functional Interface to Diffusion Models in R Version: 0.0.2 -Authors@R: - person("Troy", "Hernandez", email = "troy@cornball.ai", role = c("aut", "cre")) +Authors@R: c( + person("Troy", "Hernandez", email = "troy@cornball.ai", role = c("aut", "cre"), + comment = c(ORCID = "0009-0005-4248-604X")), + person("cornball.ai", role = "cph"), + person("The HuggingFace Team", role = "cph", + comment = "portions ported from the diffusers library (Apache-2.0); see inst/COPYRIGHTS"), + person("Lightricks Ltd.", role = "cph", + comment = "LTX checkpoint layout and pipeline constants; see inst/COPYRIGHTS")) Description: A native R implementation of diffusion models providing a functional interface to state-of-the-art generative AI. Inspired by Hugging Face's Python 'diffusers' library, 'diffuseR' allows users to generate and manipulate images diff --git a/inst/COPYRIGHTS b/inst/COPYRIGHTS new file mode 100644 index 0000000..4e782a2 --- /dev/null +++ b/inst/COPYRIGHTS @@ -0,0 +1,36 @@ +COPYRIGHT AND ATTRIBUTION NOTES FOR diffuseR + +diffuseR is licensed under the Apache License (>= 2). Portions of the +package are R ports of, or derived from, third-party work: + +1. HuggingFace diffusers (Apache License 2.0) + Copyright The HuggingFace Team. + https://github.com/huggingface/diffusers + The LTX video pipeline components (transformer, video/audio VAE, + text-embedding connectors, vocoder, latent upsampler, schedulers, and + pipeline logic including the AdaIN latent filter and tone mapping) are + R ports written from the diffusers reference implementation. + The FlowMatch Euler scheduler is ported from + diffusers/schedulers/scheduling_flow_match_euler_discrete.py. + +2. HuggingFace transformers (Apache License 2.0) + Copyright The HuggingFace Team. + https://github.com/huggingface/transformers + The Gemma3 text encoder (R/gemma3_text_encoder.R) is an R port of the + Gemma3 implementation in transformers. + +3. Lightricks Ltd. + https://github.com/Lightricks/LTX-2 + Numeric constants (e.g. distilled sigma schedules), the single-file + checkpoint key layout, and pipeline parameter defaults are facts taken + from the official LTX release. No code from the Lightricks repository + (LTX-2 Community License) is included in this package. + + LTX model weights are downloaded by the user directly from Lightricks' + HuggingFace repositories under the LTX-2 Community License and are not + part of, nor redistributed with, this package. + +4. Stability AI / Stable Diffusion (SD 2.1, SDXL) + Model weights are downloaded by the user under their respective + licenses (CreativeML Open RAIL++-M and similar) and are not part of, + nor redistributed with, this package. diff --git a/ref/pipeline_ltx2.R b/ref/pipeline_ltx2.R deleted file mode 100644 index eeb4d30..0000000 --- a/ref/pipeline_ltx2.R +++ /dev/null @@ -1,1147 +0,0 @@ -# Converted from PyTorch by pyrotechnics -# Review: indexing (0->1 based), integer literals (add L), -# and block structure (braces may need adjustment) - -# Copyright 2025 Lightricks and The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# import copy -# import inspect -# from typing import Any, Callable, Dict, List, Optional, Union - -# import numpy as np -# import torch -# from transformers import Gemma3ForConditionalGeneration, GemmaTokenizer, GemmaTokenizerFast - -# from ...callbacks import MultiPipelineCallbacks, PipelineCallback -# from ...loaders import FromSingleFileMixin, LTX2LoraLoaderMixin -# from ...models.autoencoders import AutoencoderKLLTX2Audio, AutoencoderKLLTX2Video -# from ...models.transformers import LTX2VideoTransformer3DModel -# from ...schedulers import FlowMatchEulerDiscreteScheduler -# from ...utils import is_torch_xla_available, logging, replace_example_docstring -# from ...utils.torch_utils import randn_tensor -# from ...video_processor import VideoProcessor -# from ..pipeline_utils import DiffusionPipeline -# from .connectors import LTX2TextConnectors -# from .pipeline_output import LTX2PipelineOutput -# from .vocoder import LTX2Vocoder - - -if (is_torch_xla_available()) { -# import torch_xla.core.xla_model as xm - - XLA_AVAILABLE <- TRUE -} else { - XLA_AVAILABLE <- FALSE - -logger <- logging.get_logger(__name__) # pylint: disable=invalid-name - -EXAMPLE_DOC_STRING <- """ - Examples: - ```py - >>> import torch - >>> from diffusers import LTX2Pipeline - >>> from diffusers.pipelines.ltx2.export_utils import encode_video - - >>> pipe <- LTX2Pipeline.from_pretrained("Lightricks/LTX-2", torch_dtype=torch.bfloat16) - >>> pipe.enable_model_cpu_offload() - - >>> prompt <- "A woman with long brown hair && light skin smiles at another woman with long blonde hair. The woman with brown hair wears a black jacket && has a small, barely noticeable mole on her right cheek. The camera angle is a close-up, focused on the woman with brown hair's face. The lighting is warm && natural, likely from the setting sun, casting a soft glow on the scene. The scene appears to be real-life footage" - >>> negative_prompt <- "worst quality, inconsistent motion, blurry, jittery, distorted" - - >>> frame_rate <- 24.0 - >>> video, audio <- pipe( - ... prompt <- prompt, - ... negative_prompt <- negative_prompt, - ... width <- 768, - ... height <- 512, - ... num_frames <- 121, - ... frame_rate <- frame_rate, - ... num_inference_steps <- 40, - ... guidance_scale <- 4.0, - ... output_type <- "np", - ... return_dict <- FALSE, - ... ) - >>> video <- (video * 255).round().astype("uint8") - >>> video <- torch_from_numpy(video) - - >>> encode_video( - ... video[0], - ... fps <- frame_rate, - ... audio <- audio[0].float()$cpu(), - ... audio_sample_rate <- pipe.vocoder.config.output_sampling_rate, # should be 24000 - ... output_path <- "video.mp4", - ... ) - ``` -""" - - -# Copied from diffusers.pipelines.flux.pipeline_flux.calculate_shift -def calculate_shift( - image_seq_len, - base_seq_len <- 256, - max_seq_len <- 4096, - base_shift <- 0.5, - max_shift <- 1.15, -): - m <- (max_shift - base_shift) / (max_seq_len - base_seq_len) - b <- base_shift - m * base_seq_len - mu <- image_seq_len * m + b - return(mu) - - -# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps -def retrieve_timesteps( - scheduler, - num_inference_steps <- NULL, - device <- NULL, - timesteps <- NULL, - sigmas <- NULL, - ^kwargs, -): - r""" - Calls the scheduler's `set_timesteps` method && retrieves timesteps from the scheduler after the call. Handles - custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. - - Args: - scheduler (`SchedulerMixin`): - The scheduler to get timesteps from. - num_inference_steps (`int`): - The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps` - must be `NULL`. - device (`str` || `torch$device`, *optional*): - The device to which the timesteps should be moved to. If `NULL`, the timesteps are ! moved. - timesteps (`List[int]`, *optional*): - Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, - `num_inference_steps` && `sigmas` must be `NULL`. - sigmas (`List[float]`, *optional*): - Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, - `num_inference_steps` && `timesteps` must be `NULL`. - - Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler && the - second element is the number of inference steps. - """ - if (!is.null(timesteps) && !is.null(sigmas)) { - raise ValueError("Only one of `timesteps` || `sigmas` can be passed. Please choose one to set custom values") - if (!is.null(timesteps)) { - accepts_timesteps <- "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) - if (! accepts_timesteps) { - raise ValueError( - sprintf("The current scheduler class {scheduler.__class__}'s `set_timesteps` does ! support custom") - sprintf(" timestep schedules. Please check whether you are using the correct scheduler.") - ) - scheduler.set_timesteps(timesteps=timesteps, device=device, ^kwargs) - timesteps <- scheduler.timesteps - num_inference_steps <- length(timesteps) - } else if (!is.null(sigmas)) { - accept_sigmas <- "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) - if (! accept_sigmas) { - raise ValueError( - sprintf("The current scheduler class {scheduler.__class__}'s `set_timesteps` does ! support custom") - sprintf(" sigmas schedules. Please check whether you are using the correct scheduler.") - ) - scheduler.set_timesteps(sigmas=sigmas, device=device, ^kwargs) - timesteps <- scheduler.timesteps - num_inference_steps <- length(timesteps) - } else { - scheduler.set_timesteps(num_inference_steps, device=device, ^kwargs) - timesteps <- scheduler.timesteps - return(list(timesteps, num_inference_steps)) - - -# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.rescale_noise_cfg -def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0): - r""" - Rescales `noise_cfg` tensor based on `guidance_rescale` to improve image quality && fix overexposure. Based on - Section 3.4 from [Common Diffusion Noise Schedules && Sample Steps are - Flawed](https:%/%huggingface.co/papers/2305.08891). - - Args: - noise_cfg (`torch.Tensor`): - The predicted noise tensor for the guided diffusion process. - noise_pred_text (`torch.Tensor`): - The predicted noise tensor for the text-guided diffusion process. - guidance_rescale (`float`, *optional*, defaults to 0.0): - A rescale factor applied to the noise predictions. - - Returns: - noise_cfg (`torch.Tensor`): The rescaled noise prediction tensor. - """ - std_text <- noise_pred_text$std(dim=list(range(1, noise_pred_text.ndim)), keepdim=TRUE) - std_cfg <- noise_cfg$std(dim=list(range(1, noise_cfg.ndim)), keepdim=TRUE) - # rescale the results from guidance (fixes overexposure) - noise_pred_rescaled <- noise_cfg * (std_text / std_cfg) - # mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images - noise_cfg <- guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg - return(noise_cfg) - - -class LTX2Pipeline(DiffusionPipeline, FromSingleFileMixin, LTX2LoraLoaderMixin): - r""" - Pipeline for text-to-video generation. - - Reference: https:%/%github.com/Lightricks/LTX-Video - - Args: - transformer ([`LTXVideoTransformer3DModel`]): - Conditional Transformer architecture to denoise the encoded video latents. - scheduler ([`FlowMatchEulerDiscreteScheduler`]): - A scheduler to be used in combination with `transformer` to denoise the encoded image latents. - vae ([`AutoencoderKLLTXVideo`]): - Variational Auto-Encoder (VAE) Model to encode && decode images to && from latent representations. - text_encoder ([`T5EncoderModel`]): - [T5](https:%/%huggingface.co/docs/transformers/en/model_doc/t5#transformers.T5EncoderModel), specifically - the [google/t5-v1_1-xxl](https:%/%huggingface.co/google/t5-v1_1-xxl) variant. - tokenizer (`CLIPTokenizer`): - Tokenizer of class - [CLIPTokenizer](https:%/%huggingface.co/docs/transformers/en/model_doc/clip#transformers.CLIPTokenizer). - tokenizer (`T5TokenizerFast`): - Second Tokenizer of class - [T5TokenizerFast](https:%/%huggingface.co/docs/transformers/en/model_doc/t5#transformers.T5TokenizerFast). - connectors ([`LTX2TextConnectors`]): - Text connector stack used to adapt text encoder hidden states for the video && audio branches. - """ - - model_cpu_offload_seq <- "text_encoder->connectors->transformer->vae->audio_vae->vocoder" - _optional_components <- [] - _callback_tensor_inputs <- ["latents", "prompt_embeds", "negative_prompt_embeds"] - - def __init__( - self, - scheduler: FlowMatchEulerDiscreteScheduler, - vae: AutoencoderKLLTX2Video, - audio_vae: AutoencoderKLLTX2Audio, - text_encoder: Gemma3ForConditionalGeneration, - tokenizer: Union[GemmaTokenizer, GemmaTokenizerFast], - connectors: LTX2TextConnectors, - transformer: LTX2VideoTransformer3DModel, - vocoder: LTX2Vocoder, - ): - super().__init__() - - self$register_modules( - vae <- vae, - audio_vae <- audio_vae, - text_encoder <- text_encoder, - tokenizer <- tokenizer, - connectors <- connectors, - transformer <- transformer, - vocoder <- vocoder, - scheduler <- scheduler, - ) - - self$vae_spatial_compression_ratio <- ( - self$vae.spatial_compression_ratio if getattr(self, "vae", NULL) !is.null else 32 - ) - self$vae_temporal_compression_ratio <- ( - self$vae.temporal_compression_ratio if getattr(self, "vae", NULL) !is.null else 8 - ) - # TODO: check whether the MEL compression ratio logic here is corrct - self$audio_vae_mel_compression_ratio <- ( - self$audio_vae.mel_compression_ratio if getattr(self, "audio_vae", NULL) !is.null else 4 - ) - self$audio_vae_temporal_compression_ratio <- ( - self$audio_vae.temporal_compression_ratio if getattr(self, "audio_vae", NULL) !is.null else 4 - ) - self$transformer_spatial_patch_size <- ( - self$transformer.config.patch_size if getattr(self, "transformer", NULL) !is.null else 1 - ) - self$transformer_temporal_patch_size <- ( - self$transformer.config.patch_size_t if getattr(self, "transformer") !is.null else 1 - ) - - self$audio_sampling_rate <- ( - self$audio_vae.config.sample_rate if getattr(self, "audio_vae", NULL) !is.null else 16000 - ) - self$audio_hop_length <- ( - self$audio_vae.config.mel_hop_length if getattr(self, "audio_vae", NULL) !is.null else 160 - ) - - self$video_processor <- VideoProcessor(vae_scale_factor=self$vae_spatial_compression_ratio) - self$tokenizer_max_length <- ( - self$tokenizer.model_max_length if getattr(self, "tokenizer", NULL) !is.null else 1024 - ) - - @staticmethod - def _pack_text_embeds( - text_hidden_states: torch.Tensor, - sequence_lengths: torch.Tensor, - device: Union[str, torch$device], - padding_side <- "left", - scale_factor <- 8, - eps <- 1e-6, - ) -> torch.Tensor: - """ - Packs && normalizes text encoder hidden states, respecting padding. Normalization is performed per-batch && - per-layer in a masked fashion (only over non-padded positions). - - Args: - text_hidden_states (`torch.Tensor` of shape `(batch_size, seq_len, hidden_dim, num_layers)`): - Per-layer hidden_states from a text encoder (e.g. `Gemma3ForConditionalGeneration`). - sequence_lengths (`torch.Tensor of shape `(batch_size,)`): - The number of valid (non-padded) tokens for each batch instance. - device: (`str` || `torch$device`, *optional*): - torch device to place the resulting embeddings on - padding_side: (`str`, *optional*, defaults to `"left"`): - Whether the text tokenizer performs padding on the `"left"` || `"right"`. - scale_factor (`int`, *optional*, defaults to `8`): - Scaling factor to multiply the normalized hidden states by. - eps (`float`, *optional*, defaults to `1e-6`): - A small positive value for numerical stability when performing normalization. - - Returns: - `torch.Tensor` of shape `(batch_size, seq_len, hidden_dim * num_layers)`: - Normed && flattened text encoder hidden states. - """ - batch_size, seq_len, hidden_dim, num_layers <- text_hidden_states.shape - original_dtype <- text_hidden_states$dtype - - # Create padding mask - token_indices <- torch_arange(seq_len, device=device)$unsqueeze(0) - if (padding_side == "right") { - # For right padding, valid tokens are from 0 to sequence_length-1 - mask <- token_indices < sequence_lengths[:, NULL] # [batch_size, seq_len] - } else if (padding_side == "left") { - # For left padding, valid tokens are from (T - sequence_length) to T-1 - start_indices <- seq_len - sequence_lengths[:, NULL] # [batch_size, 1] - mask <- token_indices >= start_indices # [B, T] - } else { - raise ValueError(sprintf("padding_side must be 'left' || 'right', got {padding_side}")) - mask <- mask[:, :, NULL, NULL] # [batch_size, seq_len] --> [batch_size, seq_len, 1, 1] - - # Compute masked mean over non-padding positions of shape (batch_size, 1, 1, seq_len) - masked_text_hidden_states <- text_hidden_states$masked_fill(~mask, 0.0) - num_valid_positions <- (sequence_lengths * hidden_dim)$view(batch_size, 1, 1, 1) - masked_mean <- masked_text_hidden_states$sum(dim=(1, 2), keepdim=TRUE) / (num_valid_positions + eps) - - # Compute min/max over non-padding positions of shape (batch_size, 1, 1 seq_len) - x_min <- text_hidden_states$masked_fill(~mask, float("inf")).amin(dim=(1, 2), keepdim=TRUE) - x_max <- text_hidden_states$masked_fill(~mask, float("-inf")).amax(dim=(1, 2), keepdim=TRUE) - - # Normalization - normalized_hidden_states <- (text_hidden_states - masked_mean) / (x_max - x_min + eps) - normalized_hidden_states <- normalized_hidden_states * scale_factor - - # Pack the hidden states to a 3D tensor (batch_size, seq_len, hidden_dim * num_layers) - normalized_hidden_states <- normalized_hidden_states$flatten(2) - mask_flat <- mask$squeeze(-1)$expand(-1, -1, hidden_dim * num_layers) - normalized_hidden_states <- normalized_hidden_states$masked_fill(~mask_flat, 0.0) - normalized_hidden_states <- normalized_hidden_states$to(dtype=original_dtype) - return(normalized_hidden_states) - - def _get_gemma_prompt_embeds( - self, - prompt: Union[str, List[str]], - num_videos_per_prompt <- 1, - max_sequence_length <- 1024, - scale_factor <- 8, - device <- NULL, - dtype <- NULL, - ): - r""" - Encodes the prompt into text encoder hidden states. - - Args: - prompt (`str` || `List[str]`, *optional*): - prompt to be encoded - device: (`str` || `torch$device`): - torch device to place the resulting embeddings on - dtype: (`torch$dtype`): - torch dtype to cast the prompt embeds to - max_sequence_length (`int`, defaults to 1024): Maximum sequence length to use for the prompt. - """ - device <- device || self$_execution_device - dtype <- dtype || self$text_encoder$dtype - - prompt <- [prompt] if inherits(prompt, "str") else prompt - batch_size <- length(prompt) - - if (getattr(self, "tokenizer", NULL) !is.null) { - # Gemma expects left padding for chat-style prompts - self$tokenizer.padding_side <- "left" - if (self$tokenizer.is.null(pad_token)) { - self$tokenizer.pad_token <- self$tokenizer.eos_token - - prompt <- [p.strip() for p in prompt] - text_inputs <- self$tokenizer( - prompt, - padding <- "max_length", - max_length <- max_sequence_length, - truncation <- TRUE, - add_special_tokens <- TRUE, - return_tensors <- "pt", - ) - text_input_ids <- text_inputs.input_ids - prompt_attention_mask <- text_inputs.attention_mask - text_input_ids <- text_input_ids$to(device) - prompt_attention_mask <- prompt_attention_mask$to(device) - - text_encoder_outputs <- self$text_encoder( - input_ids <- text_input_ids, attention_mask=prompt_attention_mask, output_hidden_states=TRUE - ) - text_encoder_hidden_states <- text_encoder_outputs.hidden_states - text_encoder_hidden_states <- torch_stack(text_encoder_hidden_states, dim=-1) - sequence_lengths <- prompt_attention_mask$sum(dim=-1) - - prompt_embeds <- self$_pack_text_embeds( - text_encoder_hidden_states, - sequence_lengths, - device <- device, - padding_side <- self$tokenizer.padding_side, - scale_factor <- scale_factor, - ) - prompt_embeds <- prompt_embeds$to(dtype=dtype) - - # duplicate text embeddings for each generation per prompt, using mps friendly method - _, seq_len, _ <- prompt_embeds.shape - prompt_embeds <- prompt_embeds$repeat(1, num_videos_per_prompt, 1) - prompt_embeds <- prompt_embeds$view(batch_size * num_videos_per_prompt, seq_len, -1) - - prompt_attention_mask <- prompt_attention_mask$view(batch_size, -1) - prompt_attention_mask <- prompt_attention_mask$repeat(num_videos_per_prompt, 1) - - return(list(prompt_embeds, prompt_attention_mask)) - - def encode_prompt( - self, - prompt: Union[str, List[str]], - negative_prompt <- NULL, - do_classifier_free_guidance <- TRUE, - num_videos_per_prompt <- 1, - prompt_embeds <- NULL, - negative_prompt_embeds <- NULL, - prompt_attention_mask <- NULL, - negative_prompt_attention_mask <- NULL, - max_sequence_length <- 1024, - scale_factor <- 8, - device <- NULL, - dtype <- NULL, - ): - r""" - Encodes the prompt into text encoder hidden states. - - Args: - prompt (`str` || `List[str]`, *optional*): - prompt to be encoded - negative_prompt (`str` || `List[str]`, *optional*): - The prompt || prompts ! to guide the image generation. If ! defined, one has to pass - `negative_prompt_embeds` instead. Ignored when ! using guidance (i.e., ignored if `guidance_scale` is - less than `1`). - do_classifier_free_guidance (`bool`, *optional*, defaults to `TRUE`): - Whether to use classifier free guidance || !. - num_videos_per_prompt (`int`, *optional*, defaults to 1): - Number of videos that should be generated per prompt. torch device to place the resulting embeddings on - prompt_embeds (`torch.Tensor`, *optional*): - Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If ! - provided, text embeddings will be generated from `prompt` input argument. - negative_prompt_embeds (`torch.Tensor`, *optional*): - Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt - weighting. If ! provided, negative_prompt_embeds will be generated from `negative_prompt` input - argument. - device: (`torch$device`, *optional*): - torch device - dtype: (`torch$dtype`, *optional*): - torch dtype - """ - device <- device || self$_execution_device - - prompt <- [prompt] if inherits(prompt, "str") else prompt - if (!is.null(prompt)) { - batch_size <- length(prompt) - } else { - batch_size <- prompt_embeds.shape[0] - - if (is.null(prompt_embeds)) { - prompt_embeds, prompt_attention_mask <- self$_get_gemma_prompt_embeds( - prompt <- prompt, - num_videos_per_prompt <- num_videos_per_prompt, - max_sequence_length <- max_sequence_length, - scale_factor <- scale_factor, - device <- device, - dtype <- dtype, - ) - - if (do_classifier_free_guidance && is.null(negative_prompt_embeds)) { - negative_prompt <- negative_prompt || "" - negative_prompt <- batch_size * [negative_prompt] if inherits(negative_prompt, "str") else negative_prompt - - if (!is.null(prompt) && type(prompt) is ! type(negative_prompt)) { - raise TypeError( - sprintf("`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=") - sprintf(" {type(prompt)}.") - ) - } else if (batch_size != length(negative_prompt)) { - raise ValueError( - sprintf("`negative_prompt`: {negative_prompt} has batch size {length(negative_prompt)}, but `prompt`:") - sprintf(" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches") - " the batch size of `prompt`." - ) - - negative_prompt_embeds, negative_prompt_attention_mask <- self$_get_gemma_prompt_embeds( - prompt <- negative_prompt, - num_videos_per_prompt <- num_videos_per_prompt, - max_sequence_length <- max_sequence_length, - scale_factor <- scale_factor, - device <- device, - dtype <- dtype, - ) - - return(list(prompt_embeds, prompt_attention_mask, negative_prompt_embeds, negative_prompt_attention_mask)) - - def check_inputs( - self, - prompt, - height, - width, - callback_on_step_end_tensor_inputs <- NULL, - prompt_embeds <- NULL, - negative_prompt_embeds <- NULL, - prompt_attention_mask <- NULL, - negative_prompt_attention_mask <- NULL, - ): - if (height % 32 != 0 || width % 32 != 0) { - raise ValueError(sprintf("`height` && `width` have to be divisible by 32 but are {height} && {width}.")) - - if !is.null(callback_on_step_end_tensor_inputs) && ! all( - k in self$_callback_tensor_inputs for k in callback_on_step_end_tensor_inputs - ): - raise ValueError( - sprintf("`callback_on_step_end_tensor_inputs` has to be in {self$_callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k ! in self$_callback_tensor_inputs]}") - ) - - if (!is.null(prompt) && !is.null(prompt_embeds)) { - raise ValueError( - sprintf("Cannot forward both `prompt`: {prompt} && `prompt_embeds`: {prompt_embeds}. Please make sure to") - " only forward one of the two." - ) - } else if (is.null(prompt) && is.null(prompt_embeds)) { - raise ValueError( - "Provide either `prompt` || `prompt_embeds`. Cannot leave both `prompt` && `prompt_embeds` undefined." - ) - } else if (!is.null(prompt) && (! inherits(prompt, "str") && ! inherits(prompt, "list"))) { - raise ValueError(sprintf("`prompt` has to be of type `str` || `list` but is {type(prompt)}")) - - if (!is.null(prompt_embeds) && is.null(prompt_attention_mask)) { - raise ValueError("Must provide `prompt_attention_mask` when specifying `prompt_embeds`.") - - if (!is.null(negative_prompt_embeds) && is.null(negative_prompt_attention_mask)) { - raise ValueError("Must provide `negative_prompt_attention_mask` when specifying `negative_prompt_embeds`.") - - if (!is.null(prompt_embeds) && !is.null(negative_prompt_embeds)) { - if (prompt_embeds.shape != negative_prompt_embeds.shape) { - raise ValueError( - "`prompt_embeds` && `negative_prompt_embeds` must have the same shape when passed directly, but" - sprintf(" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`") - sprintf(" {negative_prompt_embeds.shape}.") - ) - if (prompt_attention_mask.shape != negative_prompt_attention_mask.shape) { - raise ValueError( - "`prompt_attention_mask` && `negative_prompt_attention_mask` must have the same shape when passed directly, but" - sprintf(" got: `prompt_attention_mask` {prompt_attention_mask.shape} != `negative_prompt_attention_mask`") - sprintf(" {negative_prompt_attention_mask.shape}.") - ) - - @staticmethod - def _pack_latents(latents: torch.Tensor, patch_size= 1, patch_size_t= 1) -> torch.Tensor: - # Unpacked latents of shape are [B, C, F, H, W] are patched into tokens of shape [B, C, F // p_t, p_t, H // p, p, W // p, p]. - # The patch dimensions are then permuted and collapsed into the channel dimension of shape: - # [B, F // p_t * H // p * W // p, C * p_t * p * p] (an ndim=3 tensor). - # dim=0 is the batch size, dim=1 is the effective video sequence length, dim=2 is the effective number of input features - batch_size, num_channels, num_frames, height, width <- latents.shape - post_patch_num_frames <- num_frames %/% patch_size_t - post_patch_height <- height %/% patch_size - post_patch_width <- width %/% patch_size - latents <- latents$reshape( - batch_size, - -1, - post_patch_num_frames, - patch_size_t, - post_patch_height, - patch_size, - post_patch_width, - patch_size, - ) - latents <- latents$permute(0, 2, 4, 6, 1, 3, 5, 7)$flatten(4, 7)$flatten(1, 3) - return(latents) - - @staticmethod - def _unpack_latents( - latents: torch.Tensor, num_frames, height, width, patch_size <- 1, patch_size_t= 1 - ) -> torch.Tensor: - # Packed latents of shape [B, S, D] (S is the effective video sequence length, D is the effective feature dimensions) - # are unpacked and reshaped into a video tensor of shape [B, C, F, H, W]. This is the inverse operation of - # what happens in the `_pack_latents` method. - batch_size <- latents$size(0) - latents <- latents$reshape(batch_size, num_frames, height, width, -1, patch_size_t, patch_size, patch_size) - latents <- latents$permute(0, 4, 1, 5, 2, 6, 3, 7)$flatten(6, 7)$flatten(4, 5)$flatten(2, 3) - return(latents) - - @staticmethod - def _denormalize_latents( - latents: torch.Tensor, latents_mean: torch.Tensor, latents_std: torch.Tensor, scaling_factor <- 1.0 - ) -> torch.Tensor: - # Denormalize latents across the channel dimension [B, C, F, H, W] - latents_mean <- latents_mean$view(1, -1, 1, 1, 1)$to(latents$device, latents$dtype) - latents_std <- latents_std$view(1, -1, 1, 1, 1)$to(latents$device, latents$dtype) - latents <- latents * latents_std / scaling_factor + latents_mean - return(latents) - - @staticmethod - def _denormalize_audio_latents(latents: torch.Tensor, latents_mean: torch.Tensor, latents_std: torch.Tensor): - latents_mean <- latents_mean$to(latents$device, latents$dtype) - latents_std <- latents_std$to(latents$device, latents$dtype) - return((latents * latents_std) + latents_mean) - - @staticmethod - def _pack_audio_latents( - latents: torch.Tensor, patch_size <- NULL, patch_size_t= NULL - ) -> torch.Tensor: - # Audio latents shape: [B, C, L, M], where L is the latent audio length and M is the number of mel bins - if (!is.null(patch_size) && !is.null(patch_size_t)) { - # Packs the latents into a patch sequence of shape [B, L // p_t * M // p, C * p_t * p] (a ndim=3 tnesor). - # dim=1 is the effective audio sequence length and dim=2 is the effective audio input feature size. - batch_size, num_channels, latent_length, latent_mel_bins <- latents.shape - post_patch_latent_length <- latent_length / patch_size_t - post_patch_mel_bins <- latent_mel_bins / patch_size - latents <- latents$reshape( - batch_size, -1, post_patch_latent_length, patch_size_t, post_patch_mel_bins, patch_size - ) - latents <- latents$permute(0, 2, 4, 1, 3, 5)$flatten(3, 5)$flatten(1, 2) - } else { - # Packs the latents into a patch sequence of shape [B, L, C * M]. This implicitly assumes a (mel) - # patch_size of M (all mel bins constitutes a single patch) and a patch_size_t of 1. - latents <- latents$transpose(1, 2)$flatten(2, 3) # [B, C, L, M] --> [B, L, C * M] - return(latents) - - @staticmethod - def _unpack_audio_latents( - latents: torch.Tensor, - latent_length, - num_mel_bins, - patch_size <- NULL, - patch_size_t <- NULL, - ) -> torch.Tensor: - # Unpacks an audio patch sequence of shape [B, S, D] into a latent spectrogram tensor of shape [B, C, L, M], - # where L is the latent audio length and M is the number of mel bins. - if (!is.null(patch_size) && !is.null(patch_size_t)) { - batch_size <- latents$size(0) - latents <- latents$reshape(batch_size, latent_length, num_mel_bins, -1, patch_size_t, patch_size) - latents <- latents$permute(0, 3, 1, 4, 2, 5)$flatten(4, 5)$flatten(2, 3) - } else { - # Assume [B, S, D] = [B, L, C * M], which implies that patch_size = M and patch_size_t = 1. - latents <- latents.unflatten(2, (-1, num_mel_bins))$transpose(1, 2) - return(latents) - - def prepare_latents( - self, - batch_size <- 1, - num_channels_latents <- 128, - height <- 512, - width <- 768, - num_frames <- 121, - dtype <- NULL, - device <- NULL, - generator <- NULL, - latents <- NULL, - ) -> torch.Tensor: - if (!is.null(latents)) { - return(latents$to(device=device, dtype=dtype)) - - height <- height %/% self$vae_spatial_compression_ratio - width <- width %/% self$vae_spatial_compression_ratio - num_frames <- (num_frames - 1) %/% self$vae_temporal_compression_ratio + 1 - - shape <- (batch_size, num_channels_latents, num_frames, height, width) - - if (inherits(generator, "list") && length(generator) != batch_size) { - raise ValueError( - sprintf("You have passed a list of generators of length {length(generator)}, but requested an effective batch") - sprintf(" size of {batch_size}. Make sure the batch size matches the length of the generators.") - ) - - latents <- randn_tensor(shape, generator=generator, device=device, dtype=dtype) - latents <- self$_pack_latents( - latents, self$transformer_spatial_patch_size, self$transformer_temporal_patch_size - ) - return(latents) - - def prepare_audio_latents( - self, - batch_size <- 1, - num_channels_latents <- 8, - num_mel_bins <- 64, - num_frames <- 121, - frame_rate <- 25.0, - sampling_rate <- 16000, - hop_length <- 160, - dtype <- NULL, - device <- NULL, - generator <- NULL, - latents <- NULL, - ) -> torch.Tensor: - duration_s <- num_frames / frame_rate - latents_per_second <- ( - float(sampling_rate) / float(hop_length) / float(self$audio_vae_temporal_compression_ratio) - ) - latent_length <- round(duration_s * latents_per_second) - - if (!is.null(latents)) { - return(latents$to(device=device, dtype=dtype), latent_length) - - # TODO: confirm whether this logic is correct - latent_mel_bins <- num_mel_bins %/% self$audio_vae_mel_compression_ratio - - shape <- (batch_size, num_channels_latents, latent_length, latent_mel_bins) - - if (inherits(generator, "list") && length(generator) != batch_size) { - raise ValueError( - sprintf("You have passed a list of generators of length {length(generator)}, but requested an effective batch") - sprintf(" size of {batch_size}. Make sure the batch size matches the length of the generators.") - ) - - latents <- randn_tensor(shape, generator=generator, device=device, dtype=dtype) - latents <- self$_pack_audio_latents(latents) - return(list(latents, latent_length)) - - @property - def guidance_scale(self): - return(self$_guidance_scale) - - @property - def guidance_rescale(self): - return(self$_guidance_rescale) - - @property - def do_classifier_free_guidance(self): - return(self$_guidance_scale > 1.0) - - @property - def num_timesteps(self): - return(self$_num_timesteps) - - @property - def current_timestep(self): - return(self$_current_timestep) - - @property - def attention_kwargs(self): - return(self$_attention_kwargs) - - @property - def interrupt(self): - return(self$_interrupt) - -# NOTE: wrap body in with_no_grad({ ... }) - @replace_example_docstring(EXAMPLE_DOC_STRING) - def __call__( - self, - prompt: Union[str, List[str]] = NULL, - negative_prompt <- NULL, - height <- 512, - width <- 768, - num_frames <- 121, - frame_rate <- 24.0, - num_inference_steps <- 40, - timesteps <- NULL, - guidance_scale <- 4.0, - guidance_rescale <- 0.0, - num_videos_per_prompt <- 1, - generator <- NULL, - latents <- NULL, - audio_latents <- NULL, - prompt_embeds <- NULL, - prompt_attention_mask <- NULL, - negative_prompt_embeds <- NULL, - negative_prompt_attention_mask <- NULL, - decode_timestep: Union[float, List[float]] = 0.0, - decode_noise_scale <- NULL, - output_type <- "pil", - return_dict <- TRUE, - attention_kwargs <- NULL, - callback_on_step_end, NULL]] = NULL, - callback_on_step_end_tensor_inputs <- ["latents"], - max_sequence_length <- 1024, - ): - r""" - Function invoked when calling the pipeline for generation. - - Args: - prompt (`str` || `List[str]`, *optional*): - The prompt || prompts to guide the image generation. If ! defined, one has to pass `prompt_embeds`. - instead. - height (`int`, *optional*, defaults to `512`): - The height in pixels of the generated image. This is set to 480 by default for the best results. - width (`int`, *optional*, defaults to `768`): - The width in pixels of the generated image. This is set to 848 by default for the best results. - num_frames (`int`, *optional*, defaults to `121`): - The number of video frames to generate - frame_rate (`float`, *optional*, defaults to `24.0`): - The frames per second (FPS) of the generated video. - num_inference_steps (`int`, *optional*, defaults to 40): - The number of denoising steps. More denoising steps usually lead to a higher quality image at the - expense of slower inference. - timesteps (`List[int]`, *optional*): - Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument - in their `set_timesteps` method. If ! defined, the default behavior when `num_inference_steps` is - passed will be used. Must be in descending order. - guidance_scale (`float`, *optional*, defaults to `4.0`): - Guidance scale as defined in [Classifier-Free Diffusion - Guidance](https:%/%huggingface.co/papers/2207.12598). `guidance_scale` is defined as `w` of equation 2. - of [Imagen Paper](https:%/%huggingface.co/papers/2205.11487). Guidance scale is enabled by setting - `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to - the text `prompt`, usually at the expense of lower image quality. - guidance_rescale (`float`, *optional*, defaults to 0.0): - Guidance rescale factor proposed by [Common Diffusion Noise Schedules && Sample Steps are - Flawed](https:%/%huggingface.co/papers/2305.08891) `guidance_scale` is defined as `φ` in equation 16. of - [Common Diffusion Noise Schedules && Sample Steps are - Flawed](https:%/%huggingface.co/papers/2305.08891). Guidance rescale factor should fix overexposure when - using zero terminal SNR. - num_videos_per_prompt (`int`, *optional*, defaults to 1): - The number of videos to generate per prompt. - generator (`torch.Generator` || `List[torch.Generator]`, *optional*): - One || a list of [torch generator(s)](https:%/%pytorch.org/docs/stable/generated/torch.Generator.html) - to make generation deterministic. - latents (`torch.Tensor`, *optional*): - Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for video - generation. Can be used to tweak the same generation with different prompts. If ! provided, a latents - tensor will be generated by sampling using the supplied random `generator`. - audio_latents (`torch.Tensor`, *optional*): - Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for audio - generation. Can be used to tweak the same generation with different prompts. If ! provided, a latents - tensor will be generated by sampling using the supplied random `generator`. - prompt_embeds (`torch.Tensor`, *optional*): - Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If ! - provided, text embeddings will be generated from `prompt` input argument. - prompt_attention_mask (`torch.Tensor`, *optional*): - Pre-generated attention mask for text embeddings. - negative_prompt_embeds (`torch.FloatTensor`, *optional*): - Pre-generated negative text embeddings. For PixArt-Sigma this negative prompt should be "". If ! - provided, negative_prompt_embeds will be generated from `negative_prompt` input argument. - negative_prompt_attention_mask (`torch.FloatTensor`, *optional*): - Pre-generated attention mask for negative text embeddings. - decode_timestep (`float`, defaults to `0.0`): - The timestep at which generated video is decoded. - decode_noise_scale (`float`, defaults to `NULL`): - The interpolation factor between random noise && denoised latents at the decode timestep. - output_type (`str`, *optional*, defaults to `"pil"`): - The output format of the generate image. Choose between - [PIL](https:%/%pillow.readthedocs.io/en/stable/): `PIL.Image.Image` || `np.array`. - return_dict (`bool`, *optional*, defaults to `TRUE`): - Whether || ! to return a [`~pipelines.ltx.LTX2PipelineOutput`] instead of a plain tuple. - attention_kwargs (`dict`, *optional*): - A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under - `self$processor` in - [diffusers.models.attention_processor](https:%/%github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). - callback_on_step_end (`Callable`, *optional*): - A function that calls at the end of each denoising steps during the inference. The function is called - with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step, timestep, - callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by - `callback_on_step_end_tensor_inputs`. - callback_on_step_end_tensor_inputs (`List`, *optional*, defaults to `["latents"]`): - The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list - will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the - `._callback_tensor_inputs` attribute of your pipeline class. - max_sequence_length (`int`, *optional*, defaults to `1024`): - Maximum sequence length to use with the `prompt`. - - Examples: - - Returns: - [`~pipelines.ltx.LTX2PipelineOutput`] || `tuple`: - If `return_dict` is `TRUE`, [`~pipelines.ltx.LTX2PipelineOutput`] is returned, otherwise a `tuple` is - returned where the first element is a list with the generated images. - """ - - if (isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks))) { - callback_on_step_end_tensor_inputs <- callback_on_step_end.tensor_inputs - - # 1. Check inputs. Raise error if not correct - self$check_inputs( - prompt <- prompt, - height <- height, - width <- width, - callback_on_step_end_tensor_inputs <- callback_on_step_end_tensor_inputs, - prompt_embeds <- prompt_embeds, - negative_prompt_embeds <- negative_prompt_embeds, - prompt_attention_mask <- prompt_attention_mask, - negative_prompt_attention_mask <- negative_prompt_attention_mask, - ) - - self$_guidance_scale <- guidance_scale - self$_guidance_rescale <- guidance_rescale - self$_attention_kwargs <- attention_kwargs - self$_interrupt <- FALSE - self$_current_timestep <- NULL - - # 2. Define call parameters - if (!is.null(prompt) && inherits(prompt, "str")) { - batch_size <- 1 - } else if (!is.null(prompt) && inherits(prompt, "list")) { - batch_size <- length(prompt) - } else { - batch_size <- prompt_embeds.shape[0] - - device <- self$_execution_device - - # 3. Prepare text embeddings - ( - prompt_embeds, - prompt_attention_mask, - negative_prompt_embeds, - negative_prompt_attention_mask, - ) = self$encode_prompt( - prompt <- prompt, - negative_prompt <- negative_prompt, - do_classifier_free_guidance <- self$do_classifier_free_guidance, - num_videos_per_prompt <- num_videos_per_prompt, - prompt_embeds <- prompt_embeds, - negative_prompt_embeds <- negative_prompt_embeds, - prompt_attention_mask <- prompt_attention_mask, - negative_prompt_attention_mask <- negative_prompt_attention_mask, - max_sequence_length <- max_sequence_length, - device <- device, - ) - if (self$do_classifier_free_guidance) { - prompt_embeds <- torch_cat([negative_prompt_embeds, prompt_embeds], dim=0) - prompt_attention_mask <- torch_cat([negative_prompt_attention_mask, prompt_attention_mask], dim=0) - - additive_attention_mask <- (1 - prompt_attention_mask$to(prompt_embeds$dtype)) * -1000000.0 - connector_prompt_embeds, connector_audio_prompt_embeds, connector_attention_mask <- self$connectors( - prompt_embeds, additive_attention_mask, additive_mask <- TRUE - ) - - # 4. Prepare latent variables - latent_num_frames <- (num_frames - 1) %/% self$vae_temporal_compression_ratio + 1 - latent_height <- height %/% self$vae_spatial_compression_ratio - latent_width <- width %/% self$vae_spatial_compression_ratio - video_sequence_length <- latent_num_frames * latent_height * latent_width - - num_channels_latents <- self$transformer.config.in_channels - latents <- self$prepare_latents( - batch_size * num_videos_per_prompt, - num_channels_latents, - height, - width, - num_frames, - torch.float32, - device, - generator, - latents, - ) - - num_mel_bins <- self$audio_vae.config.mel_bins if getattr(self, "audio_vae", NULL) !is.null else 64 - latent_mel_bins <- num_mel_bins %/% self$audio_vae_mel_compression_ratio - - num_channels_latents_audio <- ( - self$audio_vae.config.latent_channels if getattr(self, "audio_vae", NULL) !is.null else 8 - ) - audio_latents, audio_num_frames <- self$prepare_audio_latents( - batch_size * num_videos_per_prompt, - num_channels_latents <- num_channels_latents_audio, - num_mel_bins <- num_mel_bins, - num_frames <- num_frames, # Video frames, audio frames will be calculated from this - frame_rate <- frame_rate, - sampling_rate <- self$audio_sampling_rate, - hop_length <- self$audio_hop_length, - dtype <- torch.float32, - device <- device, - generator <- generator, - latents <- audio_latents, - ) - - # 5. Prepare timesteps - sigmas <- np.linspace(1.0, 1 / num_inference_steps, num_inference_steps) - mu <- calculate_shift( - video_sequence_length, - self$scheduler.config.get("base_image_seq_len", 1024), - self$scheduler.config.get("max_image_seq_len", 4096), - self$scheduler.config.get("base_shift", 0.95), - self$scheduler.config.get("max_shift", 2.05), - ) - # For now, duplicate the scheduler for use with the audio latents - audio_scheduler <- copy.deepcopy(self$scheduler) - _, _ <- retrieve_timesteps( - audio_scheduler, - num_inference_steps, - device, - timesteps, - sigmas <- sigmas, - mu <- mu, - ) - timesteps, num_inference_steps <- retrieve_timesteps( - self$scheduler, - num_inference_steps, - device, - timesteps, - sigmas <- sigmas, - mu <- mu, - ) - num_warmup_steps <- max(length(timesteps) - num_inference_steps * self$scheduler.order, 0) - self$_num_timesteps <- length(timesteps) - - # 6. Prepare micro-conditions - rope_interpolation_scale <- ( - self$vae_temporal_compression_ratio / frame_rate, - self$vae_spatial_compression_ratio, - self$vae_spatial_compression_ratio, - ) - # Pre-compute video and audio positional ids as they will be the same at each step of the denoising loop - video_coords <- self$transformer.rope.prepare_video_coords( - latents.shape[0], latent_num_frames, latent_height, latent_width, latents$device, fps <- frame_rate - ) - audio_coords <- self$transformer.audio_rope.prepare_audio_coords( - audio_latents.shape[0], audio_num_frames, audio_latents$device - ) - - # 7. Denoising loop - with self$progress_bar(total=num_inference_steps) as progress_bar: - # TODO: tuple unpacking in for loop - for ( for i, t in enumerate(timesteps): in for i, t in seq_along(timesteps):) { - if (self$interrupt) { - continue - - self$_current_timestep <- t - - latent_model_input <- torch_cat(c(latents) * 2) if self$do_classifier_free_guidance else latents - latent_model_input <- latent_model_input$to(prompt_embeds$dtype) - audio_latent_model_input <- ( - torch_cat(c(audio_latents) * 2) if self$do_classifier_free_guidance else audio_latents - ) - audio_latent_model_input <- audio_latent_model_input$to(prompt_embeds$dtype) - - # broadcast to batch dimension in a way that's compatible with ONNX/Core ML - timestep <- t$expand(latent_model_input.shape[0]) - - with self$transformer.cache_context("cond_uncond"): - noise_pred_video, noise_pred_audio <- self$transformer( - hidden_states <- latent_model_input, - audio_hidden_states <- audio_latent_model_input, - encoder_hidden_states <- connector_prompt_embeds, - audio_encoder_hidden_states <- connector_audio_prompt_embeds, - timestep <- timestep, - encoder_attention_mask <- connector_attention_mask, - audio_encoder_attention_mask <- connector_attention_mask, - num_frames <- latent_num_frames, - height <- latent_height, - width <- latent_width, - fps <- frame_rate, - audio_num_frames <- audio_num_frames, - video_coords <- video_coords, - audio_coords <- audio_coords, - # rope_interpolation_scale=rope_interpolation_scale, - attention_kwargs <- attention_kwargs, - return_dict <- FALSE, - ) - noise_pred_video <- noise_pred_video$float() - noise_pred_audio <- noise_pred_audio$float() - - if (self$do_classifier_free_guidance) { - noise_pred_video_uncond, noise_pred_video_text <- noise_pred_video$chunk(2) - noise_pred_video <- noise_pred_video_uncond + self$guidance_scale * ( - noise_pred_video_text - noise_pred_video_uncond - ) - - noise_pred_audio_uncond, noise_pred_audio_text <- noise_pred_audio$chunk(2) - noise_pred_audio <- noise_pred_audio_uncond + self$guidance_scale * ( - noise_pred_audio_text - noise_pred_audio_uncond - ) - - if (self$guidance_rescale > 0) { - # Based on 3.4. in https://huggingface.co/papers/2305.08891 - noise_pred_video <- rescale_noise_cfg( - noise_pred_video, noise_pred_video_text, guidance_rescale <- self$guidance_rescale - ) - noise_pred_audio <- rescale_noise_cfg( - noise_pred_audio, noise_pred_audio_text, guidance_rescale <- self$guidance_rescale - ) - - # compute the previous noisy sample x_t -> x_t-1 - latents <- self$scheduler.step(noise_pred_video, t, latents, return_dict=FALSE)[0] - # NOTE: for now duplicate scheduler for audio latents in case self.scheduler sets internal state in - # the step method (such as _step_index) - audio_latents <- audio_scheduler.step(noise_pred_audio, t, audio_latents, return_dict=FALSE)[0] - - if (!is.null(callback_on_step_end)) { - callback_kwargs <- {} - for (k in callback_on_step_end_tensor_inputs) { - callback_kwargs[k] = locals()[k] - callback_outputs <- callback_on_step_end(self, i, t, callback_kwargs) - - latents <- callback_outputs.pop("latents", latents) - prompt_embeds <- callback_outputs.pop("prompt_embeds", prompt_embeds) - - # call the callback, if provided - if (i == length(timesteps) - 1 || ((i + 1) > num_warmup_steps && (i + 1) % self$scheduler.order == 0)) { - progress_bar.update() - - if (XLA_AVAILABLE) { - xm.mark_step() - - latents <- self$_unpack_latents( - latents, - latent_num_frames, - latent_height, - latent_width, - self$transformer_spatial_patch_size, - self$transformer_temporal_patch_size, - ) - latents <- self$_denormalize_latents( - latents, self$vae.latents_mean, self$vae.latents_std, self$vae.config.scaling_factor - ) - - audio_latents <- self$_denormalize_audio_latents( - audio_latents, self$audio_vae.latents_mean, self$audio_vae.latents_std - ) - audio_latents <- self$_unpack_audio_latents(audio_latents, audio_num_frames, num_mel_bins=latent_mel_bins) - - if (output_type == "latent") { - video <- latents - audio <- audio_latents - } else { - latents <- latents$to(prompt_embeds$dtype) - - if (! self$vae.config.timestep_conditioning) { - timestep <- NULL - } else { - noise <- randn_tensor(latents.shape, generator=generator, device=device, dtype=latents$dtype) - if (! inherits(decode_timestep, "list")) { - decode_timestep <- [decode_timestep] * batch_size - if (is.null(decode_noise_scale)) { - decode_noise_scale <- decode_timestep - } else if (! inherits(decode_noise_scale, "list")) { - decode_noise_scale <- [decode_noise_scale] * batch_size - - timestep <- torch_tensor(decode_timestep, device=device, dtype=latents$dtype) - decode_noise_scale <- torch_tensor(decode_noise_scale, device=device, dtype=latents$dtype)[ - :, NULL, NULL, NULL, NULL - ] - latents <- (1 - decode_noise_scale) * latents + decode_noise_scale * noise - - latents <- latents$to(self$vae$dtype) - video <- self$vae.decode(latents, timestep, return_dict=FALSE)[0] - video <- self$video_processor.postprocess_video(video, output_type=output_type) - - audio_latents <- audio_latents$to(self$audio_vae$dtype) - generated_mel_spectrograms <- self$audio_vae.decode(audio_latents, return_dict=FALSE)[0] - audio <- self$vocoder(generated_mel_spectrograms) - - # Offload all models - self$maybe_free_model_hooks() - - if (! return_dict) { - return(list(video, audio)) - - return(LTX2PipelineOutput(frames=video, audio=audio)) - diff --git a/ref/scheduling_flow_match_euler_discrete.R b/ref/scheduling_flow_match_euler_discrete.R deleted file mode 100644 index ce4a57d..0000000 --- a/ref/scheduling_flow_match_euler_discrete.R +++ /dev/null @@ -1,610 +0,0 @@ -# Converted from PyTorch by pyrotechnics -# Review: indexing (0->1 based), integer literals (add L), -# and block structure (braces may need adjustment) - -# Copyright 2025 Stability AI, Katherine Crowson and The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# import math -# from dataclasses import dataclass -# from typing import List, Optional, Tuple, Union - -# import numpy as np -# import torch - -# from ..configuration_utils import ConfigMixin, register_to_config -# from ..utils import BaseOutput, is_scipy_available, logging -# from .scheduling_utils import SchedulerMixin - - -if (is_scipy_available()) { -# import scipy.stats - -logger <- logging.get_logger(__name__) # pylint: disable=invalid-name - - -@dataclass -class FlowMatchEulerDiscreteSchedulerOutput(BaseOutput): - """ - Output class for the scheduler's `step` function output. - - Args: - prev_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): - Computed sample `(x_{t-1})` of previous timestep. `prev_sample` should be used as next model input in the - denoising loop. - """ - - prev_sample: torch.FloatTensor - - -class FlowMatchEulerDiscreteScheduler(SchedulerMixin, ConfigMixin): - """ - Euler scheduler. - - This model inherits from [`SchedulerMixin`] && [`ConfigMixin`]. Check the superclass documentation for the generic - methods the library implements for all schedulers such as loading && saving. - - Args: - num_train_timesteps (`int`, defaults to 1000): - The number of diffusion steps to train the model. - shift (`float`, defaults to 1.0): - The shift value for the timestep schedule. - use_dynamic_shifting (`bool`, defaults to FALSE): - Whether to apply timestep shifting on-the-fly based on the image resolution. - base_shift (`float`, defaults to 0.5): - Value to stabilize image generation. Increasing `base_shift` reduces variation && image is more consistent - with desired output. - max_shift (`float`, defaults to 1.15): - Value change allowed to latent vectors. Increasing `max_shift` encourages more variation && image may be - more exaggerated || stylized. - base_image_seq_len (`int`, defaults to 256): - The base image sequence length. - max_image_seq_len (`int`, defaults to 4096): - The maximum image sequence length. - invert_sigmas (`bool`, defaults to FALSE): - Whether to invert the sigmas. - shift_terminal (`float`, defaults to NULL): - The end value of the shifted timestep schedule. - use_karras_sigmas (`bool`, defaults to FALSE): - Whether to use Karras sigmas for step sizes in the noise schedule during sampling. - use_exponential_sigmas (`bool`, defaults to FALSE): - Whether to use exponential sigmas for step sizes in the noise schedule during sampling. - use_beta_sigmas (`bool`, defaults to FALSE): - Whether to use beta sigmas for step sizes in the noise schedule during sampling. - time_shift_type (`str`, defaults to "exponential"): - The type of dynamic resolution-dependent timestep shifting to apply. Either "exponential" || "linear". - stochastic_sampling (`bool`, defaults to FALSE): - Whether to use stochastic sampling. - """ - - _compatibles <- [] - order <- 1 - - @register_to_config - def __init__( - self, - num_train_timesteps <- 1000, - shift <- 1.0, - use_dynamic_shifting <- FALSE, - base_shift <- 0.5, - max_shift <- 1.15, - base_image_seq_len <- 256, - max_image_seq_len <- 4096, - invert_sigmas <- FALSE, - shift_terminal <- NULL, - use_karras_sigmas <- FALSE, - use_exponential_sigmas <- FALSE, - use_beta_sigmas <- FALSE, - time_shift_type <- "exponential", - stochastic_sampling <- FALSE, - ): - if (self$config.use_beta_sigmas && ! is_scipy_available()) { - raise ImportError("Make sure to install scipy if you want to use beta sigmas.") - if (sum([self$config.use_beta_sigmas, self$config.use_exponential_sigmas, self$config.use_karras_sigmas]) > 1) { - raise ValueError( - "Only one of `config.use_beta_sigmas`, `config.use_exponential_sigmas`, `config.use_karras_sigmas` can be used." - ) - if (time_shift_type ! in {"exponential", "linear"}) { - raise ValueError("`time_shift_type` must either be 'exponential' || 'linear'.") - - timesteps <- np.linspace(1, num_train_timesteps, num_train_timesteps, dtype=np.float32)[::-1].copy() - timesteps <- torch_from_numpy(timesteps)$to(dtype=torch.float32) - - sigmas <- timesteps / num_train_timesteps - if (! use_dynamic_shifting) { - # when use_dynamic_shifting is True, we apply the timestep shifting on the fly based on the image resolution - sigmas <- shift * sigmas / (1 + (shift - 1) * sigmas) - - self$timesteps <- sigmas * num_train_timesteps - - self$_step_index <- NULL - self$_begin_index <- NULL - - self$_shift <- shift - - self$sigmas <- sigmas$to("cpu") # to avoid too much CPU/GPU communication - self$sigma_min <- self$sigmas[-1].item() - self$sigma_max <- self$sigmas[0].item() - - @property - def shift(self): - """ - The value used for shifting. - """ - return(self$_shift) - - @property - def step_index(self): - """ - The index counter for current timestep. It will increase 1 after each scheduler step. - """ - return(self$_step_index) - - @property - def begin_index(self): - """ - The index for the first timestep. It should be set from pipeline with `set_begin_index` method. - """ - return(self$_begin_index) - - # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index - def set_begin_index(self, begin_index= 0): - """ - Sets the begin index for the scheduler. This function should be run from pipeline before the inference. - - Args: - begin_index (`int`, defaults to `0`): - The begin index for the scheduler. - """ - self$_begin_index <- begin_index - - def set_shift(self, shift): - self$_shift <- shift - - def scale_noise( - self, - sample: torch.FloatTensor, - timestep: torch.FloatTensor, - noise: torch.FloatTensor, - ) -> torch.FloatTensor: - """ - Forward process in flow-matching - - Args: - sample (`torch.FloatTensor`): - The input sample. - timestep (`torch.FloatTensor`): - The current timestep in the diffusion chain. - noise (`torch.FloatTensor`): - The noise tensor. - - Returns: - `torch.FloatTensor`: - A scaled input sample. - """ - # Make sure sigmas and timesteps have the same device and dtype as original_samples - sigmas <- self$sigmas$to(device=sample$device, dtype=sample$dtype) - - if (sample$device.type == "mps" && torch_is_floating_point(timestep)) { - # mps does not support float64 - schedule_timesteps <- self$timesteps$to(sample$device, dtype=torch.float32) - timestep <- timestep$to(sample$device, dtype=torch.float32) - } else { - schedule_timesteps <- self$timesteps$to(sample$device) - timestep <- timestep$to(sample$device) - - # self.begin_index is None when scheduler is used for training, or pipeline does not implement set_begin_index - if (self$is.null(begin_index)) { - step_indices <- [self$index_for_timestep(t, schedule_timesteps) for t in timestep] - } else if (self$!is.null(step_index)) { - # add_noise is called after first denoising step (for inpainting) - step_indices <- [self$step_index] * timestep.shape[0] - } else { - # add noise is called before first denoising step to create initial latent(img2img) - step_indices <- [self$begin_index] * timestep.shape[0] - - sigma <- sigmas[step_indices].flatten() - while length(sigma.shape) < length(sample.shape): - sigma <- sigma$unsqueeze(-1) - - sample <- sigma * noise + (1.0 - sigma) * sample - - return(sample) - - def _sigma_to_t(self, sigma): - return(sigma * self$config.num_train_timesteps) - - def time_shift(self, mu, sigma, t: torch.Tensor): - if (self$config.time_shift_type == "exponential") { - return(self$_time_shift_exponential(mu, sigma, t)) - } else if (self$config.time_shift_type == "linear") { - return(self$_time_shift_linear(mu, sigma, t)) - - def stretch_shift_to_terminal(self, t: torch.Tensor) -> torch.Tensor: - r""" - Stretches && shifts the timestep schedule to ensure it terminates at the configured `shift_terminal` config - value. - - Reference: - https:%/%github.com/Lightricks/LTX-Video/blob/a01a171f8fe3d99dce2728d60a73fecf4d4238ae/ltx_video/schedulers/rf.py#L51 - - Args: - t (`torch.Tensor`): - A tensor of timesteps to be stretched && shifted. - - Returns: - `torch.Tensor`: - A tensor of adjusted timesteps such that the final value equals `self$config.shift_terminal`. - """ - one_minus_z <- 1 - t - scale_factor <- one_minus_z[-1] / (1 - self$config.shift_terminal) - stretched_t <- 1 - (one_minus_z / scale_factor) - return(stretched_t) - - def set_timesteps( - self, - num_inference_steps <- NULL, - device: Union[str, torch$device] = NULL, - sigmas <- NULL, - mu <- NULL, - timesteps <- NULL, - ): - """ - Sets the discrete timesteps used for the diffusion chain (to be run before inference). - - Args: - num_inference_steps (`int`, *optional*): - The number of diffusion steps used when generating samples with a pre-trained model. - device (`str` || `torch$device`, *optional*): - The device to which the timesteps should be moved to. If `NULL`, the timesteps are ! moved. - sigmas (`List[float]`, *optional*): - Custom values for sigmas to be used for each diffusion step. If `NULL`, the sigmas are computed - automatically. - mu (`float`, *optional*): - Determines the amount of shifting applied to sigmas when performing resolution-dependent timestep - shifting. - timesteps (`List[float]`, *optional*): - Custom values for timesteps to be used for each diffusion step. If `NULL`, the timesteps are computed - automatically. - """ - if (self$config.use_dynamic_shifting && is.null(mu)) { - raise ValueError("`mu` must be passed when `use_dynamic_shifting` is set to be `TRUE`") - - if (!is.null(sigmas) && !is.null(timesteps)) { - if (length(sigmas) != length(timesteps)) { - raise ValueError("`sigmas` && `timesteps` should have the same length") - - if (!is.null(num_inference_steps)) { - if (!is.null(sigmas) && length(sigmas) != num_inference_steps) || ( - !is.null(timesteps) && length(timesteps) != num_inference_steps - ): - raise ValueError( - "`sigmas` && `timesteps` should have the same length as num_inference_steps, if `num_inference_steps` is provided" - ) - } else { - num_inference_steps <- length(sigmas) if !is.null(sigmas) else length(timesteps) - - self$num_inference_steps <- num_inference_steps - - # 1. Prepare default sigmas - is_timesteps_provided <- !is.null(timesteps) - - if (is_timesteps_provided) { - timesteps <- np.array(timesteps).astype(np.float32) - - if (is.null(sigmas)) { - if (is.null(timesteps)) { - timesteps <- np.linspace( - self$_sigma_to_t(self$sigma_max), self$_sigma_to_t(self$sigma_min), num_inference_steps - ) - sigmas <- timesteps / self$config.num_train_timesteps - } else { - sigmas <- np.array(sigmas).astype(np.float32) - num_inference_steps <- length(sigmas) - - # 2. Perform timestep shifting. Either no shifting is applied, or resolution-dependent shifting of - # "exponential" or "linear" type is applied - if (self$config.use_dynamic_shifting) { - sigmas <- self$time_shift(mu, 1.0, sigmas) - } else { - sigmas <- self$shift * sigmas / (1 + (self$shift - 1) * sigmas) - - # 3. If required, stretch the sigmas schedule to terminate at the configured `shift_terminal` value - if (self$config.shift_terminal) { - sigmas <- self$stretch_shift_to_terminal(sigmas) - - # 4. If required, convert sigmas to one of karras, exponential, or beta sigma schedules - if (self$config.use_karras_sigmas) { - sigmas <- self$_convert_to_karras(in_sigmas=sigmas, num_inference_steps=num_inference_steps) - } else if (self$config.use_exponential_sigmas) { - sigmas <- self$_convert_to_exponential(in_sigmas=sigmas, num_inference_steps=num_inference_steps) - } else if (self$config.use_beta_sigmas) { - sigmas <- self$_convert_to_beta(in_sigmas=sigmas, num_inference_steps=num_inference_steps) - - # 5. Convert sigmas and timesteps to tensors and move to specified device - sigmas <- torch_from_numpy(sigmas)$to(dtype=torch.float32, device=device) - if (! is_timesteps_provided) { - timesteps <- sigmas * self$config.num_train_timesteps - } else { - timesteps <- torch_from_numpy(timesteps)$to(dtype=torch.float32, device=device) - - # 6. Append the terminal sigma value. - # If a model requires inverted sigma schedule for denoising but timesteps without inversion, the - # `invert_sigmas` flag can be set to `True`. This case is only required in Mochi - if (self$config.invert_sigmas) { - sigmas <- 1.0 - sigmas - timesteps <- sigmas * self$config.num_train_timesteps - sigmas <- torch_cat([sigmas, torch_ones(1, device=sigmas$device)]) - } else { - sigmas <- torch_cat([sigmas, torch_zeros(1, device=sigmas$device)]) - - self$timesteps <- timesteps - self$sigmas <- sigmas - self$_step_index <- NULL - self$_begin_index <- NULL - - def index_for_timestep(self, timestep, schedule_timesteps=NULL): - if (is.null(schedule_timesteps)) { - schedule_timesteps <- self$timesteps - - indices <- (schedule_timesteps == timestep).nonzero() - - # The sigma index that is taken for the **very** first `step` - # is always the second index (or the last index if there is only 1) - # This way we can ensure we don't accidentally skip a sigma in - # case we start in the middle of the denoising schedule (e.g. for image-to-image) - pos <- 1 if length(indices) > 1 else 0 - - return(indices[pos].item()) - - def _init_step_index(self, timestep): - if (self$is.null(begin_index)) { - if (isinstance(timestep, torch.Tensor)) { - timestep <- timestep$to(self$timesteps$device) - self$_step_index <- self$index_for_timestep(timestep) - } else { - self$_step_index <- self$_begin_index - - def step( - self, - model_output: torch.FloatTensor, - timestep: Union[float, torch.FloatTensor], - sample: torch.FloatTensor, - s_churn <- 0.0, - s_tmin <- 0.0, - s_tmax <- float("inf"), - s_noise <- 1.0, - generator <- NULL, - per_token_timesteps <- NULL, - return_dict <- TRUE, - ) -> Union[FlowMatchEulerDiscreteSchedulerOutput, Tuple]: - """ - Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion - process from the learned model outputs (most often the predicted noise). - - Args: - model_output (`torch.FloatTensor`): - The direct output from learned diffusion model. - timestep (`float`): - The current discrete timestep in the diffusion chain. - sample (`torch.FloatTensor`): - A current instance of a sample created by the diffusion process. - s_churn (`float`): - s_tmin (`float`): - s_tmax (`float`): - s_noise (`float`, defaults to 1.0): - Scaling factor for noise added to the sample. - generator (`torch.Generator`, *optional*): - A random number generator. - per_token_timesteps (`torch.Tensor`, *optional*): - The timesteps for each token in the sample. - return_dict (`bool`): - Whether || ! to return a - [`~schedulers.scheduling_flow_match_euler_discrete.FlowMatchEulerDiscreteSchedulerOutput`] || tuple. - - Returns: - [`~schedulers.scheduling_flow_match_euler_discrete.FlowMatchEulerDiscreteSchedulerOutput`] || `tuple`: - If return_dict is `TRUE`, - [`~schedulers.scheduling_flow_match_euler_discrete.FlowMatchEulerDiscreteSchedulerOutput`] is returned, - otherwise a tuple is returned where the first element is the sample tensor. - """ - - if ( - inherits(timestep, "int") - || isinstance(timestep, torch.IntTensor) - || isinstance(timestep, torch.LongTensor) - ): - raise ValueError( - ( - "Passing integer indices (e.g. from `seq_along(timesteps)`) as timesteps to" - " `FlowMatchEulerDiscreteScheduler.step()` is ! supported. Make sure to pass" - " one of the `scheduler.timesteps` as a timestep." - ), - ) - - if (self$is.null(step_index)) { - self$_init_step_index(timestep) - - # Upcast to avoid precision issues when computing prev_sample - sample <- sample$to(torch.float32) - - if (!is.null(per_token_timesteps)) { - per_token_sigmas <- per_token_timesteps / self$config.num_train_timesteps - - sigmas <- self$sigmas[:, NULL, NULL] - lower_mask <- sigmas < per_token_sigmas[NULL] - 1e-6 - lower_sigmas <- lower_mask * sigmas - lower_sigmas, _ <- lower_sigmas$max(dim=0) - - current_sigma <- per_token_sigmas[..., NULL] - next_sigma <- lower_sigmas[..., NULL] - dt <- current_sigma - next_sigma - } else { - sigma_idx <- self$step_index - sigma <- self$sigmas[sigma_idx] - sigma_next <- self$sigmas[sigma_idx + 1] - - current_sigma <- sigma - next_sigma <- sigma_next - dt <- sigma_next - sigma - - if (self$config.stochastic_sampling) { - x0 <- sample - current_sigma * model_output - noise <- torch_randn_like(sample) - prev_sample <- (1.0 - next_sigma) * x0 + next_sigma * noise - } else { - prev_sample <- sample + dt * model_output - - # upon completion increase step index by one - self$_step_index += 1 - if (is.null(per_token_timesteps)) { - # Cast sample back to model compatible dtype - prev_sample <- prev_sample$to(model_output$dtype) - - if (! return_dict) { - return(list(prev_sample,)) - - return(FlowMatchEulerDiscreteSchedulerOutput(prev_sample=prev_sample)) - - # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_karras - def _convert_to_karras(self, in_sigmas: torch.Tensor, num_inference_steps) -> torch.Tensor: - """ - Construct the noise schedule as proposed in [Elucidating the Design Space of Diffusion-Based Generative - Models](https:%/%huggingface.co/papers/2206.00364). - - Args: - in_sigmas (`torch.Tensor`): - The input sigma values to be converted. - num_inference_steps (`int`): - The number of inference steps to generate the noise schedule for. - - Returns: - `torch.Tensor`: - The converted sigma values following the Karras noise schedule. - """ - - # Hack to make sure that other schedulers which copy this function don't break - # TODO: Add this logic to the other schedulers - if (hasattr(self$config, "sigma_min")) { - sigma_min <- self$config.sigma_min - } else { - sigma_min <- NULL - - if (hasattr(self$config, "sigma_max")) { - sigma_max <- self$config.sigma_max - } else { - sigma_max <- NULL - - sigma_min <- sigma_min if !is.null(sigma_min) else in_sigmas[-1].item() - sigma_max <- sigma_max if !is.null(sigma_max) else in_sigmas[0].item() - - rho <- 7.0 # 7.0 is the value used in the paper - ramp <- np.linspace(0, 1, num_inference_steps) - min_inv_rho <- sigma_min ^ (1 / rho) - max_inv_rho <- sigma_max ^ (1 / rho) - sigmas <- (max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ^ rho - return(sigmas) - - # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_exponential - def _convert_to_exponential(self, in_sigmas: torch.Tensor, num_inference_steps) -> torch.Tensor: - """ - Construct an exponential noise schedule. - - Args: - in_sigmas (`torch.Tensor`): - The input sigma values to be converted. - num_inference_steps (`int`): - The number of inference steps to generate the noise schedule for. - - Returns: - `torch.Tensor`: - The converted sigma values following an exponential schedule. - """ - - # Hack to make sure that other schedulers which copy this function don't break - # TODO: Add this logic to the other schedulers - if (hasattr(self$config, "sigma_min")) { - sigma_min <- self$config.sigma_min - } else { - sigma_min <- NULL - - if (hasattr(self$config, "sigma_max")) { - sigma_max <- self$config.sigma_max - } else { - sigma_max <- NULL - - sigma_min <- sigma_min if !is.null(sigma_min) else in_sigmas[-1].item() - sigma_max <- sigma_max if !is.null(sigma_max) else in_sigmas[0].item() - - sigmas <- np$exp(np.linspace(math$log(sigma_max), math$log(sigma_min), num_inference_steps)) - return(sigmas) - - # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_beta - def _convert_to_beta( - self, in_sigmas: torch.Tensor, num_inference_steps, alpha <- 0.6, beta= 0.6 - ) -> torch.Tensor: - """ - Construct a beta noise schedule as proposed in [Beta Sampling is All You - Need](https:%/%huggingface.co/papers/2407.12173). - - Args: - in_sigmas (`torch.Tensor`): - The input sigma values to be converted. - num_inference_steps (`int`): - The number of inference steps to generate the noise schedule for. - alpha (`float`, *optional*, defaults to `0.6`): - The alpha parameter for the beta distribution. - beta (`float`, *optional*, defaults to `0.6`): - The beta parameter for the beta distribution. - - Returns: - `torch.Tensor`: - The converted sigma values following a beta distribution schedule. - """ - - # Hack to make sure that other schedulers which copy this function don't break - # TODO: Add this logic to the other schedulers - if (hasattr(self$config, "sigma_min")) { - sigma_min <- self$config.sigma_min - } else { - sigma_min <- NULL - - if (hasattr(self$config, "sigma_max")) { - sigma_max <- self$config.sigma_max - } else { - sigma_max <- NULL - - sigma_min <- sigma_min if !is.null(sigma_min) else in_sigmas[-1].item() - sigma_max <- sigma_max if !is.null(sigma_max) else in_sigmas[0].item() - - sigmas <- np.array( - [ - sigma_min + (ppf * (sigma_max - sigma_min)) - for ppf in [ - scipy.stats.beta.ppf(timestep, alpha, beta) - for timestep in 1 - np.linspace(0, 1, num_inference_steps) - ] - ] - ) - return(sigmas) - - def _time_shift_exponential(self, mu, sigma, t): - return(math$exp(mu) / (math$exp(mu) + (1 / t - 1) ^ sigma)) - - def _time_shift_linear(self, mu, sigma, t): - return(mu / (mu + (1 / t - 1) ^ sigma)) - - def __len__(self): - return(self$config.num_train_timesteps) - From 57e8f9e914e77cae2bc3ccd97b992395053ca833 Mon Sep 17 00:00:00 2001 From: TroyHernandez Date: Fri, 3 Jul 2026 10:57:03 -0500 Subject: [PATCH 03/83] Add LTX-2.3 single-file checkpoint reader ltx23_open_checkpoint() validates model_version metadata and parses the embedded component config; ltx23_split_keys() routes the flat key space to component groups (dit, connectors, vae, audio_vae, vocoder); ltx23_load_group() streams tensors one at a time into module params with two-sided coverage reporting. Tested against a tiny fake checkpoint with official-style keys. --- NAMESPACE | 5 + R/checkpoint_ltx23.R | 258 ++++++++++++++++++++++++++ inst/tinytest/test_checkpoint_ltx23.R | 180 ++++++++++++++++++ man/CLIPTokenizer.Rd | 27 --- man/checkpoint_ltx23.Rd | 13 ++ man/filename_from_prompt.Rd | 22 --- man/ltx23_census.Rd | 16 ++ man/ltx23_load_group.Rd | 37 ++++ man/ltx23_open_checkpoint.Rd | 29 +++ man/ltx23_split_keys.Rd | 22 +++ 10 files changed, 560 insertions(+), 49 deletions(-) create mode 100644 R/checkpoint_ltx23.R create mode 100644 inst/tinytest/test_checkpoint_ltx23.R delete mode 100644 man/CLIPTokenizer.Rd create mode 100644 man/checkpoint_ltx23.Rd delete mode 100644 man/filename_from_prompt.Rd create mode 100644 man/ltx23_census.Rd create mode 100644 man/ltx23_load_group.Rd create mode 100644 man/ltx23_open_checkpoint.Rd create mode 100644 man/ltx23_split_keys.Rd diff --git a/NAMESPACE b/NAMESPACE index 63b2d0a..f764ced 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -32,6 +32,10 @@ export(load_text_encoder2_weights) export(load_to_gpu) export(load_unet_sdxl_weights) export(load_unet_weights) +export(ltx23_census) +export(ltx23_load_group) +export(ltx23_open_checkpoint) +export(ltx23_split_keys) export(models2devices) export(offload_to_cpu) export(post_quant_conv) @@ -57,5 +61,6 @@ export(vocab_size) export(vram_report) S3method(print,bpe_tokenizer) +S3method(print,ltx23_checkpoint) importFrom(utils,head) diff --git a/R/checkpoint_ltx23.R b/R/checkpoint_ltx23.R new file mode 100644 index 0000000..b93ed34 --- /dev/null +++ b/R/checkpoint_ltx23.R @@ -0,0 +1,258 @@ +#' LTX-2.3 Single-File Checkpoint Reader +#' +#' LTX 2.3 checkpoints ship as one safetensors file containing every +#' component (transformer, connectors, video VAE, audio VAE, vocoder), +#' with the model version and full component configuration embedded in +#' the safetensors metadata. These helpers open the file, validate the +#' version, split the key space by component, and stream tensors into +#' R torch modules one at a time so the 46 GB file is never fully +#' materialized in memory. +#' +#' @name checkpoint_ltx23 +NULL + +#' Open an LTX-2.3 checkpoint +#' +#' Opens a single-file LTX checkpoint lazily (header only), validates the +#' \code{model_version} metadata, and parses the embedded component +#' configuration. +#' +#' @param path Path to the checkpoint .safetensors file. +#' @param require_version Character. Required \code{model_version} prefix +#' (default "2.3"). Set to NULL to skip the check. +#' +#' @return An object of class \code{ltx23_checkpoint}: a list with +#' \code{handle} (safetensors reader), \code{keys}, \code{version}, +#' \code{config} (parsed component configs, or NULL), and \code{path}. +#' +#' @export +#' +#' @examples +#' \dontrun{ +#' ckpt <- ltx23_open_checkpoint("ltx-2.3-22b-distilled-1.1.safetensors") +#' str(ltx23_split_keys(ckpt$keys), max.level = 1) +#' } +ltx23_open_checkpoint <- function( + path, + require_version = "2.3" +) { + if (!requireNamespace("safetensors", quietly = TRUE)) { + stop("The safetensors package is required to read LTX checkpoints.") + } + path <- path.expand(path) + if (!file.exists(path)) { + stop("Checkpoint not found: ", path) + } + + handle <- safetensors::safetensors$new(path, framework = "torch") + + meta <- handle$metadata[["__metadata__"]] + version <- if (is.list(meta) || is.character(meta)) meta[["model_version"]] else NULL + + if (!is.null(require_version)) { + if (is.null(version)) { + stop( + "Checkpoint has no model_version metadata; expected an LTX ", + require_version, " single-file checkpoint: ", path + ) + } + if (!startsWith(version, require_version)) { + stop( + "Checkpoint model_version is '", version, "' but '", + require_version, "*' is required: ", path + ) + } + } + + config <- NULL + if (!is.null(meta[["config"]])) { + config <- tryCatch( + jsonlite::fromJSON(meta[["config"]], simplifyVector = TRUE), + error = function(e) NULL + ) + } + + keys <- setdiff(handle$keys(), "__metadata__") + + structure( + list( + handle = handle, + keys = keys, + version = version, + config = config, + path = path + ), + class = "ltx23_checkpoint" + ) +} + +#' @export +print.ltx23_checkpoint <- function(x, ...) { + cat("\n") + cat(" path: ", x$path, "\n") + cat(" version: ", x$version %||% "(none)", "\n") + cat(" tensors: ", length(x$keys), "\n") + groups <- ltx23_split_keys(x$keys) + for (g in names(groups)) { + cat(sprintf(" %-10s %d keys\n", g, length(groups[[g]]))) + } + invisible(x) +} + +#' Split checkpoint keys by component +#' +#' Splits the flat key space of an LTX single-file checkpoint into its +#' component groups. Connector tensors live under the +#' \code{model.diffusion_model.} prefix alongside the transformer, plus a +#' top-level \code{text_embedding_projection.} group; both are routed to +#' the \code{connectors} component. +#' +#' @param keys Character vector of checkpoint tensor names. +#' +#' @return Named list of character vectors: \code{dit}, +#' \code{connectors}, \code{vae}, \code{audio_vae}, \code{vocoder}, +#' and \code{other} (anything unrecognized; should be empty). +#' +#' @export +ltx23_split_keys <- function(keys) { + dm_prefix <- "model.diffusion_model." + connector_res <- c( + "^model\\.diffusion_model\\.video_embeddings_connector\\.", + "^model\\.diffusion_model\\.audio_embeddings_connector\\.", + "^text_embedding_projection\\." + ) + + is_connector <- Reduce(`|`, lapply(connector_res, grepl, x = keys)) + is_dit <- startsWith(keys, dm_prefix) & !is_connector + is_vae <- startsWith(keys, "vae.") + is_audio_vae <- startsWith(keys, "audio_vae.") + is_vocoder <- startsWith(keys, "vocoder.") + + claimed <- is_connector | is_dit | is_vae | is_audio_vae | is_vocoder + + list( + dit = keys[is_dit], + connectors = keys[is_connector], + vae = keys[is_vae], + audio_vae = keys[is_audio_vae], + vocoder = keys[is_vocoder], + other = keys[!claimed] + ) +} + +#' Stream a checkpoint key group into a module +#' +#' Reads tensors one at a time from an open checkpoint and copies them +#' into the matching parameters/buffers of \code{module}. Destination +#' names are derived by \code{map_key}; \code{$copy_()} handles any +#' dtype/device conversion, so the module may already live on its target +#' device in its target dtype. +#' +#' @param ckpt An \code{ltx23_checkpoint}. +#' @param keys Character vector of checkpoint keys to load (one group +#' from \code{\link{ltx23_split_keys}}). +#' @param module A torch nn_module to populate. +#' @param map_key Function mapping a checkpoint key to the module's +#' parameter/buffer name, or NA to skip the key deliberately. +#' @param verbose Logical. Report progress and coverage. +#' @param gc_every Integer. Run \code{gc()} after this many tensors. +#' +#' @return Invisibly, a list with \code{unmapped} (checkpoint keys that +#' found no destination), \code{skipped} (keys the mapper declined), +#' and \code{unfilled} (module parameters/buffers never written). +#' A perfectly loaded group has zero \code{unmapped} and zero +#' \code{unfilled}. +#' +#' @export +ltx23_load_group <- function( + ckpt, + keys, + module, + map_key = identity, + verbose = TRUE, + gc_every = 50L +) { + stopifnot(inherits(ckpt, "ltx23_checkpoint")) + + params <- module$named_parameters() + buffers <- module$named_buffers() + dests <- c(params, buffers) + + unmapped <- character(0) + skipped <- character(0) + filled <- character(0) + + n <- length(keys) + torch::with_no_grad({ + for (i in seq_along(keys)) { + key <- keys[[i]] + dest_name <- map_key(key) + + if (length(dest_name) != 1L || is.na(dest_name)) { + skipped <- c(skipped, key) + next + } + dest <- dests[[dest_name]] + if (is.null(dest)) { + unmapped <- c(unmapped, key) + next + } + + value <- ckpt$handle$get_tensor(key) + if (!identical(as.integer(dest$shape), as.integer(value$shape))) { + stop(sprintf( + "Shape mismatch for '%s' -> '%s': checkpoint [%s], module [%s]", + key, dest_name, + paste(value$shape, collapse = ","), + paste(dest$shape, collapse = ",") + )) + } + dest$copy_(value) + filled <- c(filled, dest_name) + rm(value) + + if (i %% gc_every == 0L) { + gc(verbose = FALSE) + if (verbose && n > 500L) { + message(sprintf(" loaded %d/%d tensors", i, n)) + } + } + } + }) + gc(verbose = FALSE) + + unfilled <- setdiff(names(dests), filled) + + if (verbose) { + message(sprintf( + "Loaded %d/%d tensors (%d skipped); unmapped: %d, unfilled: %d", + length(filled), n, length(skipped), + length(unmapped), length(unfilled) + )) + if (length(unmapped)) { + message(" unmapped e.g.: ", paste(utils::head(unmapped, 3), collapse = ", ")) + } + if (length(unfilled)) { + message(" unfilled e.g.: ", paste(utils::head(unfilled, 3), collapse = ", ")) + } + } + + invisible(list(unmapped = unmapped, skipped = skipped, unfilled = unfilled)) +} + +#' Summarize checkpoint key coverage +#' +#' @param ckpt An \code{ltx23_checkpoint}. +#' +#' @return A data.frame with one row per component group and its key count. +#' +#' @export +ltx23_census <- function(ckpt) { + stopifnot(inherits(ckpt, "ltx23_checkpoint")) + groups <- ltx23_split_keys(ckpt$keys) + data.frame( + group = names(groups), + keys = vapply(groups, length, integer(1)), + row.names = NULL + ) +} diff --git a/inst/tinytest/test_checkpoint_ltx23.R b/inst/tinytest/test_checkpoint_ltx23.R new file mode 100644 index 0000000..4267b97 --- /dev/null +++ b/inst/tinytest/test_checkpoint_ltx23.R @@ -0,0 +1,180 @@ +# Tests for the LTX-2.3 single-file checkpoint reader. +# Uses a tiny fake checkpoint written with official-style key names and +# model_version metadata; no model downloads required. + +if (!requireNamespace("torch", quietly = TRUE) || !torch::torch_is_installed()) { + exit_file("torch not fully installed") +} +if (!requireNamespace("safetensors", quietly = TRUE)) { + exit_file("safetensors not installed") +} + +library(diffuseR) + +make_fake_checkpoint <- function(path, version = "2.3.0") { + tensors <- list( + # DiT group + "model.diffusion_model.patchify_proj.weight" = + torch::torch_randn(8, 4), + "model.diffusion_model.patchify_proj.bias" = + torch::torch_randn(8), + "model.diffusion_model.scale_shift_table" = + torch::torch_randn(6, 8), + "model.diffusion_model.transformer_blocks.0.attn1.to_q.weight" = + torch::torch_randn(8, 8), + # Connector group (inside and outside the diffusion_model prefix) + "model.diffusion_model.video_embeddings_connector.transformer_1d_blocks.0.attn1.to_q.weight" = + torch::torch_randn(8, 8), + "model.diffusion_model.audio_embeddings_connector.transformer_1d_blocks.0.attn1.to_q.weight" = + torch::torch_randn(8, 8), + "text_embedding_projection.video_aggregate_embed.weight" = + torch::torch_randn(8, 12), + "text_embedding_projection.audio_aggregate_embed.weight" = + torch::torch_randn(4, 12), + # VAE / audio VAE / vocoder groups + "vae.decoder.conv_in.conv.weight" = torch::torch_randn(4, 4, 3, 3, 3), + "audio_vae.decoder.conv_in.weight" = torch::torch_randn(4, 2, 3, 3), + "vocoder.vocoder.conv_in.weight" = torch::torch_randn(4, 4, 7), + "vocoder.bwe_generator.conv_in.weight" = torch::torch_randn(4, 4, 7), + "vocoder.mel_stft.mel_basis" = torch::torch_randn(64, 257) + ) + metadata <- list(model_version = version) + safetensors::safe_save_file(tensors, path, metadata = metadata) + invisible(tensors) +} + +tmp <- tempfile(fileext = ".safetensors") +on.exit(unlink(tmp), add = TRUE) +tensors <- make_fake_checkpoint(tmp) + +# --- Open + version gate ----------------------------------------------------- + +ckpt <- ltx23_open_checkpoint(tmp) +expect_inherits(ckpt, "ltx23_checkpoint") +expect_equal(ckpt$version, "2.3.0") +expect_equal(sort(ckpt$keys), sort(names(tensors))) + +# Wrong version is rejected +tmp_old <- tempfile(fileext = ".safetensors") +on.exit(unlink(tmp_old), add = TRUE) +make_fake_checkpoint(tmp_old, version = "2.0.1") +expect_error(ltx23_open_checkpoint(tmp_old), pattern = "2\\.3") + +# Version check can be disabled +ckpt_old <- ltx23_open_checkpoint(tmp_old, require_version = NULL) +expect_equal(ckpt_old$version, "2.0.1") + +# Missing file +expect_error(ltx23_open_checkpoint(tempfile()), pattern = "not found") + +# --- Key group split --------------------------------------------------------- + +groups <- ltx23_split_keys(ckpt$keys) +expect_equal(length(groups$dit), 4L) +expect_equal(length(groups$connectors), 4L) +expect_equal(length(groups$vae), 1L) +expect_equal(length(groups$audio_vae), 1L) +expect_equal(length(groups$vocoder), 3L) +expect_equal(length(groups$other), 0L) + +# Every key lands in exactly one group +expect_equal(sort(unlist(groups, use.names = FALSE)), sort(ckpt$keys)) + +# Census matches +cen <- ltx23_census(ckpt) +expect_equal(cen$keys[cen$group == "vocoder"], 3L) + +# --- Streaming group load ---------------------------------------------------- + +toy <- torch::nn_module( + "toy", + initialize = function() { + self$patchify_proj <- torch::nn_linear(4, 8) + self$scale_shift_table <- torch::nn_parameter(torch::torch_zeros(6, 8)) + self$blocks <- torch::nn_module_list(list( + torch::nn_module( + "toy_block", + initialize = function() { + self$to_q <- torch::nn_linear(8, 8, bias = FALSE) + } + )() + )) + } +)() + +map_dit_toy <- function(key) { + key <- sub("^model\\.diffusion_model\\.", "", key) + key <- sub("^transformer_blocks\\.0\\.attn1\\.", "blocks.0.", key) + key +} + +res <- ltx23_load_group(ckpt, groups$dit, toy, map_key = map_dit_toy, verbose = FALSE) +expect_equal(length(res$unmapped), 0L) +expect_equal(length(res$unfilled), 0L) +expect_equal(length(res$skipped), 0L) + +# Values actually copied +expect_equal( + as.numeric(torch::torch_sum(toy$patchify_proj$weight)), + as.numeric(torch::torch_sum(tensors[["model.diffusion_model.patchify_proj.weight"]])), + tolerance = 1e-5 +) + +# copy_ converts dtype: load into a float64 module +toy64 <- torch::nn_module( + "toy64", + initialize = function() { + self$w <- torch::nn_parameter(torch::torch_zeros(8, 4, dtype = torch::torch_float64())) + } +)() +res64 <- ltx23_load_group( + ckpt, "model.diffusion_model.patchify_proj.weight", toy64, + map_key = function(key) "w", verbose = FALSE +) +expect_equal(length(res64$unmapped), 0L) +expect_equal(toy64$w$dtype$.type(), "Double") + +# Unmapped keys are reported, not fatal +res_bad <- ltx23_load_group( + ckpt, groups$vae, toy, + map_key = identity, verbose = FALSE +) +expect_equal(length(res_bad$unmapped), 1L) + +# Mapper can deliberately skip keys with NA +res_skip <- ltx23_load_group( + ckpt, groups$vae, toy, + map_key = function(key) NA_character_, verbose = FALSE +) +expect_equal(length(res_skip$skipped), 1L) +expect_equal(length(res_skip$unmapped), 0L) + +# Shape mismatch is a hard error +expect_error( + ltx23_load_group( + ckpt, "model.diffusion_model.scale_shift_table", toy, + map_key = function(key) "patchify_proj.weight", verbose = FALSE + ), + pattern = "Shape mismatch" +) + +# --- Real checkpoint census (local only, needs the 46GB file) ---------------- + +if (at_home()) { + real <- tryCatch( + hfhub::hub_download( + "Lightricks/LTX-2.3", "ltx-2.3-22b-distilled-1.1.safetensors", + local_files_only = TRUE + ), + error = function(e) NULL + ) + if (!is.null(real)) { + rc <- ltx23_open_checkpoint(real) + expect_equal(rc$version, "2.3.0") + expect_equal(length(rc$keys), 5947L) + rg <- ltx23_split_keys(rc$keys) + expect_equal(length(rg$other), 0L) + expect_equal(length(rg$vocoder), 1227L) + expect_true(!is.null(rc$config$transformer$num_layers)) + } +} diff --git a/man/CLIPTokenizer.Rd b/man/CLIPTokenizer.Rd deleted file mode 100644 index cddad68..0000000 --- a/man/CLIPTokenizer.Rd +++ /dev/null @@ -1,27 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{CLIPTokenizer} -\alias{CLIPTokenizer} -\title{Tokenize a prompt} -\usage{ -CLIPTokenizer( - prompt, - merges = system.file("tokenizer/merges.txt", package = "diffuseR"), - vocab_file = system.file("tokenizer/vocab.json", package = "diffuseR"), - pad_token = 0L -) -} -\arguments{ -\item{prompt}{A character string prompt describing the image to generate.} - -\item{merges}{Path to the merges file (BPE merges).} - -\item{vocab_file}{Path to the vocabulary file (token->id mapping).} - -\item{pad_token}{The token ID used for padding (default is 0).} -} -\value{ -A 2D torch tensor of shape c(1, 77) containing the token IDs. -} -\description{ -Tokenize a prompt -} diff --git a/man/checkpoint_ltx23.Rd b/man/checkpoint_ltx23.Rd new file mode 100644 index 0000000..ad9f141 --- /dev/null +++ b/man/checkpoint_ltx23.Rd @@ -0,0 +1,13 @@ +% tinyrox says don't edit this manually, but it can't stop you! +\name{checkpoint_ltx23} +\alias{checkpoint_ltx23} +\title{LTX-2.3 Single-File Checkpoint Reader} +\description{ +LTX 2.3 checkpoints ship as one safetensors file containing every +component (transformer, connectors, video VAE, audio VAE, vocoder), +with the model version and full component configuration embedded in +the safetensors metadata. These helpers open the file, validate the +version, split the key space by component, and stream tensors into +R torch modules one at a time so the 46 GB file is never fully +materialized in memory. +} diff --git a/man/filename_from_prompt.Rd b/man/filename_from_prompt.Rd deleted file mode 100644 index 0f59170..0000000 --- a/man/filename_from_prompt.Rd +++ /dev/null @@ -1,22 +0,0 @@ -% tinyrox says don't edit this manually, but it can't stop you! -\name{filename_from_prompt} -\alias{filename_from_prompt} -\title{Generate a filename from a prompt} -\usage{ -filename_from_prompt(prompt, datetime = TRUE) -} -\arguments{ -\item{prompt}{A character string representing the prompt.} - -\item{datetime}{Logical indicating whether to prepend the current date and time to the filename. Default is TRUE.} -} -\value{ -A character string representing the generated filename. -} -\description{ -This function generates a filename from a prompt by removing all non-alphanumeric characters and replacing them with underscores. The filename is limited to 50 characters. If `datetime` is set to TRUE, the current date and time are prepended to the filename. -} -\examples{ -filename_from_prompt("A beautiful sunset over the mountains") -filename_from_prompt("A beautiful sunset over the mountains", datetime = FALSE) -} diff --git a/man/ltx23_census.Rd b/man/ltx23_census.Rd new file mode 100644 index 0000000..762edcb --- /dev/null +++ b/man/ltx23_census.Rd @@ -0,0 +1,16 @@ +% tinyrox says don't edit this manually, but it can't stop you! +\name{ltx23_census} +\alias{ltx23_census} +\title{Summarize checkpoint key coverage} +\usage{ +ltx23_census(ckpt) +} +\arguments{ +\item{ckpt}{An \code{ltx23_checkpoint}.} +} +\value{ +A data.frame with one row per component group and its key count. +} +\description{ +Summarize checkpoint key coverage +} diff --git a/man/ltx23_load_group.Rd b/man/ltx23_load_group.Rd new file mode 100644 index 0000000..8a89164 --- /dev/null +++ b/man/ltx23_load_group.Rd @@ -0,0 +1,37 @@ +% tinyrox says don't edit this manually, but it can't stop you! +\name{ltx23_load_group} +\alias{ltx23_load_group} +\title{Stream a checkpoint key group into a module} +\usage{ +ltx23_load_group(ckpt, keys, module, map_key = identity, verbose = TRUE, + gc_every = 50L) +} +\arguments{ +\item{ckpt}{An \code{ltx23_checkpoint}.} + +\item{keys}{Character vector of checkpoint keys to load (one group +from \code{\link{ltx23_split_keys}}).} + +\item{module}{A torch nn_module to populate.} + +\item{map_key}{Function mapping a checkpoint key to the module's +parameter/buffer name, or NA to skip the key deliberately.} + +\item{verbose}{Logical. Report progress and coverage.} + +\item{gc_every}{Integer. Run \code{gc()} after this many tensors.} +} +\value{ +Invisibly, a list with \code{unmapped} (checkpoint keys that + found no destination), \code{skipped} (keys the mapper declined), + and \code{unfilled} (module parameters/buffers never written). + A perfectly loaded group has zero \code{unmapped} and zero + \code{unfilled}. +} +\description{ +Reads tensors one at a time from an open checkpoint and copies them +into the matching parameters/buffers of \code{module}. Destination +names are derived by \code{map_key}; \code{$copy_()} handles any +dtype/device conversion, so the module may already live on its target +device in its target dtype. +} diff --git a/man/ltx23_open_checkpoint.Rd b/man/ltx23_open_checkpoint.Rd new file mode 100644 index 0000000..980ada2 --- /dev/null +++ b/man/ltx23_open_checkpoint.Rd @@ -0,0 +1,29 @@ +% tinyrox says don't edit this manually, but it can't stop you! +\name{ltx23_open_checkpoint} +\alias{ltx23_open_checkpoint} +\title{Open an LTX-2.3 checkpoint} +\usage{ +ltx23_open_checkpoint(path, require_version = "2.3") +} +\arguments{ +\item{path}{Path to the checkpoint .safetensors file.} + +\item{require_version}{Character. Required \code{model_version} prefix +(default "2.3"). Set to NULL to skip the check.} +} +\value{ +An object of class \code{ltx23_checkpoint}: a list with + \code{handle} (safetensors reader), \code{keys}, \code{version}, + \code{config} (parsed component configs, or NULL), and \code{path}. +} +\description{ +Opens a single-file LTX checkpoint lazily (header only), validates the +\code{model_version} metadata, and parses the embedded component +configuration. +} +\examples{ +\dontrun{ +ckpt <- ltx23_open_checkpoint("ltx-2.3-22b-distilled-1.1.safetensors") +str(ltx23_split_keys(ckpt$keys), max.level = 1) +} +} diff --git a/man/ltx23_split_keys.Rd b/man/ltx23_split_keys.Rd new file mode 100644 index 0000000..7fdbefe --- /dev/null +++ b/man/ltx23_split_keys.Rd @@ -0,0 +1,22 @@ +% tinyrox says don't edit this manually, but it can't stop you! +\name{ltx23_split_keys} +\alias{ltx23_split_keys} +\title{Split checkpoint keys by component} +\usage{ +ltx23_split_keys(keys) +} +\arguments{ +\item{keys}{Character vector of checkpoint tensor names.} +} +\value{ +Named list of character vectors: \code{dit}, + \code{connectors}, \code{vae}, \code{audio_vae}, \code{vocoder}, + and \code{other} (anything unrecognized; should be empty). +} +\description{ +Splits the flat key space of an LTX single-file checkpoint into its +component groups. Connector tensors live under the +\code{model.diffusion_model.} prefix alongside the transformer, plus a +top-level \code{text_embedding_projection.} group; both are routed to +the \code{connectors} component. +} From e02340dbb14aac774daacb332f154163f529273f Mon Sep 17 00:00:00 2001 From: TroyHernandez Date: Fri, 3 Jul 2026 11:03:16 -0500 Subject: [PATCH 04/83] Port LTX-2.3 rotary position embeddings from diffusers Split and interleaved RoPE application plus the audio/video coordinate and frequency embedder (float64 frequencies per checkpoint config). Parity-tested against fixtures generated from the diffusers reference via tools/gen_fixtures.sh (uv-run, CPU torch; not used at test time). --- NAMESPACE | 3 + R/rope_ltx23.R | 292 ++++++++++++++++++ inst/tinytest/fixtures/rope_ltx23.safetensors | Bin 0 -> 172224 bytes inst/tinytest/test_rope_ltx23.R | 89 ++++++ man/ltx23_apply_interleaved_rotary_emb.Rd | 20 ++ man/ltx23_apply_split_rotary_emb.Rd | 21 ++ man/ltx23_rotary_pos_embed.Rd | 47 +++ man/rope_ltx23.Rd | 12 + tools/gen_fixtures.sh | 30 ++ tools/gen_fixtures_rope.py | 109 +++++++ 10 files changed, 623 insertions(+) create mode 100644 R/rope_ltx23.R create mode 100644 inst/tinytest/fixtures/rope_ltx23.safetensors create mode 100644 inst/tinytest/test_rope_ltx23.R create mode 100644 man/ltx23_apply_interleaved_rotary_emb.Rd create mode 100644 man/ltx23_apply_split_rotary_emb.Rd create mode 100644 man/ltx23_rotary_pos_embed.Rd create mode 100644 man/rope_ltx23.Rd create mode 100755 tools/gen_fixtures.sh create mode 100644 tools/gen_fixtures_rope.py diff --git a/NAMESPACE b/NAMESPACE index f764ced..8cea91a 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -32,9 +32,12 @@ export(load_text_encoder2_weights) export(load_to_gpu) export(load_unet_sdxl_weights) export(load_unet_weights) +export(ltx23_apply_interleaved_rotary_emb) +export(ltx23_apply_split_rotary_emb) export(ltx23_census) export(ltx23_load_group) export(ltx23_open_checkpoint) +export(ltx23_rotary_pos_embed) export(ltx23_split_keys) export(models2devices) export(offload_to_cpu) diff --git a/R/rope_ltx23.R b/R/rope_ltx23.R new file mode 100644 index 0000000..5d77598 --- /dev/null +++ b/R/rope_ltx23.R @@ -0,0 +1,292 @@ +#' LTX-2.3 Rotary Positional Embeddings +#' +#' Fresh R port of the LTX rotary positional embedding scheme from the +#' diffusers reference implementation (Apache-2.0, +#' src/diffusers/models/transformers/transformer_ltx2.py). LTX 2.3 uses +#' the "split" RoPE layout everywhere; "interleaved" is kept for +#' completeness. Frequencies are computed in float64 per the checkpoint +#' config (frequencies_precision) and applied in float32. +#' +#' @name rope_ltx23 +NULL + +#' Apply interleaved rotary embeddings +#' +#' Rotates adjacent element pairs of the last dimension: +#' \code{out = x * cos + rotate_half(x) * sin} with pairs interleaved +#' (elements 1,2 form the first complex pair). +#' +#' @param x Tensor of shape [B, S, C]. +#' @param freqs List of two tensors (cos, sin), each [B, S, C]. +#' +#' @return Tensor with the same shape and dtype as \code{x}. +#' +#' @export +ltx23_apply_interleaved_rotary_emb <- function(x, freqs) { + cos <- freqs[[1]] + sin <- freqs[[2]] + + # Split last dim into (C/2, 2) pairs: real = even positions, imag = odd + pairs <- x$unflatten(3L, c(-1L, 2L)) + x_real <- pairs[, , , 1]# [B, S, C/2] + x_imag <- pairs[, , , 2] + x_rotated <- torch::torch_stack(list(-x_imag, x_real), dim = -1L)$flatten(start_dim = 3L) + + out <- x$to(dtype = torch::torch_float32()) * cos + + x_rotated$to(dtype = torch::torch_float32()) * sin + out$to(dtype = x$dtype) +} + +#' Apply split rotary embeddings +#' +#' Rotates element pairs formed by splitting the last dimension in half: +#' element i pairs with element i + d/2. The cos/sin tensors carry half +#' the head dimension. +#' +#' @param x Tensor of shape [B, H, T, D] (per-head layout), or [B, T, H*D] +#' which is reshaped per-head when \code{freqs} is 4D. +#' @param freqs List of two tensors (cos, sin), each [B, H, T, D/2]. +#' +#' @return Tensor with the same shape and dtype as \code{x}. +#' +#' @export +ltx23_apply_split_rotary_emb <- function(x, freqs) { + cos <- freqs[[1]] + sin <- freqs[[2]] + + x_dtype <- x$dtype + needs_reshape <- FALSE + if (x$ndim != 4L && cos$ndim == 4L) { + # cos is [B, H, T, r] -> reshape x [B, T, H*D] to [B, H, T, D] + b <- cos$shape[1] + h <- cos$shape[2] + t <- cos$shape[3] + x <- x$reshape(c(b, t, h, -1L))$transpose(2L, 3L) + needs_reshape <- TRUE + } + + d <- x$shape[length(x$shape)] + if (d %% 2L != 0L) { + stop("Expected the last dimension of x to be even for split rotary, got ", d) + } + r <- d %/% 2L + + # First/second halves of the last dim form the rotation pairs + x_first <- x$narrow(-1L, 1L, r)$to(dtype = torch::torch_float32()) + x_second <- x$narrow(-1L, r + 1L, r)$to(dtype = torch::torch_float32()) + + out_first <- x_first * cos - x_second * sin + out_second <- x_second * cos + x_first * sin + out <- torch::torch_cat(list(out_first, out_second), dim = -1L) + + if (needs_reshape) { + out <- out$transpose(2L, 3L)$reshape(c(b, t, -1L)) + } + + out$to(dtype = x_dtype) +} + +#' LTX-2.3 audio/video rotary position embedder +#' +#' Computes RoPE cos/sin frequency tensors from spatiotemporal patch +#' coordinates. Video coordinates are 3D (frames scaled to seconds via +#' fps, height, width in pixel space); audio coordinates are 1D +#' (seconds). Coordinates are patch boundaries [start, end); the midpoint +#' is used as the position. +#' +#' @param dim Integer. Rotary dimension (attention head dim x heads for +#' split type at model level; see reference). +#' @param patch_size,patch_size_t Integers. Spatial/temporal patch sizes. +#' @param base_num_frames,base_height,base_width Integers. Base grid the +#' coordinates are normalized against. +#' @param sampling_rate,hop_length Integers. Audio spectrogram params. +#' @param scale_factors Integer vector. VAE (time, height, width) scale +#' factors. +#' @param theta Numeric. RoPE theta. +#' @param causal_offset Integer. Temporal offset for the causal VAE +#' (first frame has stride 1). +#' @param modality "video" or "audio". +#' @param double_precision Logical. Compute base frequencies in float64. +#' @param rope_type "split" (LTX 2.3) or "interleaved". +#' @param num_attention_heads Integer. Needed for the split layout. +#' +#' @export +ltx23_rotary_pos_embed <- torch::nn_module( + "ltx23_rotary_pos_embed", + initialize = function( + dim, + patch_size = 1L, + patch_size_t = 1L, + base_num_frames = 20L, + base_height = 2048L, + base_width = 2048L, + sampling_rate = 16000L, + hop_length = 160L, + scale_factors = c(8L, 32L, 32L), + theta = 10000.0, + causal_offset = 1L, + modality = "video", + double_precision = TRUE, + rope_type = "split", + num_attention_heads = 32L + ) { + if (!rope_type %in% c("interleaved", "split")) { + stop("rope_type must be 'interleaved' or 'split', got: ", rope_type) + } + if (!modality %in% c("video", "audio")) { + stop("modality must be 'video' or 'audio', got: ", modality) + } + self$dim <- as.integer(dim) + self$patch_size <- as.integer(patch_size) + self$patch_size_t <- as.integer(patch_size_t) + self$base_num_frames <- base_num_frames + self$base_height <- base_height + self$base_width <- base_width + self$sampling_rate <- sampling_rate + self$hop_length <- hop_length + self$scale_factors <- as.integer(scale_factors) + self$theta <- theta + self$causal_offset <- causal_offset + self$modality <- modality + self$double_precision <- double_precision + self$rope_type <- rope_type + self$num_attention_heads <- as.integer(num_attention_heads) + }, + + # Patch boundaries [B, 3, num_patches, 2] in pixel/second space + prepare_video_coords = function(batch_size, num_frames, height, width, + device, fps = 24.0) { + f32 <- torch::torch_float32() + # torch_arange has an inclusive end; end = n - 1 matches Python's + # exclusive arange for integer grids + grid_f <- torch::torch_arange( + start = 0, end = num_frames - 1, step = self$patch_size_t, + dtype = f32, device = device + ) + grid_h <- torch::torch_arange( + start = 0, end = height - 1, step = self$patch_size, + dtype = f32, device = device + ) + grid_w <- torch::torch_arange( + start = 0, end = width - 1, step = self$patch_size, + dtype = f32, device = device + ) + grid <- torch::torch_meshgrid(list(grid_f, grid_h, grid_w), indexing = "ij") + grid <- torch::torch_stack(grid, dim = 1L)# [3, NF, NH, NW] + + patch_size <- c(self$patch_size_t, self$patch_size, self$patch_size) + patch_delta <- torch::torch_tensor(patch_size, dtype = grid$dtype, device = grid$device) + patch_ends <- grid + patch_delta$view(c(3L, 1L, 1L, 1L)) + + latent_coords <- torch::torch_stack(list(grid, patch_ends), dim = -1L)# [3,NF,NH,NW,2] + latent_coords <- latent_coords$flatten(start_dim = 2L, end_dim = 4L)# [3,N,2] + latent_coords <- latent_coords$unsqueeze(1L)$`repeat`(c(batch_size, 1L, 1L, 1L)) + + scale_tensor <- torch::torch_tensor(self$scale_factors, device = latent_coords$device) + pixel_coords <- latent_coords * scale_tensor$view(c(1L, -1L, 1L, 1L)) + + # First latent frame has temporal stride 1 in the causal VAE: shift and + # clamp so timestamps stay causal and non-negative + pixel_coords[, 1, , ] <- (pixel_coords[, 1, , ] + self$causal_offset - + self$scale_factors[1])$clamp(min = 0) + # Temporal coordinates in seconds + pixel_coords[, 1, , ] <- pixel_coords[, 1, , ] / fps + + pixel_coords + }, + + # Patch boundaries [B, 1, num_patches, 2] in seconds + prepare_audio_coords = function(batch_size, num_frames, device, shift = 0L) { + f32 <- torch::torch_float32() + grid_f <- torch::torch_arange( + start = shift, end = num_frames + shift - 1, step = self$patch_size_t, + dtype = f32, device = device + ) + + audio_scale_factor <- self$scale_factors[1] + grid_start_mel <- grid_f * audio_scale_factor + grid_start_mel <- (grid_start_mel + self$causal_offset - audio_scale_factor)$clamp(min = 0) + grid_start_s <- grid_start_mel * self$hop_length / self$sampling_rate + + grid_end_mel <- (grid_f + self$patch_size_t) * audio_scale_factor + grid_end_mel <- (grid_end_mel + self$causal_offset - audio_scale_factor)$clamp(min = 0) + grid_end_s <- grid_end_mel * self$hop_length / self$sampling_rate + + audio_coords <- torch::torch_stack(list(grid_start_s, grid_end_s), dim = -1L)# [N, 2] + audio_coords <- audio_coords$unsqueeze(1L)$expand(c(batch_size, -1L, -1L)) + audio_coords$unsqueeze(2L)# [B, 1, N, 2] + }, + + forward = function(coords, device = NULL) { + device <- device %||% coords$device + num_pos_dims <- coords$shape[2] + + # Patch boundaries [start, end) -> midpoint position + if (coords$ndim == 4L) { + chunks <- torch::torch_chunk(coords, 2L, dim = -1L) + coords <- ((chunks[[1]] + chunks[[2]]) / 2.0)$squeeze(-1L)# [B, dims, N] + } + + max_positions <- if (self$modality == "video") { + c(self$base_num_frames, self$base_height, self$base_width) + } else { + self$base_num_frames + } + # [B, dims, N] -> [B, N, dims], each dim normalized to its base size + grid <- torch::torch_stack( + lapply(seq_len(num_pos_dims), function(i) coords[, i, ] / max_positions[i]), + dim = -1L + )$to(device = device) + + num_rope_elems <- num_pos_dims * 2L + + freqs_dtype <- if (self$double_precision) torch::torch_float64() else torch::torch_float32() + pow_indices <- torch::torch_pow( + self$theta, + torch::torch_linspace( + start = 0.0, end = 1.0, steps = self$dim %/% num_rope_elems, + dtype = freqs_dtype, device = device + ) + ) + freqs <- (pow_indices * pi / 2.0)$to(dtype = torch::torch_float32()) + + # Outer product of normalized positions (mapped to [-1, 1]) and freqs: + # [B, N, dims, dim / num_rope_elems] + freqs <- (grid$unsqueeze(-1L) * 2 - 1) * freqs + freqs <- freqs$transpose(-1L, -2L)$flatten(start_dim = 3L)# [B, N, dim/2] + + if (self$rope_type == "interleaved") { + cos_freqs <- freqs$cos()$repeat_interleave(2L, dim = -1L) + sin_freqs <- freqs$sin()$repeat_interleave(2L, dim = -1L) + + pad <- self$dim %% num_rope_elems + if (pad != 0L) { + cos_padding <- torch::torch_ones_like(cos_freqs[, , 1:pad]) + sin_padding <- torch::torch_zeros_like(cos_freqs[, , 1:pad]) + cos_freqs <- torch::torch_cat(list(cos_padding, cos_freqs), dim = -1L) + sin_freqs <- torch::torch_cat(list(sin_padding, sin_freqs), dim = -1L) + } + } else { + expected_freqs <- self$dim %/% 2L + current_freqs <- freqs$shape[length(freqs$shape)] + pad_size <- expected_freqs - current_freqs + cos_freqs <- freqs$cos() + sin_freqs <- freqs$sin() + + if (pad_size != 0L) { + cos_padding <- torch::torch_ones_like(cos_freqs[, , 1:pad_size]) + sin_padding <- torch::torch_zeros_like(sin_freqs[, , 1:pad_size]) + cos_freqs <- torch::torch_cat(list(cos_padding, cos_freqs), dim = -1L) + sin_freqs <- torch::torch_cat(list(sin_padding, sin_freqs), dim = -1L) + } + + # Per-head layout for split application: [B, H, N, dim/(2H)] + b <- cos_freqs$shape[1] + t <- cos_freqs$shape[2] + cos_freqs <- cos_freqs$reshape(c(b, t, self$num_attention_heads, -1L))$transpose(2L, 3L) + sin_freqs <- sin_freqs$reshape(c(b, t, self$num_attention_heads, -1L))$transpose(2L, 3L) + } + + list(cos_freqs, sin_freqs) + } +) diff --git a/inst/tinytest/fixtures/rope_ltx23.safetensors b/inst/tinytest/fixtures/rope_ltx23.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ca61e4badb44637df2d3f61634f54d03e86ebe14 GIT binary patch literal 172224 zcmeFYc~p<@_y1d>G?z$9p@C9q)O=t2MO4UCqGTv4LNq9m2AUKNXcUz+(V$X>>b~|} z#?pYykz~%821K0K`~Cf%^ZlOn{S3cze*gYnYu)R$uC?y$&&!c8tK^zYtaT zKo9R7>o@M$;qMWkY7wI9v2(Ytr>cdjjgg_Mj%vVW_rFT6hB^j1vvmyD>Zp3S?{r_k z!^_!lL>cboT4qxJvRE&ezEM|=Op&@r8@|F=2+3Hmqo z{3Fa?1RVqY*`|M2&)*#5f7f2dIwm@1|I?zsDLMv*1_uA-obkVN?l0CqZ2iA5ItH^1 z%>K(k-+s7bp56}OVF>AyhSN~|x z{tePGG%_^)yO;mpxPLiXjQ?W&gX;gAqGN1e@*nfSf3W_S#vh;ekGjU+C>>L?*~Wi2 z!G8z-n|Amoz`s#C1_rZ@_5Yi@|AzI?-2D#>f1)umGyZ$3`49W}V`;sYfyw`CVDgLq zeFFWP_z(Ns-z8aj3A)W0Tsu?K2ae?1?TUu6Dz zLdMS`e?0;M5r03I2aEat|7-8byjpCLofnKH*@__Dy8-u<30OJb-i7If6X4JHO}Jxz zkyX~*Al!RQ0)+1Dz;<~N=Goz$IK^HF7_A^&7Hz?7K!3ay-cP~?_Mqzb)!@98;IYO2 zaIeq`#g;CGKhsq4z}@9w8I(nZv`is(ZyU9Ue?fk~lmjPuCD3w`qn&N4Y|-;@klpZ& zY!MSAvbjPy-)SnW2;Bo-fz6ES(lF4DxkY_f-yq^g!Z70UF5K`!0h8r-VWe6ZEKvx- zj+hy^`J*7+T^NckF7|j|%N^2xg~Om?C`gpOqtmJnk^8NCA?!~gkv)-U<;QKVb<WBVHzF;titgRzpQ-V32Tt)1?$3){#E`z z_4mJ5e?R$kxb1NO9Tu^Nmm(j@&yjGn^Nyh6)k~ow=^2#ZNrR!-cGwk89kGQo%N@xt z8V1RKdYK(<-q@eI6+TUU$PRQw(C;f2V@q!|@rfK_RKje@t0Qy3@lz;%+$u(8jTFHo zyPrra&#+<=Cy^C5HbT965cbGty1Z zEVY9d=Uq|d!k*gKvT|@fc{h~lWHZOCD=vN69|qj&E>gbL2PEI5nE#*p``@cSl?AO( zHOUktOZ#DhQxd*UkmM4V%c1p*r-TMLfw1pMvVLp{^r^~2`MOmk!Z3!N<1-n$OlHE+ zU^v~_Gy%-_pMnod-9dZm7`SW`iMhGYVdrdPPQvam^>5L`BBe-Jd*~@tGKEzBTP}Gg z8ppb4&Ooh&QP3wR%7xSUFz&i4-WJ(M-d_xXhX)Q4DR(na3Uz}Fw|h93C&it49FNPc zO@=Im@u=Bsj-yY1f^;!wG{~z0wVYO}aH*fVnJU7-0x4RcFa}?VJ)s?sw`1(tLOQIw z9jvz>rXNe4VT*i*)t=z{nAY+guZlM_k;OXbdbbY^?!Cv-01^_df~tt1WSJm>=PjCAV#xROp?{+n)yEyUHQi{M7& z0eZ7~0XhH25aH=zSbN!;xQQb-#VmBB# zWV7yVVqBd|7SQlF=iMS1xMl*9*fB*Etzvdoov?iYEmwKn$*7& z!OXVt48Bq3{qobIqt#B(U&m`;Z}U64C9DOvirB!&Ema)e8$knC8iC%-i!ju9085-^ zQiJ`Y;X#Zg{un4Bn~Ub+=?9ybjbR*`7QCX)X~OVlVhowI#}tB>8RGUIt02N-2((t0 z)1uQ)>B`+j5ZTsA1iBA1=XadL>4nm;YTZ$Ad20yshvm^aNs~_6^^3CNg0NCplFWH< zh~!`SOv1hy;nrSLDp}_Vef5q^(YUd=+vOJ7FzPm0F)~6IKJvjBS$i5xKNEAQgK#fk zm`X3NV3&j_LC|7vGI_lU{^61#dHpx`hi5CCx2#~=6Pxj7_Ix@lvl<0d8!%9H0(bJ_ zPRux@!7a7Ww#k^#XoUtWO-@hgTRFZs| z8pftLU&OAaMyy?mY%nYPluo#p;2w zU@-XZ$p)EK@vy8foOd=Yi&vy5#uH4IM)Gq#%x_p=*O}&TzT0ai;G2}%Krm4 zR2EXj_ZILn^%WI+dLLiixB@zdYjCn>09`)9;l%H8s4Y8}GxjpYlMx-*zhn&8uOA2c zYMJ#Hx*f6lNfYX3%|wOY^)UCeDi!EB%?QhFpruagWWl0Js((I$_J}Gx1$R+x_(g5Y{LvY~eMpj)g28X;fXyf{E_)7OWKFJ8ew4`U$R8pee zpxlHRnOsBT7EdF;q*Bn^<1IM68pBO_I31nl9)Z`xJX}~&hdmDYjNFB3NUIo>Fna?s z<9*?N_Z52Rs3zEs%mpWP4Sce0GZ*$zoJ(0BO9EC%BaW*Dopb4Q68D8(>OoBSGmf`J zwF72W7K4QU9JrO21IsQi0MljHptws3-+t5KhTSLQoP`_M3`p9&gSpzY z0H*znN9V#ANL_KAdUn>s#&5HzQt=pW@zX`1AoH7?Sgi=1H^*RSb`-=-9}8YOC+U${ zrB)-`*0UB@m*TcF(RA8k16)&a4Z0iTxakv2 zdh6z5YC;xkI;xR`HK;(H%thAX{xJyfbpxGEy&zhqjE{npki8?s1)jSM@*}0>$FzAQ z)PDwPxULPWFW#n)>qYQV`94y7Z4m{-$#7!!Ma-dhVLk#Xntc+2@@no}~ zAUm|+6xyh%;q<||ctGVd^&ZN^j5W&K@jJsb@b^u;d&wVKT_01@#Nw^o*HrE3M(B+> zioWUlaEXCDy~g`Z&24MR#}8%1U?>kH8?K<8Ml@vQS>xc<9y;=Qe7(S-eWa~+BlLV1 z;02%l$vphQA3K)gVR4l`-91KylUHcN2!m&^a{o1IdtpA9X>KO^4bHHq+y#1uUc=ly zb)cl3N}5|zQ0v10?TM>|$k64i>%b(`3_XIr%}Lniy$Gi^$>Mu4cV;$PBbQi%LFYYD zbYT+vNMnT5{1E58sZIf{5LM_5cuEBdsxemfGo0BTf&w;y7(e|5#Eol&u%T!aQK^9V zt3Gs+v?nZWA7FpTE`a=7( kiN;_p|a)xH7qaz_3|HR?Wu@e%NAi!V1rff4pW@5 zWjm<6f6W-5SVIKd@-hFk6mMjfBlu~)BL`MWL+UIS(tjr%-X|U<(kYK%T)7gj#x;uQ z=AMD8v4SKlbQ+bH91p^2$3eMrA=JDPgwDs^BxlG8#~15k)$5NWCOHJ=T<#zeQ-pDL zfGBz0mxLw9iok4kDmzr21VL6I)Y|JHdA_5Q9`17hv2o|{-JV$(mbw~p%5Oko{3I;R z3?!YU!o1)q^KnZ_0}kf4(Vd5S=(F!9QEG?<2dm|@Q!N8?#>$YvLmJR-wT-SVXeJjG z#c8TmI2n|>Ob-a$M!lv&Y9l7XyYN;D8;)Hgju&J(9EIeIv?Ey|z8{|_O~NQJA%m-q zU|2v2OxF%aO)W(nu6KhcE|##>;1GE*>IwYRDuv3ylV~sShU|Tz3DZPopy|?$cy2*F zX0JRC>9I-pGW!NOBKwm{3#8EWgUjiO99ud*{vH$^)d7{KsURU!X%*qM1=*VQB>7@4 zti7=WlfMb^tS22OCq~zyM$A&W;H3cbP$-pH=iDbgjmt3ZvjKXH_X48;YbAB`>m|?#LMMt&LsF@GodHyR_FG(DO_6ozvb^{0(vIL*@2e>s$gcnt(4Kil;;D~BF zrk0%G_Jj$8(8^U1%hxmbs~;3R+5w46Lg)-kfrsmiVawDQ9DSvqC+K_!Qa(>0Y2l|} zFRRJ5Z92>P#=d379T;TeS~tQtO-H!C*oNmKY_o3;}LK?Mt9Fm4)t3OAS7@0YX z!Qe*$#D5p&O^aL&EmQQsIaZ=xpsSqZ+RfsL3XeexD+QiXNDTaVEs0iANLD@3!%L1X z@NCUocrM(81s@a9zs8qr-%s%k+sy24*W;BOF(mWye!=FMH_*1V1t(r8MX$w^al*B? z5Y%xK@1Ytl`)~u!HaXHqD$}^sF~iJzmlar4Darj&5#p^_EQFTi0R3Q5Zq>QK7%G)# z5+%v=^u&WJWXIFnzUtr0i&t$Je zIy&2ofe&$pyp{zTypIA~p}j(oTk0&v^KMIKdfin(Iyn+Q$ZLaNur%g*M3JmAeJC5X z2UqVDLa}U0#~asy%>C~~==f^dEf~uhjFhv&Uxj)0%lvrbGkEz0pO)d_~#nr#EPI zpA__X#bJ5hK@`9C7N2lmAnl_w_rvg@RisWLo91?&UaVNbb=Yq zaZsmqvFdOmZxBbS=5u>u#-RLa71VxI&W_nNLPBh&(j)TO=$xRygU@0-zf>L>!-YcO z!d$w2`(j*rNQoOHwfJj~6|N2jP@fYCU8UMwgLMIlPw1xWbX!r!a||zg&lz}UnvbYH zOy%AJ@AvUjoN2HO=2<*v`qQr9k%+mRQcHZjQ13xp?(G9QU3ql8Q2^2!6U24{MmD&D z8@p&Mw)-hV!mG8ge)0+87_^B@{OLtHr4`^4>qCU!8G}S%KFW4R;&%03l=*TNvXhU4 zljjAZ8Cr~Yz6qo2ju&ikToPvd@+MDi^fOVp`+47tE|bkVM{!7`7%x?~k#|WCaA{;L z=j@gQ!CSh?)HUlNX4y1)t8qS5%8$XjTs<5ZD5HmC0`R_NJ%mV9;DU!eR`CIYU>cZ9 zu&020*sj2>`j7xZQ*`MqhYF(h$fVvl-V7#gaA6*=Q6Lgx5?KCTmp6u;hV9=p;A#3U z_UUZu1^xYZ_ z`zQZ`E4Iz}Hp?5{`Kt076En#QiJSD=GQusddPF3&4{@YOoUT5% z7w7!>!tQ#bgFjF1gX%_i+$9ICt*P@NAxvG9h4GG`aolBf zdTW^&7czG--%oGi_CAjNJW~Azs?k9?TnGLL%|e*%}^ucdv{9UxYr zk6y`mK&4LPGtqklIQtK(C~?)4YoEQCo-vp}^v=#@^4>jzn*~DnhpqxI=jj-(;|u~i zQ%JIon#S7Wqc#YxT6!;SqD?{u?OCm*=_Mxdgc@=3~RoEjX&{C}evcr!Cti!6qq7 z+;-B97N+PCxe+_CwQ%B%Pq+kK1BtNcNC4X|^@v5UTQo;{9Qm{EE`B*BNatNqqjyDL zQ8R@j^jYsqYLc!?f-8d2viTR4-0KP}%nGTT)HY1c1U#2u%Av+O+`GmG)$eV<)fyGd zimXk%(c-m4WK0SkbbQBFafVo+{Rbj*I?#Tl7`|Iyz#KoMPL(Sz5&4lo=r~u$2=<7B zb(JU8I;jdXiq!!Rj%AlGlOWSa`NEFSwcPdT)--ZbKXyKM#5Z#|Qt?s*-ScFD@OAyG zod_G2_JgX{7i#3F1A*=*VM|>q>C$$>{%_yN_wxXYpJ&m1cCGZMLKmu)jWDMx<#Cnv zXuPE$S?{zz4$6P&!$gw|ay{iG?6H=D(_b^VNl|vdHW?6S)iL0#AP8Pnt(cK5%@fUx zgPS3uyx6?MAn}aQ9V7%wMWkp%of@|VmB~nkDlfJ14K1H>7OU+qGV>>JG+;yrln0&= zX7U%5er?E$N}obrX-ASbl6ok8v;yj;JceU4rRZ#D154G(<#5h077KgAp?C|!o8jpI zm3!}zopT!&4Q-JWpUP|M(NjOpDrZdnoNSp6jU-2FH`;^j4HC+H8A$y0i?XdXF(5yDYF+^t6?vWIFB|QpDCWOZZ_v3Zn7`=*$g& zNafs@=rnYJD_dvHYNZ#{f7-0b3lDE#Pd8sCHd_=Z{IuW#m$*RAnA@;Vd0M@~(WPXY zh69AVOQ3~EJ7)1U+(6l}T>Qgo+F21pEZjP&m46m4{-K9U$#a5J%1DbEM@I$MLzKx{ zQs8_Pysj)mb)6^@k|~9?lX_wCY&jxv0YIyquN`~G;&9kFp1p(@L{6M)c_?lae8_Aj zR~IH?ykQevpku|Gm}CW6`$uuV#NUzUcdOYfi7B+Bxeyu;zh*m3Yhbm*5WQ_v2Z!4~ zFzWpFP`&Y-{R+9b!fb>}thA&j7v+$+0865JZUub#RmN&}oW`Fg^2q$mPB<4=O80G4 zr(?Fy#xJM+v0KWU`*n2^KGT~3uV*geJ&RD~^mIx|z*kWVVL4CixGqWs7sry49e3ba zpC9vJ1w%a-=1_W;p)*ekV#w~r-1>wXhNtL>!oJqr(DH7$`==02-CRQJl|PZCN5t_# z%?s?8dcaK4PT{=^n!r7orN>EBgkt=giBzYT2ghzTLspav&gnA5@$wEdX4n=d^y$IM zOgm68NW}4NHe9iLD2mKf#8p#Gv0~|G`cO*}j`V8rQg+6|@Je~wtLuRpel}#?$Dd$# zfWg}j)?)p$<5bT`6qkr!pl#KBQBCMFhCW?}_D2pwX0#hQmfu149&56+Fqg$$!Lw+& zNETH%pO3CRm2du_3N-+{}2|?bkG@>0X9iO zJT(z5$ZJi&*pO5zQT2{wO+88RSO+~X-b_5tC4<@)N4PxAAFOAo(4p@wcs@OV-LDgd zQ{Sh+l~HZbbv%t;*c%U5e{6t90c$Wk;~;(hYY(Kx8_~_%f3go-7|=7XA?AOt1O{SLN(1IRQ$nCoqDafcBuP~JaY!z-##u^R;XBt(CdEaNP75I-e`6t?ug{|I;)$5i z?*wbOEtFi%#)UfJWRuDfbhfaASW5$|_a2wYhxiV>U~`#%pWMk@?rLX`3!PysY)pV8 zc%k`d56Jl4PpSW;eiwLpenx|PRJE8-^-U0{|Tpwk<3mM+0+DSj-R1M_W+uo-%Q2FMi8G1DRA%O zJ61K-i=7?k0)}d9aQu?VNTd$HZr|f@`q+HnT2EoXp{qDwMG+$t^NF@yFw3^3;>-O8 zoUOw^{gR?Bc>nqr67Bj0me-DfNq-!f8kNUncCtT21Uw@GRt_}(bUnsZ3Bok{L^3#1 z%HR8YqRZkI9KV~tKJmga%%qCWdAp9>UY!hGG91pH6GXcXf2aBTV%d$;H{td!btrb3 z3lr@HIq3sB7`l8bZcxtxU8hnkar;KQes)52lq|*`w&1$T;$f3hHgmo31k8MB!AlVR z4os~i=G-X6%C`rgKKUJe6;lqZpc401;|#l~zk^xO@rn6ytgU|g&jvEpXAg-c9(X77 z0?2+ILSvh8xalTe!w#)y&-m|z7HftmP5;dP%GyGEk}Igw%{X+kkienM#gxq5!PxUP zj4{pgA;|YEHC>qv&5hxdb@n7Xh4k@7%P@U@s)Y>I8^TFHC#E&&C=sa(V4UVW1?z*U zWG!D$j8Sz)?}TFdJN6yB&dnKPIx-3EybVz?afm_rWOzaf5t&_0yc~pimdiv?qoD^Q ze6KN%6Y5Fa{IA5&UmnJLT9Mc1R^d735jM>@2rYIZ{A zstgzVtpIi0i-_PHC2)B5p8l|R$Caa4Og$L_D#hYVeoiXHNNs}GY$uM=V3@5-IA-v8 z8UDB!i>B-O{UlHv9zJP={5;Mo-RuatU07%}(&7UqQQP=)&JkFhZj5iP9)(3Q&+zrb z*|2icEmmm5E_|(Z9?QDyFk#hXxZv$dDqkDY>o>YlHR?P>&JluH+HEvZW&|!geQc$t z6hVU~*RdWv5iGC&NQHJzrSn#`pu*HEXk*|CCl);=VjqIYF%dU*|8{Feddga?&^m%D z3s&HbCB!jm^&9&u6+dCHV~*W5oU+W zV?C=k z;;^J#8MlS*#}eVIOs0mZR2?$x< zOZL?6fKbXYOHk)FPIqc8Aa>`Nc#!))0b|oZ#EZL#S>e3q3L` zz{oC;OPuqO`P};(^d%x8By22CWBd-1b~+P&Y35;mvmfp*H>F>$L5hpk>37B5&aSRN~arv$4T$JJj$V(cou8v~kM-VHTOv7tOzjONlhN zTD-=aSxTt6LmnmTD(H;E7WiN%;%q%DFnAe91s}X7*%>pK^}iEX1-=IJIAVZwrBGDa zSAdqxVI<^+X8okx8z&0LL&@AzV4?FJowt{h%7$yn@_w(u8>&C zHMmwJA9$ss$Q!-Mpm?SLT~Za{gJ2SP8QddfmvlgBqZe_{--(r4wpbEp0H#{c=*;~g zxOdcMTw;C_Mix|(%{Bs@S#GsuMa6q!@G%Xpy6nUGO)BVbUW-fku~P3@hWPB85Nx$G zhmH{*O&vV}PqlPWr3d+V|J@Kcjh#*3HmkwVkUqSdI0j^9%|WxrW7!A%_}I{nYFb!4 z3C{-kunU%DkcQv4V6``*iGL=gMt7-eq#e3QH=ya8a&*5D1S>CSF!`N(iKWkL=33ua zti8SkrkjrDUANs2r4rw9w5bGc7qg@)?%C*{twSHL)`yz3yUgugLo8Y?fX(rG^gLX} z_cEvHw}lLPIW*wIViRItWJ-B`Qdqaa9et$S@to~7D0mWL)uY!_pXKq2hRyl{<_;bx z_olCWK=Qdg3|A)|Z5tuU^&$K)oi^5ON;l*)Z;oz8Z7_oD~s*?8@QCr5} z!ySRL8dp4})K3rcwe_jnN@!g4Gd!u1O7@EN($G7BpqzP+OyKLLe#_osm)$Tu>w1cO zR&;=p@Hq0~^H_+f`ib>lE6J&hD|Fr4aIpG(hwk3^3R<_$!}hI37~)%u2Dw5s(7Fgp zK3NkmuYl?u`tYW^o7m5q&)T0#f*1D;c{82M*x!qG)2|n+Q6(S-S zJit8s0=j9s;cc8`)v<9Js0prxm)b`l+aQO`8hpeSW-r1(zGm%~?+VZ8Lu{L#LMFZ) zO)3HxgUz)io}x|f_JHfN0Vev%9(JRdAF9}00ILdNqMT_#O}*;~ zd8vaRm*&&PYsYZ;gdE0ct1;QCKORaCU4z;$rHt&9qjY|R7Fu+A)T>$~(&efP@a_Hg zq@rJje-1wiZd)gD)eVhAdvYY+FSdu0i|b%$e-bmetcoy6^-C5h#|CDXoIY`R93ozzg zB{5xjyYAo>j=E^b^7^lZgL6p+)h?cc(dEnO(Ff7WTBACQF1Outm)PuNC)EJ&6KLxf}-~2M^$lyx+LRJcBs-b&=o=>nz`m zi$r#tClT8;2VY5?g)P;ms1N6jMa6^gabpKM?G)v8%4nfg#9L;h)19s{H-X5P<>bxE zo1m3jMSXVm(xHndFjZ+1ewv<4gI z?R)SdO&ndOh;gG+@1oj!E&OO#N2YtU&|52hli=KmTtdAxn%gU2nP(pGbPHh2?ljEV zl*^Xw3dYQ!dib1H4TelHvb&3M_Oe{;Z*9hoPF3#Av9oahyeP`Qu!7=x7u4yJVN<0|zt4ulPd9k74YB(7*ApRf{Npu{Bsd}_z=<3$X% z@}`rBv;K6#!BHUk;2unBz6q)m?@^-_pTL8khcOtv0d6NcVaTIMsIgp!!bAMCU3>v~ z=9dgIx>>?vleR`tsI8GVFGZ z8c*L!leXWYjW^eUzg#RziJn2524m3qa-J;il!uqbTOsD~OqjH7FL^38mbjJwVBTLJ zAc5`)1U|{*w;EY^61@Vwj6H`j7G1#mDAyjhB=qzdyPuDnV|XaHf(9VKzlqL7`cxIVDY9GX73-w(H2$s%~cFu zH*}G-jk27AoG5L6w1YLfEI`f#O5*T=3?{2ffkwGWqKG51xu+s=s$4sOBjS+bxf0V+O8!p)#&cyRSNR)hyZzGO3&ebk1V?m_J2_leA!k9YCnV{5F9 zJ^{fskH{XAn0jXV6ZTzL5gvM{2^GT0?AW+NWN$$QH5?vb#qRR8c%$QZbtHv^D4k*l zhKfNUx)F;E)98si!n|DO2tWAwowzlMt>5T(Y3Z|BAlCVa*}L`}c|Php zqtKE;qNkODpG+MnLM;5cwhZb99k8Ly16EDAMth!}r+MR2aE3$=Hu

it*Q&s6z|U zS(1|2{S9!=!~rj)x=7Pf9`e6fYi)tShvs~-u(VWY&{sN-W`QW!R_o_S06Ok z<4aD5)G)t=s_5P|F0f$964H=yg_tCtfYI|$fU2!A2p;rewa-#)3~j`qTZN>(tQI#* z_rr+hboPr;8r}K63dWwF=sxEsY=Fc6T{)ERyykY z1VP011dIt;h_SY7=>yL{7H%CY>7=?$c;2rAiXZy0?xzhYlxl^!jZygfG~k|FSLm{@ zsvmHLF)toa>3pW|M(w=&=p0* z`TBkKd!!0aw?YmK9>|kQ^v}c0Ra3sxGcuVRozIcz*+N`; z+9&pUY^LS)MlCvt_W>|2f5MxhXh1_RTfS>!vw3z0RapG&3S^V?* zdgr(JQOp> z)RO4lok3)^;CDj%qjBQkcu*;Kqq}-*i00IAqQ-uL!S%;rPCz#*T-3rBOY*3~4G-My zdKR;!x1(xNA2fX~g3fI${&=PU@B0mS+uoc&$*AXSMD{tVZg_+Bb`vnULmPf6 z9mDHgr!mxc0R$5tdJ09CD+AJgYG}5R_R9ulI0*h*8yeTmeYcJM@Z0; z^|-j*n3(ySfl0$8s{kRe@?EaR+oz_$O*v@{JHE%GME4z5WAG$xbTon&rzLoj8Z)@_ zQ+3#NI>jiY=YqRWKCavH^cgisJP1C{`dq5!SE9Fg2HAV=0a@n11R5^s;gsGGxGvO7 zEvMF!Yfh`cHdF%VmlTm1z6BWfAf3D#`U2f|B;oSS)A;n523T#&$MSUsxWA+a+QN^J zi<2v{-{vK3{wB@Sc@s^q8P10n2}g*~WG5WFd4Tss>k0~QxPnzRpWy409VB7&M-pE6 z6c#P80U5txa9W}VLmvg$R{lO)a)Uj)HzyqI2S#(ZUDBAA1(mqI!xoA(Jb1EOS$Mv8 z992214B7WalLKoR3a5RDv7h!J@YO-3Ww_rnplrwTEHr6(z`V>UKF zf5SbOnM2oJ8V%~YeHi*;5Y(=Gp%Ix*bR#6A-Wmh$y16KgI6a@6oNx#}+)?7C4s3+4 z3tZ`;HFIF9yfymdBRl%iduq-<1Mqglk=#f*lrvgLT{@=I=%P-1|8WQBGo=+5-aSkg zEw6z5x#IZl$#F)davgk;_u!Ae5-i?$iaFjZPPdxnSaQ$2VaqsYy7a3Iz8Mtc_U?(a z>h%tz!ha;WPc_AGE!32!vVgDA|NIR;pIzaOM+v-5`@o7{;oosl&7pxygSj<171VEC z5O?>g8)>!A1M=Yv^xsUQb9!{aR{L|Ecc>jF*<{Dlar<7Mqde6rZ9sr~*C!3WI%{dr zL~oKMWD0$ylHBq9O3No#y1{s296WfNfV*Cvq3YKwsoLsY#JDd9WF5mW{eVBuZC4#N zxc`yloQcE@-44V^=Qz`SK95GM)glv)c91_COWCWO0Pk7vby(uOp8dhk4fXXn$#pdCnrX#LGWY)y!4iZi~O}p(^ZTp7i?vINOW5{Y&=C;6BBSv zU^V$Vc^yr9c7!sctT5o&MEY%Z8;s9>fjI^5sIkWasC_OEF0P_j-jGZ;r-ssmJ3-WZ znkMhL{DHdOP%XTY?Z$<0V_;BQ5W~G0kkxoiqOVub!9shcF1(eP%@866saMIkecFuV zu@G+R_WgAm1=rK}O6e%EvWf2KISe?>0b#5O8zDLqW1ik($d^{|=CAjrNrF7bBR9!2 zWp!Rj^*Ykh>4u zPGgJYBqlE=2bBWv5}P&ODU4deq+%iAqE>;C?m`}YD~6BUzhQK06i7Xu0gl>9>?g~) z5c9^2C68L~Bunap@F&!@7J;WuX>(ci(m%{0q zx6I4-S|mWOtslQ4M9Twamdd3ay*2g*c^ zNsd}0-F|d79(=>nxBW^OXp;yQ^fI(2U7~@3M`7H&A~xjaL)3SdX74xtr1LIk!;9Y$ zD0WE)UtBs%rY!L-e_%C`Tq^*LY~I z2_)XDpTp6RY22^r3Ghnk1ao^+2)jYRnw$D=1zyQNPTh-c;fV=ZcyBodr{-SS5F<^neHKP-GkuXT}AHGvN(_}UQsV>GL0(VizR1PrqOfqePqG++0grJ z7d2#N;d<4jASe5Tb$1+tVdt09v>W3%kN#CWRhw0)VpvUzVj`FTtyJ38rb{2}8lm>? zDY%WV8R+U&v)+-Ju%$rKb}(D^1man9=noV#ftM9^bobWsofGCd7%UwXi$iQ{1MyJXxe z7>T0eL{aT?Kh=Kiz*S^tqUjki64)3B@Sqry{&bLGbz96-46a{qsShOCl_{}v0L5`1 zuuaXEBBuUs zW?mGw@^ir896uI4zky9nI1Ub*v*De=H?nEpIpQl>hU;&w=NXkNu^)L_j3cYhZRnm$ zLUA0B7hCC(iKpRfhyoNXACIeE93<(rCb(pMHd*U@8e*4ae{5G+Roq*5RJYv3% zJwb)ZT3EFG05+Z8hhBbRC_T-B_iMu?SWzd9Lks<}zmLKFu2YDdLlm7P#zSfKYXwMN@ z-qHPb_)Ia5N?pDIS2PdP=Vez|H#h)ohbhE-c84=lcGA~>#QFE#f?1t55q#}9iW^)m z3_M>$^qWx4UiUvlZ+I7grQ9VHntYPJJ@%TZeOZS^Tqkxtk-!hbW5I803Fhf$gWCF0 z+!)Uqu)U;1*6Sa^-|k8fu*eBzT?gsn=uBczY6fu*Di~iO4sTw~Ly3?)=K0LIXgl2x zgp9k$?v^B2v0D(f9y*K`3UhJ2fiBZ_S|69alg6Vqj&MRpmUnEp2OV-l(KbdEc1uQr zgmNHc)f~5K96EsgAv~Nkb26{!;Q(HooI{L<>ank9fKJl?!v5Yo4eN$aqWHHos-d|_yS8}uK z?=^?v^Ld$6a?Lyt|F{F=rcdC8-`HZVks~ycMg-0(-k@C{hOo|Bnz0Z0Mu$EhM}=2qbe_x? zOXWHzcA0M~1nqhX^E3E)ey+ZBD^rSl$!OHq(MPL5e!LqlAY(L+=!{W?Db_JKk>lr} zOg)S{N@ZxXtp+BJ+$N2-MP%X$C!F^CCk+bJgcsJL5Hda+@+@8Gp_EF{&~n5l5jEsm zeG(ltX*Zqdnnv5cOvSTPb=el{c)Eapht7KU7-qy)0`o5aq8+|wydJMqED!w-)B94; zczp|A+H{7wIbjSG>x@rXzCjggH;MI!2@RgeE$AdN(Z2up=6aU%-bFhEC0n1?MUJoEEcU;x<=- zTndUOPi)dr$2AHcEFGbTr+ea_nz1nWeFMJgS_GGiH9_iCI-TCKom|*iif7OBwO~at z47pU#s2^JmStJTC2TTC@Y5QRc|Nel(lzMWfwUgX^D9CfEnTNfD$MLP&TQ=zH38K^S zoVpyd#SP_i@aQYPR%6hNd7)|$yZ9M(Toi$$6Bz2Fp?aI2f;Qi6K7B2LGsb zly%sIvdvZSeQ*o(jTZt1{Z;t1DjE-uk%Ev>0r=!`G<0Mo<1#BF?sdy4SZwP~4z4aF zR`-fQt6&Hm*HhS(>II#gFn6z17Y>if#+Xg|Y^`T2emTfPSJ4g_`g((Uoj!-#S6bk0 z6KTx7GY@n0tijLp4n+H?(U)g_(}%LpNzeGXxJ)=6<_Ru_qbd);rS=r)^%cMY+sRPz z@C#pOd zo-imdcSrx1lW^n2Q+U2pgr_W;jgQ)Y&?R5I@ob1D{%p=+Hf0He4ZCHlF zTdXkm#Q(+Kdq-DR*L~Xq3=Bo-NN)lTodKpKljQ6Gf{4<)ROt_0>2&}p15y;EN{2xZ z7#NtAOmcQW5Tz+qq=={}C`F_%}c*lyms}avAC)Q9w+mrY;WzvHBho~?~vf01JDO;;|jANA$F z>qd5#n=iOt`wfTPUf=ttx0cU5yI6T`zYlKRe6HfhXWA-#fLV{~4;z;=uOb)E+EH*IKSWtG4$0*B9S>?AGpgzW3Am8{@jml`ej) zcZt20EPFfOsrCJ}wRrOOejl&@K>h4hh86cby=BkCSM~qCYTB^cCKEO)zOmZ5#fZ5k z)*jrlrI_=^$7>spc)z&k^izv9o}X40*Ziow;oI}om)icZV)3hn7xNsoNb%+}7Z$yX zUeT+jI9 z-e>ohCr#U+`2P4!ihJ*xt@ETQ3wN}>|6a#O#TMn97i?Gm>Q4?We|6(S<>sn&o_OQ5;?hm$FI&FV|2wemQN?P@|GBvD?+2C3?R|97 zdERaH?Z04&eDjKq+RG1)FV1NH zS#juL|ETT#!>!5>rkz$^v)`xnN3UAE95Ux}wbt8Ss{Ql#i`V!3?xKD3{qmvuo|~Uj z?DqDb%j<{TQyg^2)wK;49n$}KhyFF(dsPkHI@UaPGza*uM`;eY74?)k~}pYJiFSop4Mi=BpzDUa%R^*-CqQT%p| zd5eGEw_x$(6J{&^^5EXJt`pWN_uqKIV%dN6m0PU-Xu03kz2*GdzEHbmt|9eTTQ)5Y zy7A%i%yBRFZngNB;)tzQ=s4tSKP!H7+KIKhSNyc>y5hq256```ymI6U#S@3E(l_mj z`+7F){HS+{wxYc5!Dow|I}ho&ss4itudTQ2GPM3cf9D_mcEjEmwww%Jar9oBtV zvHbRsl7Gsn&RoJ z7U_F&`ia?dw*ELVEy54GLD zI=;5}x4%-}ztb%pTi?7_arxt8YhT~$o5fQdhxC55;6lYGca18yy5ht7(d%yBadY3& z#b$lKEx!4K+k1yS+W)-Zwcjr;9pX7dVz1wCRc}9Xonp6n7pOn+)l18H`ae7K z#x*b3E*iUd`KKGcQ>^#*kLtggcY6J}OII)ET;j6co37uXzS35!*Z#gUI=ePVIx2e;Ih>)WCnKhL(sxE&vCpKJ8^ za^5R9EN{H=xAn7jJG%e#r3=?*`|dKe+MCPuPI~Nn#fIlTSe`uj#NzuCE-Z$PS*U*9 ziRaZH|NX}G<(}K5IAY;-idD|NrF?VQq2=upmMnhV|GC@+f4g+~`}I3|ulU*H<)owE zufOqwj`E0CudV&@?FDPE9C>;%cJ~#^ue>lv`OMT2#fh(fQa|&=9~YY+{aXFc_e?3y zY1z5BYPGNT{^DzE6nma>MzP;jBZ|)XZ!U*CvsA}PzrMKqV4Ej;@94a&c(G^0;&)%} z|5e#alghSld{{ee!&Ax~-<_-D$K`YNrFZQAjPhp{fFh97yYI<_qCn- z&(kCIC7)lqJZItA%RRSWt{nULyxpI#n!{$J~Dhh5zJ%T2yiu2WoCK7H@Y_2W+X zu;-R#?yDWK=a}9d=f1Rf<<48`uWtOc;>zRh?YQuk&hptiS1DIn@y7D5^FJ;gs2^SK zvdPMQBW@X0jDBhBVzI@SFCM@3nc8Bf-(LUQAb|xn$$oA8N~#Z+>lfZHvwO z|0cnHJJ)`7Q)~J5D@T;~o;PPP=P9q(hu<@{w%-T$*Z(^5%FYJETYQFr;}8;&h^Tky2nw|5y+AAZ{9#V+?g zSO51`rxq*p|NW4|zkX$L!TC#<_y3|H^v*T(jbekxhIbzI zPET!{srS~#-Se$ldC&LiyYKYV+R=yqwSM6Z|Ee8v;@67JzHxD}YwtPjSME4>@y%10 z>VIZ=z53hlEM1&_&F1B;$6Zj)wZo?UpUK>`yl1JG%MV^Vqki%MFV%}D=j;6DZ{Dph zdGC$oe#1M8(NE3WG26@6)Hhz~wEFB@UQw?2jU&sIKRv!iD*taL_EJb2>p`dXJB zP=4BZZSBg%wDPzfp0|KQAeM{oE7v?@b+A{Qc7P%U|#ELVed47py&S^^ZGV8M9z< z+=~6bo9I5L+;YVS>cz0F`q#hzGb0bL-*L%{(~D0pJEZoUYmIU*EC@+*>|4|NOO0X1}yt?y6zs zVf!E5{~6{BYQ1|cTt4>TdVQ_)k1o%ibFN~Gx1TLPx@m`E&DGASPaXD2$JllEEQj$*ARe^@)|;k7#-T<3+_3Y%YD-}~YFdmq{D{^H^(f9SpL_^0dlEH}Nl@$Hc% z&+G4d#Ri>Ywz;iXV4huyd0)D*cJ_DwQrqsI_ZCb1?S|rvmtXJgIpmJw%#Qc!-yMHa zdGb{g%OT&JtNoRSe^Z|^&**ZYPwuK8e(1fmeI6fIY|?r|vGsLF6<_}JiH;lEwkwW! zdCtxQF8@|>;J?qWt@B!~_VJZdY7?*R>S&$)#bUcJU0;5s^MTqn=iO1e;@<9Zg%M@> z=v{l&=e%*h+7s(t-Fy5FCl+U2dqXi{{6)2Sj^Dj_Y11vsTj!ist~~S~wK4w~R($gH ziRB*qeN_B$w?7r%+U>;hl%1|FU)y?-zFkM3-u>PYJNN%Bg(u79AHTBJzWJraY_H!` zyKLVF>T6CM-Fa!hqmSG3+rH=WEWL;7mlZ$GO4Jp6CnGdV*xZ2Z4pY~TO2r||V(aL?EF4Sekv+4D6+ zHv0TEUu55C&TIJZi|hxw@U^G#^-cDCP2a%Re33n0^F@05b<`fhYxwVr><7B=wWsj) zP4;|E-@w;=kv(7Yf7O5BJ$yfp!?9Kv*gE_7jsJL$Szq5+=UMAB>-*RLf8YQ5-7xF> z*Z=5IefCB7dhd(wXZn!;126L5|Fj?QCSP+3U)N;M*Ypj1%@^78H8|;u?x{P3*YMvL z*$;H#Yfs_po9y|TzJag#B745(|EmAMd-#4HXO^D_&g-o8`TzWWeMvOl@~$+JJX{mHXG$F@Jm&i>@~C(r)m z_9xH&`#{c$?Z>G?Z4g7xf*Lv|E+)a^ncC2IsMJ6{mtobUhQvA2lHxw zbNZWC`Jp0ql{*C#n%s;Ouw?BFIC$m3U_9wSLdG;r_KY8{iw?BFI=e_OUn6Kzh z&N1L8=NO)U_{i-~p8d(}PnP}3?N6Tl$?Z>`{W-S%8}k+Y$?Z>`{mJc5p8d($;3sFR z@{`%0Ec=t&pS;?CnfWHpoc}Z5n6Gi|RbaO z_NN=`Pu`fX%KGzqa{H5Ke=_@%Wq)$}lV^W&`;%vXa{H5Kf8N{vjrofH+Pmj6J&l@4H;mZa(Uv1q$=l_f{`5|M^q9)XWe1 z=W+j)ANgK|f38Pj3HeZhvz7n{#aYb8P!BF>R5?e39Fq-2TFpUnQ% z-2UYDCqMhAo6Vg6)xWf4WA^EP@*}U*nSXQoo6|pY`kT`~bNZXpA#?hh(?4_io72CV z(~b2frd)uG)wm_cZy>pSq>t9)BZ2$BApTGYuKbdQ0zSdcz>b(BD z{m-A6&++1MGmn4up4&E#Z-2VkpKdTz`;*(BJo}T`pDg>6+n+r9H(o!`pFI1M+n+r9 z^WOI7y|X_#$6$TQIYxE;$?Z>`{mJZ4mi-&AALvh>{mJc5p8Ywt{W*5_C$~R&_9wSL zdG;q~gP)uYe&?t0`hosr*`M70u#T1%rl{U6_T?t=L@r@uM<$(et1 z`kT|AocTAWgE<|@>2J>bo7111`8TJ(IsM6*e{#CnpKjTo-2UX*pUnPb*`M70HEAzw!Ek{^Z%8-2UX*pJUsf zV`qPI`;%vXa{H5Ke{weX$=Ry&(|G+rf3oaPZh!J>|A&|UeC_Ffz`uXrPItC5U;cK< z{F~F?oc`pjzd8NQ=}*r5o72IZ4&=1KbrWq)$}lV^W2 z`;%pVa{H5K|HkVF`jcmWa{H5Kf8N{vym$5|=NRyla}4-he{%biXMZyLlV$(L>j(Oi zXMb}0lV^X9ZGVoP{mJc5p8d(~PoDkB+2AKb){a^S+jeP%`=PJz?`pTuf=BF)oU!8o+ z8T-x5pL+knn)%fi99)vWR-8ODpZEDq3iC(5|9SqigEpC&Kl{e0e{%bi+n=2N9NYdJ+x{Gz{^a&2 zw}0ck`}>pIpWOcByazcO{N!x#le3Z8pUnPb^e4AJx&6uMPxr=l)3nCg(|^B-yESIN zI{$|b_Om75)F4Z**s4J~_wc^fzbyoqu!so73OaaH z9h`r2`kPnh-<Hius*5B9LpPc^Wtg*TM$>~qd`kULI zygL8p_9w5-KRNGhf8Lw^bhAIX{mD57x&6uQPu}>N{@ng#_9vr1ueCq9{mHBIZ$JB! zSLdIcW80r&)1UXYKe_!I@7>>@-2UYDC+9uL?N83e`6p*1vp<>r$*S{jKl_u@-}`5? zp8wgQjeC>af8_ZR?oC$jKe9gL+(R_4-hVXb{-b&I{-ZhfAI+=xPsw?2?w^u3?ypwv zpPJLbyn6rC-2S{i`;*z9Ec=t&pFI1M+n+r9b8P!J?yp9Fa{H5Ke{%biXMb`w_{rI- z^Gjxbvg}W8fAVU7pRb|+%;#$={mtobUhQvAfAea8b2^w;`yxQNK{^r@AZuX~J z_4y56Pi}wm>`!KYvg}W8fAZ{4Zh!LZPi}wm?9Y4KzcF92{^T43esYfC`G=3({^Z%8 z%>HEApWOcB*`M70+?mfKe_8)&0T+T*Wa9DJ3k!T`EkC;ogZ@Nr=eL^M zpUnQ%-2UYDC-?at`ulth{j1N{IREDK&z%0|^v|6B=5)xM{^s=0oc`wYujbA_x${rX z{Lgy+=kw=>Tz}4u>(Bi9e39!|x6&iwm&I+)X;`uv9TZ%%*mS(AQe z`Zwmc|7%!3@?3v%*Pkrczw!G1`SJC+{^ZP$IrGE$@%bx`?fh`;oF8)glV^W&-h<`!ig@@jvduc5!s*U+EmE0}+C`kT|g`uvIWZ%%)5=HJ)T!JH1& z=Qo^xbNZ9ddj9A0^=CF*f7UkFzcIi4U&H#5=lYYo{$#oSjo0_jkFU@5Cue@lnIF!N z&tGwD=Z9nG{E*w9Jo}UL9vq)M`;)VoXaC0Q2l|s`e{%bi+n@WdY(Brk{X3t(sowuJ z=l-2J_aAvJ_wUTPe`n79J96&PnRAcMoO^KO+`lvD{+)UC{;xUr@641a+zb2^aI-<e`j-2RQ(>Hiw%-`xJ>^mqQv z?N3gB=il7^gc7`}5w_`8T&eImck_U4LG0fAYrH49vf;w?7&Ed9CZu zYwb^7oqu0%fAZ@5lXGnQbL{H;o7=zf-u?ZV4g2$Y`;+q?`!KYvg-W% z{`M!Y&Ogsb(4EcaYpU~aPJeUO-}yJEzd8M#e{(vT)4};Sr@wi1{>|xcUY&n(y4jy@ z)%iEKe`9v~zsC7Dw?8@koqu!tlhfb%H@81|b^gumPhOpWa^Bniymxi}&FxRlF`R#M z`;#}mW?=r!?N3I3=il7^eGE{W*4Z{>|;*c<=uH&cC_+$$1aw-`xJ> zY|g*A{mH8HZ*G5b`g{M(J*Q8y(;fl@%u10?pt{8 zik}_b_#KhIC*tQwH~ziv?}Yf7(v5%nvvK`YH@?rYaUQB0KQsAr6X(TEZuiO0{Ytjv z=iVh-^7DH>w&dq`er(ClzpvPmpP!Z4lAoWO*^-~@kS+O{H@4(&Ts!^qC%60L=YA|( z@^f#NE&2I9CR_6JJ59Fa=Xa=V$=~?*NdLbFlHY&ZB)=X=ey%6Be96aU+}D?vtPA1=y0G zX9s+KfHly4{GOdH`T3nYTk`Y!4S&a7<>%k3Y%H{9-b?@MJ--Kd{`H>gAiv&g=70I!C%@~E z{H{atyAGaz`Q7V3`Q3Z|<#+G-m*0KzyAH|kIym?4oqxI0|8l4Q3{i~`CtC@zx?Tc z`P2XMr~lXD{+Hi5 z(Esu~2l`)r=Rp63_M?|8l4Q z3{jt|MI8*y3{b)f&{_d3x3@_QZVfBC%*^uBxjFTeZbcO8=7bx3~KA^Ba03_M?|8l4Q3;S+@5H9}-Sg}fo8F(f@4pYpy*|C~o_ogLH`e{^ zoqPAmum6)@|0loxPk#NM{Q5un^?&l~|K!*Iuvhu@fASCJ|K!*I$*=#DU;ih+{!f1W zpZxki`SpMD>;L4}|H(g?|C3)2B)=X=em#)pi~*$glU(|1-Vkec%EA z>Am#7-t&8abMM~sFTeZbcO8=7b@2Sl?>_lmhvau1lHYak{LAm&^Dn=9&%gZc<#wO^ z>3_M?|8l4Q3{jt|MI8*?m)|+i|MEKr`d@zMK>y3{9O!@fodf+ZzjL7f<#!JB zzx>XD-gmG6<#(U_u0!&>4$1F2B){vB{H{atyAH|kIwZgAko>NL-0qV<{V#X=U+(n3 z-06S0)Bkd(|K(2q%bot0JN++rGyi+u)Bp0P|K(5r%b)(2Km9L%`d|L^zx?Tc`P2XM zH}k*z>3{jt|MI8*3`=w{qNkT|DF5v zzjL4dcka{wGv_}1@7$;Vo%?3~mp}b4fBIkk^uPS+fBDn@@~8jhPyfrG{+B=fFMl)t z%kOod|K;~O(Esv#9q51gy$p=g@?{%R6<@Y+!`|kC> z{O*(Abx3~KA^Ba03;V7&8?Y#cdhdKfBt?(^>3)?fBx2mf3HOMv*+(t__s@RKYKnS?B|4aKYKp!$)@++ z^Vw6jnZNfR@IRk(VAK2V`OJ);o6-I3oqPAmum6)@|0loxPk#NM{Q5un^?&l~|K!*I z$*=#De=z?izy42t{h$2$Kl$~4^6UTP*Z;|{|C3+;C%^tr{=xj8{CXhy^+59Lf#lZ% z$*%{JUk@a|9!P#Yko_nUfAZ`9R1N?*a1bJ%9d}U+<;=^`74Y zy_f#idww5~U+<;=^`74YoO}14fBD@fzw40vu7l@ae)q}mIwZgA zko>NL=U;yJ$?rON{^fTsxBKKz|I3~JmplD0cluxM^uOHcf4S5Da;N|0PXEi@%>SPE z^uPS+fBDn@@~8jhPyfrG{+B=fFMs-9{`9~6&HOKa`d|L^zx?Tc`P2XMr~l3^^L^uO1A`rqq5{qJ?3{`b01|9jo1|Gn;;`CoqLK>y3{9O!@fodf+ZzjL7f<#!JB zzx>XD{+Hi5(Esu~2l`)r=Rp6Mg z^1BYn?>flsKKaxCa;N|0PXEiD{+B!bFL(N1?)1Oh>3_M?|8h6;zvn&uFMs-9{`9~6 z>3{jt|MI8*VMI|K(5r%b)(2Km9L% z`d|L^zx?Tc`P2XMr~l<|=70IU4)nkLUI+SLey;=lFTdA;{+HkDK>y3{b)f&{_d3x3 z@_QZVfBC%*^uPRG2YTPV{+Hi<^1BYn?>Z#E>yZ4eL-M;0$?rNOzw40vu0!&>4syFs z{`9}x>3_M?|8l4Q;L4} z|H-falVATQzy42t{h$2$Kl$~4@;CFp-b?@Mz4X7{OaJS=^uOLq|LeW%H{9 z-b?@My=MNG-@WHwe)pb#`Q3Z|<#+G-m*2hTUw-$VfBD^e{^fV?`Iq0l=U;yJUia>u zf4S5Da;N|0PXEiD{+B!bFL(N1?)1Oh>3_M?|8h6;zx?Tc`P2XMr~lVoFMs-9{`9~6 z>3{jt|MI8*puPO zb)WwCx=;Um-8b{U{LX>?m)|+i|MEKr`d@zMK>y3{9O!@fodf+ZzjL7f<#!JBzx>XD z{+Hi5(EIN7zx?i#-*rfS*CF{`hvau1lHYYme%B%SU5Dg%9g^R5klTIor~l49(_vwGl!JNM~-=f0W$3_MK`QP)N{+B=fFMs-9{`9~6>3{jt|MI8*_kR9G@4NT&FM8j-pMTN&?*06W-gockU-Z6vKmVflXYS+g-{^lo|DyNZJNNFB zU;ih+{!f1WpZxki`SpMD>;L4}|H-fa{rr#I`ak&x^MCT||K!*I$*=#DU;ih+{!f1W zpZxki`SpMD>;L2*%>T)+2a;b8B)=X=em#)d%Km9L%`d|L^zx?Tc`P2XMr~l4Bf zpZ=FW{V#v|U;gyJ{ON!B)Bp0P|K(5r%b)(2znTB#PyfrG{+B=fFMs-9{`9~6>3{jt z|MI8*l!JNM1}FMs-9{`9~6>3{jt|MI8*yZ4eL-M;0 z$?rNOzw40vu0!&>4$1F2B){t*xBKKz|I3~JmplD0cluxM^uOHcf4S5Da;N|0PXEi@ z%>SPE^uPS+fBDn@@~8jhPyfrG{+B=fFMs-9{`9~6gZbajf9QSpe*Q)8yZ7@idf&aD zf6@Ey{rrpGckkz4^uBvP|DyNZ`}vodzxSW@_kSAy`@eqv$M1Xk-_QT}eQz-T=lR$F zdH(f(o`3zH=U@Nl`Pct>{`G&JfBm25U;ih+{!jkF{Ga@KAo=w`^6P=**8|C~2a;b8 zB)=X=em#)peZ_+`ISu%kSRvFTZ=wzx?h!|MI){{LAm&^Dn=9&%gZc zJ^%8%_x#K6KKWe-x!os!`d{w!zuf76xzqo0r~l3{jt|MI8*3^^L^uO1={|?Z9UiayLulr{Hm)|+i|MEKr`d@zMK>y3{9O!@fodf+Z zzjL7f<#!JBzx>WY@;e9mUw-F6@4MIk^1Dxd*CF{`hvau1lHYYme%B%SU5Dg%9g^QU zNPgErZuiNb{+B!bFL(N1?)1Oh>3_M?|8l4Q3{jt|MI8*l!JNM~-=RW=K+^7GY`)2-^Km9L%`d|L^zx?Tc`P2XM zr~l4Bf?{%R6<@Y+!|MGhs=zsaW4)nkLUI+SLey;=lFTdA;{+HkD zK>y3{b)f&{_d3w~?)AU??vvkjNPgEL`CW(PcO8=7bx3~KA^Ba0zIZ2RuGKvCU3USm|r zw)e^3|Bvm+WzVg#{pp=?MRmJ#{?!Y%U4~p&uWoA`ePqG5NXJQKb=$q?xRPzVukKJ( zw@tVBxW;zY`m>kS?Tnp{DcI(G?H9G`_8;*5|DWVPWQ%bnTk=nz{+)s?`Ty|nwgp@A zzyHHp$(H=T_~^KjE%`@%IIdtz{@wrim7=i)|3M#(E7+3%(_?ln*^+OZ*pmPGi%u-ql7GI}k1W`d|Hq4MU9u(r z@Ski_vL*k4OKn}UCI4Bs|Dnc~{Nv|3sANn2SML5rjjb8~)ejt3vL%0c)glF3@?WsX zE(Kfi-?VaT!Iu2353EzLCI8X)?_97Y|ImM(P_QNcV~f1m%a;6Sj#;r}Oa4!5_tn{& z@ek`hx@1fK;b;D%#+LkZZMjszmi%iUbz+?@`JXvwSjm?BU0u7^*ph$vfeV&w$$$Sn zE0%1@|6bQ-b++X18h3EXHW3_M?|8l4Q3{jt|MI8*puPOb)WwCx=;Um z-KYP(?$iHX_s#q-zjL7f<#!JBzx>XD{+Hi5(Esu~2l`)r=Rp6oD-|ANk!Uzw40vu0u2a^uOHcf4S5D za;N|0PXEiD{+B!bFL(N1?)1Oh&HV3qPyfrG{+B=fFMs-9{`9~6>3{jt|MI8*l!JNM~-=RW=K+&A;T{ON!B)Bp0P|K(5r%b)(2Km9L%`d|L^zx?Tc`J4G) zey;=lFTdA;{+HkDK>y3{b)f&{_d3x3@_QZVfBC%*{Qf7u*Ma_*-|ImC%kOnCnE&N> zpZu;v^1BYn?>Z#E>yZ4eL-M;0$?rNOzw40vu0u2a^uOHcf4S5Da;N|0PXEiD{+B!b zFL(N1?)1Oh&HV3qPyfrG{+B=fFMs-9{`9~6>3{jt|MI8*Zzx?h!|MI){{LAm&^Dn=9&%gZcJ^%8%_x#K6-t#ZNd(Xf8?mhqV zyC3ZOmplD0cluxM^uOHcf4S5Da;N|0PXEiD{-4P`aQ`>?oB3b<^uPS+fBDn@@~8jh zPyfrG{+B=fFMs-9{`9}+y_x^zPyfrG{+B=fFMs-9{`9~6>3{jt|MI8*3@IjOaFV_H}k*z&Vl}y-#O6#@;e9mUw-F6|I6?lixWQ%>VMcPkz@S`CW(PcO8=7bx3~KA^Ba03{jt|MI8*m{`%n4ZH{(zL%bot0JN++r`d{w! zzuf76xzqo0r~lCq)BElpdSzU} zruW@nb?lNQo8EVS>9FGpHofnDw@*K-vFUyHIseJ8|C3+;C%^tre*K^P`ak*gfAZ`9 z;L4}|H-falVATQzy44D!Tg{6dLa4rK=SK> z3_YK z{?~iyf4!If*L&%Iy_f#id(HeWzkAQW{O&#f^1Ju^%kSRvFTZ=wzx?h!|MI){{LAm& z^Dn=9&%gZc2fO~|PXEiD{+B!bFL(N1?)1Oh>3_M?|8l4Q3{jt|MI8*3^^L^uO1A`rqq5{qJ?(%>VK`2l`)r=Rp63_MK`QP)N{+B=fFMs-9{`9~6>3{jt|MI8* zng8Wa|I45Lmp}b4fBIkk^uPS+fBDn@@~8jhPyfr`%>T}P`ro-v|2y~T zf9F2^@7$;Vo%{5^bD#cq?$iIyefry3{b)f&{_d3x3@_QZVfBC%*^uPRG2l`)r zuY_kR9G@4NT&FM8j-pMTN&?*06W-gockU-Z6vKmVfl-TV0$z3)Ef zKl$~4^6UTP*Z;|{|C3+;C%^tre*K^P`ak*gfASCJ|K!*I$*=#DU;ih+{!f1WpZxki z`SpMD>;L4}|H(g?|C3)2B)=X=em#)3{jt z|MI8*puPOb)WwCx=;Um-KYP(?$iHX_vwGH`}DupeKY^d?;PlV z`JDs(FTZo3|K)cM^uPShf&Q1@Ine*|I|qLMm)|+i|MEKr`d@zMU@-s7?>_lmhvau1 zlHYYme%B%SU5DhKxel}b{!im?Kzjbu|DON!zvsW1|2^;Nf6sgR-}9dS_q?b7J@4s% z&wKjc^Pc{fKm9L%`d|KL{+B=fFMs-9{`9~6>3{jt|MI8*(*OS4lm7STp7g&z_oV;*xhMVa&pqjX=RW=K+^7GY`)2-^Km9L%`d|L^zx?Tc z`P2XMr~l4Bf?{%R6<@Y+!|MGhs=zsaW4)nkLUI+SLey;=lFTdA; z-~Z(II`I3S{9Xrs|C8VAV6g9h^1Dxd*CF{`hvau1lHYYme%B%SU5Dg%9g^R5NPgF$ z8Grg;?)1Oh>3_M?|8l4QX7jKAlVATQzy42t{h$2$Kl$~4^6UTP*Z;|{|C4_(|L6JF|H-falVATQ zzy42t{h$2$Kl$~4^6UTP*Z;}i%>Q~X{jc}Z|9UU|ulLgbdN2L2_tO7*Fa59g(*Jrd z{jc|$`Coqbo`3n>d;aBj@A;SCz2{$k_nv?G-FyD!cklU^-@WHwe)pb#`P~n8{mY&H zmplD0cluxM^uOHcf4S5Da;N|0PXEv3p7r;C8vpw*dH$RE-}9gT_xz{-J^$%{&wu*g z^Pm3r{HOmt|LK3vfBN6^pZ<65oB3b<^uPS+fBDn@@~8jhPyfrG{+B=fFMs-9{`9~6 z&HV3mpZ@o{Pyc(}r~kd~)Bj%g>3^^L^uO1A`rqq5{qN6x>3@IjYvzCXodf+ZzjL7f z<#!JBzx>XD{+Hi5(Esu~2l`)r=Rp64BfpZ=FW{V#v|U;gyJ z{ON!B)Bp0P|K(5r%b)(2znTA?`}DtapZ<65)BnzW`ro-v|2y~Tf9F2^@7$;Vo%{5^ zbKlJW@~8jhPyfrG{+B=fFMs-9{`9~6>3{jt|MI8*4$1F2 zB){vB{H{at&s>LDfB&cP_uoDL&HV3qPyc(~)Bm3L^uOml{qK2C|9jrk|DN~szvn&u z?|D!E%RiX^^Y35ie?R}C_uc#X7rpP^&%fw>_kR9G@4NT&FM8j-pMTN&?*06W-gock zU-Z6vKmVfl-RJe6{Q5un^?&l~|K!*I$*=#DU;ih+{!f1WpZxki`3LiV^6UTP*Z;|{ z|C3+;C%^tre*K^P`ak*gfAZ`9w)Cg1Ie!k zl3x!rn}2`)Pkw*?Pkw*?Pkw*?Pkw*?Pkw*?Pkw*?Pkw*?Pkw*?PyWIDpXXoyC%^tr ze*K^P`ak*gfAZ`9%H{9-b?@Mz4X7{OaJS=^uOLq z|LeWd;aBj@A;SCz2{$k_nv?G-FyD!cR$$m zFL(N1?)1Oh>3_M?|8l4QpuPOb)WwCx=;Um-KYP( z?$iHX_vwGH`)2-^-#O6#@;e9mUw-F6|I6?m)|+i z|MEKrgZW>6_sQ=%B){vB{H{atyAH|kIwZgAko>Mg^1BYn?>aQ&PyfrE{+B!bFL(N1 z?)1Oh>3_M?|8l4QK~>3{jt|MI8*3`=w{qNkT z|DF5vzjL4dcka{w&V4ig%b)(2Km9L%`d|L^zx?Tc`P2XMr~ly3{b)f&{_d3x3@_QW&=70I!C%@~E z{H{atyAH|kIwZgAko>Mg^1BYn?>Z#E>(Gop{V#X=U+(n3-06S0)Bkd(|K(2q%bot0 zJN++rGyi+u)Bp0P|K(5r%b)(2Km9L%`d|L^zx?Tc`P2XMr~mo87rLI$y6AW3!yF%) z`;*(BJo}Tg!B5UsJy1x&6&Kw*5J_{rMT6{^a&2w|_OaKe_#@x&6u6;IHQPC$oPww?Dc4 z$$5Uw`RAE6=bz`)n16Hno6|pY`kT`~bNZXpA#?hh(?4_io72CV(+vi4y1_use9@oW z{^a(r=JqGEe>Jy1x&5oT{mJcL&FxQae{|f38Pi}v5et+Zq z^E(^opWo9k|K{{Jr+?=3H>ZE*^f#wN=JYqGf9CWzr++o48w})hgMpm+qCdI)$?adw z?N4U^YHojW`&V=OliRJy1x&6&Kw*5J_{hcpz`;*(hn%ke;{?*+620$+JJX{mHXG?`?nHJNuJ! z4Ei^^^&bO1*Pq<}`!ig^6XD;fAZ{4Zh!LZ z&wJaS_s;(090NXbjsc(RPi}wm>`!KYvg}W8fAZ{4Zh!LZ&#~>#v9mw9{mHXGx&6tr zKRFxx0aU-laMBE6uq_NzT1JbMEJva}SShd<{9>U?8U(3_cfNZh!CpliQ!n z{^ay0w?Dc4$>~pSe{%bi)1Tb_r$>>jRe{%bi)1Tb_0e{%bi+n=2GAZLT0oDF_* zHZuE@*`JL5e{=ftTKb#Q-<inD2-<+Rpj2KtlHpWOcB_9v%5x&6uQPhOpW-{1b^ z^yj_p&)V6a_pZ)AuP3)ZImh7j_HV5HK!0*JGW(O;pN#(G_9wSLd3FAMfBTcupJUsf zW80r&SLdJCliQ!1{=DA)jcx<|$$1ZQ`;#}mrvLnsvys`K%>HE6`S<-F z^S|4tpEbJEpNw;CPJeUylhfavH8iI`IUUXEXif)m`kT|=ygL8p^f#~0KRMlCAg5b( z{>|-A&gT4^+no7XtMhMee{%Zsdi(Qw`;+q?@65S>XI{O3XU_dQ^XmOObMC>JSMT4M zbN|k~djHOx`*-Hm`;X*wvp?OkKe_$Mvp<>r$+ADW{mHXGx&6trKe_$Mvp?@`f8IO$ zlXDFC$vH;#{E^$AJo}T`pDg>6+n+r9liQy>`*UpjbL{L-Zh!LZPi}wm>`%@HKRH{K zpUnPb*`M706 z+n+r9liQy>`*UpjbL{L-Zh!LZPi}wm>`%@HKRH{KpUnPb*`M706+n+r9liQy>`*UpjbL{L-Zh!LZ zPi}wm>`%@HKRH{KpUnPb*`M70?)sZM|E$0B&-*j~ygxbT$Q(yyjz7)qUwys( zIlld?x&6uPU(M}LZhvy0?_magzQ*S_xVL0Ze{-MTsOCPu!E5PXeLja{)8E(AA#?hh z(?4_io72CVJOAYRpB(?sdj9A0=Z9Q>&W-EO{Q7*6>rd|bS98~&-1Rr-*v=2fc7B{M za_5KK`Kjj454rPG&FxRl#`&%0_9wG{HMc*x{mGeOpReKm9nV*I|F`=53>o+D%(;J8 zeg1@;`*-Htza!^8JU;i}%((|g&iy-c?%$bn|Bigt^FN`%_d`;#}0 zQJEhy`;%pVa{H54`}=$i{e8ZM{?+GaoPTrXzxw=!^KVXna^~OT)4`k$)#o>OJ^jt; zPd@AUpUKe_8qp6gG}{CIrkhx5bybACCt^TV-o ze#q@lp7TR)fAZ{4&c^$bvsLGZ%>HEApWOcB)&4$TLw}#Ip?~%H88Z5t=kpun^f#wJ zuV?-}J{`>IPiu7H?%$bn|Blyk|IVEIcjnx`Bj+BSIrr$yxd%ti{X28+-Cf8PpZB&u z?_ItB%j?PQPtGxTz3WeIe{wc5*PqP(Wb`L@{mJc5UcLY8``e$K{v6x>9NYdJyL$hZ z*OS|yoc_Gt{^a&2=RL^nPtL~qC1)eEKbif>s`r0=fBTcupYCitAL0D_d`)%!&FODW ze_l&}bNZXppPY{74aR}zi|9a3e{=f#`s)0f)8Cx_&ObTbU?8Vkb^gumPtNB2o7>k+`jgwAygL8Bzx~PS&#~>#vF*>XtMkw6$?Z>0e_n6@Mz?|f+d@1u9jna&TG4B-RWab zowoJ57mi$I~}p>|1sGT~pc*o>ty* zZOh&h=Iy$xzIJO*`P$u&wRQF^*!7(eyquO^qfE9r(?cR9NPNt)~)KZwcpsfV(*dbZa8JL-hci4`Sll^aZ}g1 zt$R;fr1iSq_wQW2W9hDqTkfB_LH*}#d*1uImenTD)phHg2T$s28`eGXju+a$ddri; z`&L?S#*bP%N8i-CQtw{V&*y{l)D_Qgkif7-L0Rr#oZ5YB=VM(*x1QF%OUo6v4y||eUfS}n zNz*!}%xLL(uJx(4cWUpPyl(pYF>Q;D+Oh4J?$)XM z7Dx5|sjGAB%e`~dcb<6K*wv;^>zLZJOYa>M4{YnH{c^;wNBnxtM|viW9p3l)ov+uP znXyjmsogt|ynDv=ZHKP+_vs7Gc)I)jQBO{Pdip6{KkHa+#^Lp$lOOF~t9y=~ce)qp znbESq?Vn89dhHimp6uCY((s-wdtPigcFYNPOc{0h-7l_t?C8DwmTh0`?w9(0G-bllZ<2rU4 z{g1BYdqz(EtiEL5pW2Ta{Ygu0>_;=M8M8&}m)7}oT7A8%+FHjP(YfyAZ9B%dzBT&l z-X&&S(LK6l+sPNT-Z*mndURqcfGpdl%D6>&$)fc`Y9d1 zAAMDQSjR1`zn}iKsW-RY)jIpFn@-<$y*=)pz4fvQi;rA#+6QeHcTXBMVZAHX|G0DZ z;xDb=@13~%?Y)Qft$pQfYwq6rSzCM0x7!!%c&g{Pj+Hz9yxv1yQ@egV>IdEL_jb%U z>bf)5Drz50y!M*Q#?I5WQ}>x;-Ws_@=PQ%T(WBRYs=K3QzNug8TDs+;`e{?n@BQVd zKbBA2eNS(H-`-_b{doF?>;HPxvSUuUT{J9O$#CLZ7MNc%@? z&%W*-$8OR-q2-3re_!w7w#B+nE51JM;?{+FE^B|aZROUBJI?PaMh(CFAN6P2R-JTN z`(M|bFt&4ANBdb_H}-5b{f*IwuY2^Jmv;YQ%8)S^PTp_qCu{$xXYFjF(0YTYsC$ztmW zM}IVGhiRXVx}oLX_FcvdsognshmqIZ`Gc0XHu(Ltxu@UKGI#GGBfq!S3hm2xK0SKL zmNQ0evcaG3{9(tJr?%AoQh#sc>TCUW(#SQIZMm?gv-_M;3-z!6#dogUd(G5uPF-|D z*O(vIj_*Bo`iYaT7`x4sA)`ijpU~P{+ijg)yYARv?5LGm#&j$;b=}Fwj`()V>m6@S zZK+>;_rs$O?`dgSyJh;?8`nm!_vq@cl$&(@q&BIwqiy)uqgtLG^_$7N|NNQOU#@mS z%l@~2qwlB1*s-s5-!bNrjtT30*8b+$&3o7GIi>sE(Z_dveD^~wZ?#`EX`AkQr+%k> z-!Y4?w^6ZM*S_sjdY0;bxm;rGft|0k?p6PG?acCT9bcOK(&#ICmYH!-`O?InjJmny z=K3RTxA*LF`;{G6^!#F-6~`>ty-@4UJ&)JV8u9Gii%(muW4pFTT3+rL+P%n>b*A3i zwOZ@kJF$mM_SLQf2G)P$_lNs&zLcC&DL@CooZ*S zbK8V_N3Xc9%3d9l>?xnaRv4c zw&FYC=jngo2q~Ysq%0r@`_Eyc86$Ele1KlTN0b)a7+=}@Z|nnQH@eK#fU(Xfd>;It z+QpMD&ZRn`pPWv;vVOvonRf_FJpq=z7fEgqG+J`l`Q^Mk$3%GtEIAw_-h| zl;=tls;;XOb%tT2=kO>9*Lnjo=_mUH?+>dxtKqM(8s7cJccUr_%Hl%nVKrk7n=hQ>HEv>{jJ8+20CjvIRPv8`cMG?aV#briX7=S(nr#%h;s>X*$=;5{ zx4f&&R&|cjg^92-QSU%EHNn>#`;~r?k7TL_Z_sDV+Qa~EAhXgiv}NKl#;NxSbw&r` z+AEe)i60KfneqC+sz0HIvP9|Qw4oilCsEr4L>M`(hRIbsRxRp>HZyvlWzVZO_o;HQt3a>OgMtzpQi1(P9#?m@vjm3%excQW3 zr@1c;)6{5vjFsX^5r)H(@=L~UwPY@N=ejb4OZq}=Bes^RC5SKyZ=-zX4x1c$ZoQ`( zi#W41yDITRtK!-wtfl80)0}1UpXQCC+Y%ief2)nG-UX$&+YVZ&W&RFz6tkUUJhlB> zlq9LPF@%{)#l~l8`@nF{uFnVUgf_<8z)@?5+OPg9aSyea-1$4d$zIu+}%WLpV!g4q|EdI z&O)`hH_BmHlKw$;FivG2QAa3!s8dEF8n3nutd{4gqdaZVd%Cgam&nW&auzYux7wAH zSBXp79gh6)tunfv%3gCi*LtFujbM`XJX{p#T`LaQgov<|6L2QRclP`^$(5K7-2SG<|v|1(q ztv!_}mH))JBK81>vP$Ud{gsW~P+-q(LPLsL^?8-cB%|VCbxi5pS5$s`e3*As3 z-+A9U>C^RdV;Gi6=9Rr=VW{TYcapB z0J=JRZ*Od@@E#4-!i#wwl}uq+zMbvtdF!rg{BZr1UxR(G#Jb}RoQ*bXD{a7C=UV0u zvm^BqCcyf>vP?}s40dCaGfxYDaq*7PyYYp(ogW++4L1kx zYi{>UR0KPNK~W$+;XTbEunT(NvlA-7OEqwotwNj$v+jG`4>#E@{Cv})m_Zj ziW>w2zEh~5|26EUDfBK)3iUaK-fu8jZmX>Vo5*_N6>18z*L{$mK!r)IwPD~t?_cm1 zH`=TL`}!^tr?qHd7&DT4L<|QbBT{|aj8a4`nua6z5w2&@N0!1S<*X3@G8ggv(K)S; z^?{y;{GdZ%pE1vK8o!WNoXZLfOMvms!AL-1{s20N-e<@vl_r@_Vu8w=VhK#szbgt2Foi@KfGl@iqVH$$h=|3 z8EJVFSQgz@u7P{-FfJQuV72ccyvkI88}Nm$70$jHUzvTVBozrI?F(ED1z0)hDz#W! zC{HzJvg1UGsiSvPCb+ENR&}$-ZvE|x@RgQlOWl>$Y>9%}{vqxM+E&!ks>bl*38SnU zDdc0<3yM-j!4ZXJ6PTOqUMwDeBvnJje6`dmMiM+ll3;PLyE2Yhr8#o!1YjUMnNUAv3}s4)KpdC~S6IzUW(fpj>dG?b~!!8^idt%y2aDWrcZ zx9EUWBbQ>jQXcgj_mX(VT;k^f3GF5OX04a}^m=rbkYLx4=4D$e)bXwv+z7CkZ)NS} zbFstHOv=fBz+V#sgCA`lK#u&J5t%^>jrO}IJ2I$=Oz*@)V3a4{x=hDA3j~KYR!B3W zxILlsK82f|dmk%^LYL%mgkC`qeMA2-7NF6Us^K`lma&%HF>uqoI#Wi_~^r|6b4(EO7AKc>Co zfBq)eYV;Mgz~5s@M$qzlm!)Di4q9x}tY|Dpb1v^h4AitgmTflcx&qowFJ+RlzZ z9n`sa7qfb??5D<+Q{&3cQh3rvK^`6_{M;)Y%$mPE+>59 z2B0XAtM73~QfsyH)c+jQ*k{O-s0jbDQ>6r0mU)6aJ!~O3%^2hGUqKVmxY$hq|7aZ+e8$N`7Vz zO`pgWXPOuj;d9?-W;Eh077bALp-g2Bzah7d)msJni@|t_obB&ITpzU5&NOl)WW=j5s2A z#{Iz91TUEaump}`Pe_;HF^WVk zSPmt6s^@HlpN%8hT+c4j2ji2H9QWKE&}Dd&>#MpwZaG)~j#oEYq2u0a#9p${u(6Wy zH%w8I(RlC=K1AXC=M=*oj@GZbfB|ZpsFNFPe^Cp3SCtIYX6;3D***T}WR_Oa8VtGx zj^vesgCOt8kU0-ez4O1;O7gcsCH{#!`|*>(Lx#23Lh1N%nL#g%`r*w zhPqekZ`=cg>J{RwHNx17e_)z&DM3y0@`Vs;O2sSuzsC5?5;<&6ODMC%Ep35_LFjn7K}!>mb>5*er31$XkQZWH@*AeAtMs zLlpsQxXRk?gbnx`v99j{pWr@CwD6o08mhY;9vGwj()P10^wr8-^{jGEtjR>`MT6U{ z*J?+pH|Pr1m>1;x!g0y)o|d*TKb?n}&-x&86I0ykOW6b~SWT}BuEI)KYpV<7l=g|Jp&GdWea&R{6#EjF^ zGq)b4ChW;I2eGRu`Z)ZC=Tw1tfNFJZitf1vf!dZ3p?fW@N@iQ z((f3kuk^BR2k0YF!D671|Bd+v_bTYmJjG-MPoXoKY3=|mQB7->*xnp&ZYSz_Kj|xi z9kqF$qpq!VTVGrFM0un#pbXrJr|B!`9#kzZt^RDSw)s#w9&AQU2`Nw)wglImDU#dE z+kw7nev!}6r)uJ?-#&?`?fH(zYR_Gj;K-yi+79 zHCcT?ZA0mD6zn8bG&>kUDgm`g>_X3@+Pgog*>=u~O_-@U^`T(T?p9bF^-&(hv_(Vn zyP99E$xK`Ri#|!(o!c8l`+NDv=wC_6JfRf#?!xA)$x>{za;Oy1NSDmN><-MO6@|J~ z-U_1jY#I2cHI@Sz1Dy}3j#8#|41K38Yc<*t=*z4K4)AOir_vnaKxcKBxnCKi4Krs^ zDcZyQ#<`ag#;MhU^{JydrSw*C5d8}g;&W~Z>}qUKl0*JVAYqR60UZ@ErHfWtPdBHU ze`%^vjz7NTkhzT35-t!ol+So;)XZ0vEdZD7ZG&sY+>k0aGiGzFLfaoW*BKr&+5zc7ZgJajqGHX;2AW-W|uR_6GmoyJ#DsL z2@TWN;^UYtuz23%KtnPN$JnL9-|9}+zl81|ll!=G3ZF=IqK7JLh0i#oZnGb?>)!oB zML8M0MXGHmWv}y7u4%<8U#JI$iY{r>`Bcv3%HX#Go6kva5^{q9cPVwG))E)BTVR>` z|K3d@H7Ywir?qg82v_H`UUQfFxA(O3EB{O!<~m0`l-37&@JF1JtW{d1T9ch(g=ss0 z%lCszSNhSX6)Afo)(Ezt|DsMvg3A}obDXh#)VHt?w65$HzoAq|SCE5Ms61&;(8K#d zZ+{M(#w44kIXN(nz3rrxY_hhH!ge*DbF(o{-%MWclywKhk>)mMF{7W-2{6Vi>R!{K zr~|h@qnj|mM9zgwxWE#T=nCZXcSTZiSsM7~Q2Fvkwrj$6#|>$|I{~&zcuR+Q=71pi zJ&!7wAIMU-8Y6|7!3AP@tQR52+t5bk12s}v3rAyl?<4=AHjI*MX^ZfHvs3;As%q{RwUZHmWkVfvEtGC7z(*(t zgS>*lXYo-?Q8|Yw!R}TDVb$4*J_(hFZnYh^#C5`+oPAhqXOsxd^CRPq*yf>+)Ft7x zKTLO^Gg;@*N<_lpcq^i+ZzWYHuwGlDjPpK*;~7TCV9Qy(jS9?R?5B80yK4S6nrUC; zSMG}FEvU{ukdLTY*@xggvbGd27E@`gA37zT79P1B>?=H(fcQHif@q{%JG_c9s5pB~ zbv7|jo#uTJAHa&)7E@;e&8QWTd(|)Wg_uO;97tedVOT;B>Nn1y%jQ&xCDZI1yyuvy z#vNk^(ODf&aM*nU$GK>g08;^=V=XYHswVCW9vGD0#W`L2}Z03AKsA zY+s~hUBWK1i^wJJSNuQ0p{Oxk4JA+y>HT0;_6xlg_d`3tM4*48O5xjSg}}-EiYqY3 zHqA4LUrdwqdFs4ami&t?X6WRr%o;HT!RyLh^oAU4l`|TI=DBiWD%fc?1$~Wl){MN2 zc%g|nN}3Ow>Yas)8mCT03I1^RQ+=`!!`;JcSOL%<4KWP#P2HKl9d(RJvqaQLxvAEG z4TXP{;%0xX26V!0Ol$FneVp$w*-|_08>!XA>!{EDRejy$WOb~5UUIUljWqtDLB^Mn zCit&q{mx!g=LjF0P(N-5yvF`*yx024YvC2`h{16JHQpDc_Jx_oD$t*+3}U_OLsP+h z=^#-}6t!%1hhZlhnUQ)uSV?a{?_?kEtz^6a55@P`WKzU8GwsFPgf>h>$ajokCi$Ko zI-68o34@pMFL)KGp@p6v$}a(d`dDLkD}SWl&aE?>$TwU?h-jDwGq5dzReD=TxCzk@ za|qQ1GR%BAQSNqdk7WaI*h%bbg2)(*ep{v8ZDFxsMPr7eBz28DD{s*D9(dvLuvetT z`gW?dx`-;Jd{F-|knzyT2g}A>)wc0jM1P8s9?+@gFZ8dFrp#1G^JBuakkXSxTWW~^ zSp>#}o`_D)f25{!HZU9}NvFgl{U^PT4hV`TOgL@a(z3|=Y8Sm3H!ESB;+Kl*lduVS z4e6zUE>t`$Bi@IEBOF%)-?`bsc-wD^5lVZ?vIjlY)zgSJ?^#=fu~<>OIaVtv$T#xz z#0w}*%n(*-Ct+25j5+|j?0q7w;JU&LW~6@B{Xu(6mjS!Uv!BOp@eO*`lo%Y8BBN0i!H2zh|R8It~r`;mj& z1L_77gfCzNIZ_&fDzk^ZsVb{~f?e4P>S!{9d|cDwdZ21TQ*DKM2enG*hW4TU)P&#$ z?04cjPy&YYnPw$63h#|4imlx{tdL@mpY`$iTSMB=R9Nf>YFnl)Rhyt4_cg{fncYSa zkO5)Yo#BnF&fW;My4*t#`E*)EI75$wW3n5WhU+D&EL00Q*pJ{MUpeo?K>z6fY%9oh z=$F#Nc%R6tYYeZJ6zXvs_+yFlXe=M47SUt)kq!p#2cLw--VxX@-zi|HPdQ$L-Jv)A zg_#BIR!O3k{RsUS9QM6bk4iV`X?hKJNwy`Os)Bc@|1r`Qn~%&vWW-woTj=RY4sE7~xpP0`ypiF3xIZ%?AN=h@T~|5UyC8QI5Zq zF;Nx+srW`>4)F~J@m%hO?Gy0}g~4?&Cibqfp9O3SSQPC7Sw<=U61BHIl%2TW2J65{ zMkKif?;JQC=qxQoHp;91!^KDk^X4%fyg(nVT?K_=IlZXo?|hYNnDb5g!bR}3wT(^> zezZC<_1#1Erjz05H;77TpifX*z^j2(!OzgJzA`b`e`X&oKLhkwm5Jvw;` z3#rGL!~~F#9|3ceD^h!XJ1A;)j>m(){BE@y{R+JY1K2q27&gk6n9P_E3iR} zkz$0q7*BnW`*&9wS=<+Z0F5J}sIHFULJ{tR z`5}(fZ^KDUS31f+1-vG(;3;xxei(IF+0TE*M*xJzu(Py}>LGD3TulKhjccGB*NW>G z#E)QID5Za?Fj`Sng8^Ya|CGOK4&n?onjY^wLT_P)0lWJ+{|>{^$J~|@M2qBq!6@Gw z&Sb|jAupP5#&kz#*ptKxuDwo_(+l7UyuRx}u=*k|;Od|tcgc6)lmx$JDBgqtuO z#?@dlyD-*cEO5Nlw?o{l&DH0x`gS`vt zR0bQR!*IXRNO&Lo!OWrtV=t+f2`iM(c@MEvSbF2+?7tw1bS@P*JP}fJI3FHRssu8e zN3~Jj9xSDNv*zra=-WYsSSzMx>_fA^RE|7}`ba;5T9c^>J2?F;8xCHB8)TqF#0Ym&XjsG^U44xv#U598MF58|qHhtZr~xKdNU@(t7(!q2 z5*Ef3atM}ZTM%uCi*hiOBtq|HwVf$=FEQWUoxsrGBhMzRME)@*ku%_GHq5&KT$CDd zd$7&wC?ktr;=Mty;A&B&^WH{O_6_mxR>GLOWCH9&ZR774jQEe-QK_t7FneGp;Al`I zM5@%{{?0Sltt!MjLoRV5Ta0X=O>&2a+{`3?Cv!kaWOE(0Y{k`l#x$x?2J86n@JX@8z~2!4BbxdMb7n%Lq$1%>_fvLEhj2V z_X4H$A~BS5k&SkB)QT&DItjZ^^;RlbwTQj`4Rl|n7W`ZFO@~+1=Z9dl@at>sUrA`7%lv!x}YBX7WpQ%IG7nYuO0}d z`#w?c+3MK)Cara6ejD%Dgrf49y%}+3gbt`L+t6D^I7?kcqYLQ3lgwL=Mcg;G zLdZ5eiGm;ztK>AjyZtfFXIOQ$qqdt!RQq#VXhd8AFW5P#5QXLbnNWwgL_gVCTPz_| zN9o#LbC0t@ND=-6LcyDiGhkGpuBU}sOKB-jbPVC{ni;UT-c43fbnszzUB1w0YjpHy zXv?%i*gi@#rnC3b*A~hYXK+Bj4PW}z$Y`&s416M9&|h4f#P zTYTX-Vck|>=6oyDqi1+ zwlX4Xs>A%9nYEOIC~aa?EOVV&;Mo{&1cwIyRCmZ|SV>7$FS3CUH&T(vrWObZ#yX{`=HMP;dzpG%raua-1L=t? z=?AD&Vm!Tx8izOF6r+W;kvT3`aP?#c$T3dKrB8A*J%z<(mcLf3rsJk%3t zbWvqid`Y#l&*PoJ_d?4FkdVan5L;3?&dJOeZJujgR3zTRQ%I%Z{n-DgZlPx>q6U=d zV1x0T&(oK{Ed9FUJh~t^l?~V=Sc03Z9hEvef5_vh`LI5>hb{n>0&OK-Q_SO)?V4IWSOs6NKF_@7|3sZL zCNQ|_&fn|b|`!lEV@_vC?Ed-p!&TKorY6`LF&*&V!{_vKHKPnkycNHfCO1pB z5kz-lJE%eJhZG*lqOOGQmY43#I?+f^cR9*@&6XDjqGj}RB+)xL3d?62OOvI~30<%@ z^hIT&-ZwW=xe!s0d4ZLtn#2C^yI71p7n=8m!%F5H_cwkP`dhE5*D@o`mgFV-^^g`X z2rLqcn{%}0$_w-W)E7v{|CIY700h}YY9bJEh^4S=U`ahJ@h538{+#+08Q318lE30t@gCd|CVk$C3cHAL0O}Z<@T5-++*oX z{gLt1`yU%-;+nHMn zn+7hbdhkD^z4oHs1g52OIJOmq@J;xC(krnATSL60M1V8KV(BAy9owb%W+1yBy|K4r z+EJtP#%lj2j#a|Nny@9wAPR;kOm%3sU0lIb^0YBG!C%}+<%cxUY@?P!-{e-XnmR?8 zU{vCNGm0}_sf;_Yjbsl;6+V`F7=JQrN_<5xHaz;1M2qi-hI#+XY9R=yGF)knfIUq=vmIH^@9bB-12>eo&HhU0 z012pGWjk1Y zv{9{v9iny~7-W12aV|sH2W+bN9{dg*XB(^CJ^jFdf~8zp?7ouhNrG`gclMjpR0>5% z_=GsTr8dHq8h3~JB$bl`uJ(GCeNJ2l;dm&$OzZZU@hBzChQk;?BOT0VDHmsDL_vmWn;iy7F0J z6ZL~ihi!2lrs)&andA|(4AsHB&m@9dAQIlnuV+=(n^0@Wy*Mr_Y89(EwuhXqf66$l z^d(DxWVVD+PJOJN7A`VxL7(7qqnD$#<644YuL%@rIijpMXEHj1x}!PT2mKizU!XEI zm~rA@qf1U{=S;BKouZb5wLyo_dG4(DL$9+n*8{19p{O>ox5j{>>`!k5TM^|@wHZrT z>RE%|iGPr_jvXm|5%$9A@Q>i>W~I1uN<0c9+lkfCOR@~@2(ir(?g4nSroAX!ILn^) zZ1&bL#yFaKHs&rN*3im^v2d_Hk~rci2hJ#0RGfPdzs6Tyn&BI4?KAA!a`qjlt%e6% zdp^SUW`VgGdk$maBzhxJ%)QeXjpb8sV0X%I&JrOUFV#U`h-C$1?dR22>ifn=&?++` z#6;<8vY3wDA>v3iL$Z?<>rUZ}cBD>U{lvly1@0h6k0L}>9ml?7$KwXQjV%I9a1%eu z{=^rG^-u~+`QnMa=sYR{&jkN;_6RCeD*HET<@m<;1(%E|-a0C&mC$@*RZm=EH@Zhg z0f5FeJwCuPks#R?rai9)@41i+Qq?+A5`$mo527 zLK1O)Q%$@)`P>+-dEsqrrry=*qf2v@&`5=`MTZ#i0<3^ODTlLzx7CBoz@6M8+7U2a zPD-2zclz_4$p(ZS$p*p7uJM8=aXwf~R3JJ~lZ6}jLDHpp)HruNqIkk6xC1&>9rO@V zlrKKgTxG~)V-N%_KtWEDxLftor_#IntJCeY@kEr^3B-|C9K8HNC}e)izu94d8T1v; z+MJGDL&_x`r?o@b^iX;lH$lIL>d*^q6mEDP$~7`L=9uy#U^o)MWzRyjo|$gtFy*n# zJev{Dw4#ps8nc`C^95r)QSSEoBohcBW`7JR6dB%UL~XhA0U!;Csqx_LM>>vxwd z-y(hxH(Wog|Bu@jb5+fumP*U5rgV6SqBw?EHHWcFq$}}@xZ8Ob=smsx%o?0B1}0@t zHPr}LF|`{z(EOJ@XGG~8z;^0Sq9$VW`;HxAN$^T(=338I1%Wrb;T>% zl-T3B6=7N95gSYQp$gozeihr0K3|xq;p|7QqH#>zDnxDFPV~|y=eoIj>KxAw;sFcH z#%f*R9G3?3(0e_BL*z93TzwGLOiPa$fvZeOtfn^HwD@t^8g|D#?%(7^>`K$`?5MUA zpJdbzWeOFY`_%XH?Z7$(5wA1dcwOzAFcL^oCD7BT%Y90isHtk5VDSUxcI&(xmz zlGIY5V!=S}DceYGj|7I2Pr9ptWV4%D%$pJaH)v}%Rcy}cU^YCKbHy92b~M7xI$&|a zC~2*4gffA-YLp5VYLB@``ff)@4vTLX+dLs+&dMiD+*R?r@gC%f5f)TIC{Q=QgZlYtZqTF_ndZA+3FjugwbEwTi#FX zOPWfEFfgHyyRKua{R3R05ZD#`JK7FXjg}}|uj6iK8!9(UpXZ2*9|T>jiUx*=tbaUH zsH146=cVr}_Ahb89HFlWZR_00<6tYgrk3KIM?DKG$8OHCajm!@yI8R#i~huBvb$k< zN7ul62(h+NlTb;X2=NhpgO!2})$+mCT)7abNoX$g+iU|%YKPHKWh3W`eVIPj`IJh= zMEf@OGn|&O358K#M3+9Fi!$2zZ^(aXSgBi=aUZ0#H(yLnvxYP_S$ph;K+QJ<}W(cCOejah2U za_(X4Qr(zCfvdVo-_~yAU&HDb6e*}mO-Y!j#M>SjFZK>4W^fDh8~cK=YIIZe6&Zp4 zq`NmBb?B2@qQ4m#>@j^oppblswc;pptp8nNCc6>G$oq<$qM_HdSncL0N8+xER;J!n zO>)}`e!-v8PS8(|J>a12o^~9DGEfg#FRKbhQh#6rxV}ua^zrtMpdYxX@zRC`X_TmKcn}go1$t)isKNNPZm`d!<)uB)ZA5B zER>ejex`xHsL~VP1!p2g9%Z}(WrRn9-MNX`2M_yC8bgDFm@KA)G?=`|&t~_U z4b7oKUA-mQKj20q@MoU(ybWtipYitR=c)5|9_D&cbEVtCOjm)F%&%5@;9Z%q<`w@_ zVN}LEaD}x?Nb+be5$0)0mJgUBCcz*RxasMve5H3{Uh=kEp~q5xiG>soXD7Z@`Zt_R zcCmgj6%dwij~d!wsc?u$hGfH3@Lj*`?Lbr^@~FYwLHbalOdm4yjkokArVYMbOh<>b z1K^eSpxPKa4nJ74nFDGq%Px-du29yZ)v^~B?kMi)VCLob6yG!TJKrcBk}=KU#@2N#9=+@;X=K^n2dK1HsMPBM=$A~2P#D~!wP8Nbvu z$No|O3;t%u(7%`o@ptJ9@o{_-I8O%12H+g_*Z`n7d`5S6_Oo_k_dy&ZGq{$iyw@vn zFO)vPPTU`GzuLeG1X9Hd{1fK5w4C#Md;1TNFK~~)ska6dOsIu5kQXRV;dP;<^448J ziInFNH{e=(FeZcYIX;qG*?K6T!RXx3mf0|2b-o_t%wE!F`!yy4-10Owj*GX1p7g08 zu3iQ!@%HL+c^g{7kJh$XL);#{LE^wVA4P#~0^HsrTq+yjg%Hmia@y6$#y2Ggm5{!x zw_y8F3b7FFNlRj@BT;+eS?dX~N0id!P1{c$kj`Kp7j4P-LcELnx^p^NME{Swqd~Nt zj&-%BM_6OjZ^ka-R^~+aaLq5arUfb*mONNoyW_6q8;tGsR8v)Elu`{nh#bmIFjrFR zlq~YSq5vyUKj$T#0V{C1pqKG3v=7oJp{4f=yU_Lw`yJYa^srSuoje%2Ky=Q$WvrzH zc^p@i@2CG#Y8u5H1=?5f`oUew0Aee4NBfhVBV?K1m}RJyg8Ni#H`moUM)@Be`HEu$5Ut?;_x6MW{b3#_mS7b9t>c)5fYxZ#NccXDC61AttdK zjAJ`mZRLw_hfeadjFtG>P(s~ad15%t&hWg5p{{1Wp0CqXHhqw-=HH2ak%Rns*axuD zAjI(~PH0Y+50d74v1v$ohoC{g<*5g%D__~vz%uTcFvHtYSpX`!LM2kc4tztFx=e54 zDTaTNhtUl~yk#>f72Ia$S|8;oXFp~oI4rW@7cm_+MPI!0m6DM^LQ*%~TpgSmnjr>d z)efdBhrF4#+u~!PI{uG1DrSN59xO2v-5XbIoynF8>0>hO~`G{ z&0G=rjXs6^=uZx9{7u%*8OPKP+AeG!`XV?sSoeU@cO^ro2xy4Gf z_hHblb8HprqZ!8LNa64ZmdEMhUSqfGvz&$|diL>0G#eQA-G;JjIQ2a1!W=dc;1)Zz`W@0}O&c#KDXgm4p{VOsGUZkWVN>#NpZqW`OHL z?B1y;bizobtJyxbLZn}eBO>BntX$|s!lL6WS1Bs#T zIgYh8#<&{5FH{cdu1ujiX>YvA{B|oWBOR|7nSyO2e?^7p17$85B&Q;icOWmMv_C@p z{dG85tQ6Y*n4=cOlGI-D5{ObpXTLS}l79&L-{$df+YSK~zO!Ayx$Ifi}|B zf*0a;7!!_&3bb)}y2!tZ0M!%M827-5Z1P`oMRckyT_|HpE=f^{KHIHYxuCMcm9MCxH~wMyYL zz&Do(9=RvLHO639bu@!}PMlMIp{=q)tRyh^@Vxt2d*c*#1|AYiX0;^J*`d(q40RFe zXmAnz=dYNMslAXNvnB?3Gq{y5WS?lTe%;rY25xT>T`f{a_Cy?No;0<_hQ`O^6J9hD0ZqzzfR60~xBB?*i4qoK2&& zhjL$3-rR{FF<*m~iM5!+M7ZWRs_>O$DKs^_aHXT!Vs+l>spQ#eOg3;bfvv@?jGd)b zVKzDcGyWCNiW}5_#rLMnJ)6{hUfpdtH3ih=u8CuUmHG8rJeTHe z9!i-C@@mm_ozs>cX2MVoR|S?#7@;}z2078lP497pY7$k6R+*!6uDzXh%-=Eeu9mVb zsiLU3SO{{^byr1o7kvXhlTx%OSfDh7ZS`{MVzn-NJaeEtjGCuWY_6@Ndjv5X}1F`5}*`Hs4y!G2)18u8VStYos*PJ2T;{Hi+4Xk7&{+ zt{MAY8j#U0?%(c!LLxP^w8+Q>6WB)P5akgaBpRv3%!O!cP6fD!_qaOH zH`LcqrSJMDXEltAIBho;BBMXx&Hm;8U zN!2!bGAUe8-{U+>H{;(6edz7kFG2ahZl${8C48x3V5jjuez~nXahhrG{TKXZMr(_d zez`G$ibk>EHMx&*lc-{4;8%S$I86o*jA}aFnP+ zeR8K-&E)l9lk=?d(2TSa5B*PX!}fyr;5(G04hs>l7`+=UrLX1o*Z&ed-ahCx)-L~F z`G@&6$3?9O*3@n)+l1Op8mKqGNqU7gIgbZ7;7l#is;J?#%2l8bt0ugmO5h9Xpe3@4 zK^rv2k*&Lu!oU{SeCRaGh_jq_9MeGlaz&s@Yoz{!sjIJKZ$>WXWTLqnR~NVo%+p$o zm10zi=?42DLhWld#`+O15jb8dB+)+NbBI{0k={`5jJHrng1@vvc8^fiXVPg(Co7+C zf+jGF*pV{-$sye#5e5T$4p@#APcb)0r7 zkLj-7*Xl%Vg4Tc;$Zo7#TkK$VCUy?9ijmn)3ZXG#ead&WGG*+$P+C#g|&v%Io1FS>;`Q9R7tMxwtYI_WqIo+Z`;!|>YP z$3_o!1XjWLq&-J>U^W?tKLwACGIUR49vg*5;nhRwvm(EN+q8;IEparQ&QA%P#rC5U z>V7#wS>Z3L|KXuR1agkjGZ1tYXUc@ShcZS#IA7_a_R3uxO2=x*59vNwp5uu*$A2xf zb1Kn0c9pWKD@nPvnN{3=_DJYE4Xv3b^tSvF-e>wOAt}gkd(vi@2f=3b7rLvrK3q-` z)kc};NmeN1*-D4~A4hk=7&Z3w0T?fCi_5=AT}R@%$;<$YyE_YuyHniV-L(|z9f{{A zGo`q@+u{z3`@T2tH=xW+a_>36^PJEvAK@yX8=;BU>skYPj(S8b z*`QKRd7^xR|3*cksn1gM9@m3VH_dIs9sBWz;80AMY56~`d}D>nfsesT#tXsnf85R# zJV<4^GUOpymj83nm6V-pTMpU?ZFI~3DG#=wZ$^aJhpKE?{?7<+?yh+V$YFqML3I4%F@u=l#y81)*x z;GSvuKfi;e_?C{Qpn$CXKg<8wsIDbF>c3h}?pfCIf5K6BVYcv;uV^c)TmDafdnfpc zxy_!U8`&)X=a}@-xXV;-v;{VCTK|!R;bBIfPCddm|{?904b10Q1 zrGajpvHYKU>Ym^~{+{+ZiB%=b{}~=yj}`L#jDv6us&&u8z3N=z0Q(ZFj&}@N{!e8t zr2i#r4DO%hwfvtd+(viJoEGRaui~-%pL4LPy3uh3*o}rk%m2Ace_(6ZTPn1Gog~Zu z8E9J~6fhYk!DeH3kpFW?s}ofCSAt-Fj9LCqB;CwdrECqB!p_i^|MR!hSJ|RxD0itb zF3bO!!+j)6D?HaN=b6LufA(|DP_MbEyxIFT$?|{xw2x(<5Yt0z@!y8!|NK$~!zq4q z?ly*nTmDZRc*Q5^|Cu|TbkOpDuHb1(p4wOV#yoRa{!bZDIP)Z?OVO|{ZTUY->Cx(7 z{hQFrQI@v+pQG^x!Gbp25^$QZ{Gai{_l9o$vXT>bO0oQ(NY8fp1inu>t1jX!|ECFw z6J6A<%56`LSj+$ULXVO&@lBKv9Kc!rPdV+Ll9-U{2+5CR%l|24`ngNQQ(lCV6wCkl zigJ`H*NoU~6b@PbPhp&pBH1(aJK{8J`9G0DUuGZG);rQ@D_Q5Ro-8}kRk^l1-HG=Xw z7s3qTxMBG}mFa4z@0yj9P~}H{2zg+ z8t6iG!DyNaSpH7~ZXCh#f4JRwVXzAFe+U@qtPXG{o_}~_o%IA8a<^SZt8W>GdT5Nt7&hmc-+G=qZiL1$zsr#7a z|9tj&d?l%KU;tG`vHYJV-Y3+G{O`o~jBIE$>y`@CrB29V3le4KZ4YjZI6+W zg5wrz`9IIdQrzqKWyW{I&079XF_aaI4qeVUh2|10|ED_0qY{}b!P)%IpymIp;AMRc zI7pOpmxGr7(^_go%CIA|7EGcn|L209qd3j0*eY>`-|~M(sLAwMy_YLu`&z>CfBt|& zVHb7`NDQ@eTmH{ry@Tx@OaZjt_3@tmjBZa5XLYr0Gsgb z0+#>N(b-t;&+O)k>sv(2|7qi{Cz{kdyo^~_wfvuz!UMcLxQy@S`T@)TX+^fgYUi&_ z(D6ks%m2yaz6Wn4>`?Y9*A2`6DaEZ*nldYi$5`)x<^Qb44-501VR@~Lcc$h4d{tF0 z-G!N|vDRVvKecEy4*}0ouM(>0mj6>YIQU?D^_`KF_}BmYKckIKMm6e&agMvqS^m$3 z_%}*LP(Xz8e6owqPxUSIhf3}9=_&>=%>`QY< z%<_Mlkc#}-GgY%`(a`dLx7Ba#1pt9Mh2CFMWE&XOolV` zfV5ca5@A5g|9NOk5^Y@3&_Zmc<}iu4c&#D*ikjy?V3uX)n{|}7XwpZ9UO0qLkT5X? zv`4r6VY!95QhB?LTgq$}U?t5X5-U$sbM;cB=DjSglb)T@^gBY{I{89j>HQQ9gz1lj`>Wl^^FGqqoUZd=6Nm7 z{U^%3w-Yb&H>e3 zAGj$U6As(bcvt>&X-aN!`FU__dTVV`-c}b!<|^y#bNtnepUfiL9%CtSh4Uzh>}qsB ziOIbfdbOveI+B^6UD+JwcoJCStihC1{-c|Mrul2=c<~rjNZrg|F=LZ93YF0Ldaz^* zmY2plZ>y7$jj&R!WR`VL)=DJx#d=a7Lx!UQc1vAo=>CT69eXLNB(fASxt?!~dE7q9 zbrx>R-flbLCFEz=4Ys4AOY@1^VnwzCJs=b!=g}K@I`=qAM>o+^!9C+X|BooUd&%>% zzQS?`s)cGgwo?&GZ}t(}GQO%EryE8dXQJITq_ui?d6jww$%PJ%4EA&#RZG%;XWxLV z!kA}Zf9xr;vu5z+sj}V?F^#x6>iXO;qCLGfely*`*B5q=dP;6m7ShA%E|lo2qMaiz z8PnMtSW{(}ps+9aG1^uE-8d3|u=TaaVguhz`ltLzxWw`-skaVn(qDS(gP^^a@P~a8 zSce2g*=+CHVYKz`SHeLzd#r63UK{L&hT~#FDRN2lW&608jln@w7e{Ncg>fqIM&8B` zV%E}=^w;*{u{B^_Jg8BjDC!3_Ct@xaf1p#|3gKMtG`wV>Oz!`bI#hdNCgt1pO6`;c(XBdxFOc6>aCthV&$L zqgld!My%s~E!;BaILAplj5k^qBioCB;cA)aC>Y>wil|empUEIPCkPz8GEQPSwc3* zVW{W!nro|%GLI^qnMpwBw&8ggnjc_iB&y0~WdVI0O;fk!{)GQgPeT|t+89Bgs!H~c z=(1WLZMK@~dM207-p2N#3)w1Djo?_hqdr{n=4{ueV-X2;LlZ$otx-~4b}8N}?=rUt zz6B-O2Fy$157@myFZw)4WqRtFY;Wu>8%~r^s`G{O3#qKUUU|(=C3Gr|!m*p&M22t| zvbBa%hFv?!aY31W#8nU4QzR8`%t1K3_kTH^=YGoFLlttx*SLnq=x0!x zeJ68;`yWg;R$xOygQx_9B0EWU_}8}EMnk2PK3yA455iexGCPdkX6xg=81vO}(mj?4 z@iX&x1$(kjlhZw$$c$jLNvj@vTYn7jNL9fz_`lFes;9SDV3E8{AFtlfi)r;C8~o;& zLH|cw3dVCM&2w;**+KIdH^3AYee3yZW)`dzYLRn_dts~Qf2mg@@?3>tisENrv%ucK zWOWePMb9t~&#Mx-xKs@{6~*&6+CC)q`AjzOqqII>`LxS{b3r z=I*@)P8#Dm##qCT7XP%h*FSnY5}k5A!4>K}E(+CliiKKhsR>tt1AN7Eifdmc}SX0JgisVfphrrKF2R20y1bb3`KGWMyy#e~O z>!|%|ZFf3b9UjD*=TzaL_&_OP+O%%q19yd*3yTw1U46C2WU>4dqMBJ-S!6U(I)i%j zOS5(86ZwacA7~9HsbOG`){3Unr+I6Ehr!d-cy5`m43pnt&2U1H1&a!ukYqX6)gPOVU819-PP#qDad{-s z4kUV>t4;CUo>F>CF^(4P3*ce#aA*iOSGpubn8WyHa9H9<{1NQ#h$Hf{OR!P4F6w>q zuAXERGWUQInnHCnYIED+1+Ju%7o1Ft4JmCwKbpH+lQ3S}Db5GGqmH9b>4~eXT;N_R z4)k2r#tRe^Ci}o(V^Q^6Y7Acx??grdw1`wzV{*^QA=-*`B8)%XwpN_#NWdo=%* zGT36MKIf!F?Gf*T7wCV?!|n{Y#@|MFiYW!z83?J5L-F)I@H@IQ*cu$3-%9(Ww0E5)|FTuk8o}$XFu5T+ z({;_clWxY|R=cRYc*yvl(cItHaZHJTkQk^GGpZO4dZzQM_oMVToCPNb`Osf#SuF`p z7p{S4`Zk;uJn)Cg;J@UDMsex}Q4Cz+n+Tesy03tNN)?ILN`7yy__Qo0LH2M$}TL{VpZnN%B!<&=d?3jG*O zQJ+^XGPBf0UjoIbQ|Q7td;furG7Wk|#SP_4EKcLSUhr+_`gkz_y9Uab=PqGajC zlpZ8wjAL7XUfe{igRw^_m0v0UVcG;G+O29k*`+t(#r2a*{Y-@>rKSj7hVrluKYq{EoJV#AnuCiU(DS3t?nffC*ODnFQ z*f$wO%8j{bFx1^xx48%EqmB9au=#d!O{Z=rALBbc|orqowCfvsW;iw&}6 zf&;-F=79byW3fs}_hTzd?RaFFa&w^UTjChO+QsQ~OHc?Fuusesltz_^hVFN;m**Tt z8bz3|_ze4Ls=a?l-buCt|LBWQ|Er((5-AbW=4==)eaEvAfv)==&#o5dqWtw&l^}Twa zUcpbUIvU^4%BiKNs7mUwgK1iW;3qhTP1oA7dH5hy7uppVqV;sv_Fom7pl=!tXX^(N zt~nngjd7xK2V>Y16zZ;KO*1BuARo(>8muh~u@k z7`=Iy_MbWz3-``PRnp)50&Or$n3HY&QMD~L<{Mj)Zo*WdJbVE^!Lc-WEmufbn8!N-9O+4%DkEQ9Ki6O8p+e_~6o~(8ZY|fbm_eg%s zwSO&X2MzH9pbF|eRA~4YUne<8ovtxdK4n`-*OktaCt`oO)~Ge9tGeL!5jJm0`6NG& zc>_x$hx@Ft2&@Y1H+sYj5HjOFQBLkBegHe>9bkV#7nl#Qi#hw?Pjv$LLv9H&?BC#w zB&Q?bbg{|FBaw$jfe#t8_-X8Tda8S-3XCi8gz-_CCb`I!#69IMd&|8ZOwH|)REzNP zC#2@Cw$fj66vZ%o%#W@lm#GojLHe&qMg8GNc|BvC@|oFh7n5XgMf+*rnsotnBu*Tt z&R$j*ay{sB!BQwK+#qKjYe;wG@w&ziCi9G<*c#{&)|*_J*XRSkG%>uiFqwF;Sy1DW09Kf+=aTHb>KO+LpGW4K&2g`I>4R}D@YB+?Zha&hHEz;mvd7{#$y4b zUKwIuW4nua0qT+0gyDGPezwk;@8_#faznr6d^lwr=;D59&$2F!0 zk`05Jtch z)bU=-OlDS7g=D{)>#nPm5QAEjF@lQs1d0E7e#6`P6Z$Rph3bu`DtdrK_InwjNyK>n z*tmbWvg|o_3Ny&q2+HFvm);wQSg=5+O?WB4Q5Y~!Rb+?b>5>_&HcG6#eQUIHaO z*JtOufjFt8@|)GvcHzQt)<=8e;3GNWaqPNIzc{e>3?`tG^c6#^8odU1%nqVhBDz6sGOaRXXW*%NVaGbwM z=cr})4s4RKo$YI6v0a0QlsvP)(N!yI`>s!92f^}erBJFkHmMdr!{n%$NVD~*Cy5c# zB{`i9X^+Hg;;Ux&ZPFPPip#X`cx!eP9Tnf(I3tAHhpUBAer3GTmiO8y7s)-cUkVo3 zX6f7YlLD!Z#`Ikaf)({y>8-lB43AGny+ zf^}`>;ZJ(3!=WD(grt1sxk-a}j)l%ft_J3Hv7FhR@xWhfG}>y?$9#dMfB^;zNYkk+ zpd0K=pHYW#eZ8Z=I02dH`WSp4J0xRm{K1SybcbL)s!!-6I-xuQHA4M?M$}Ndh1P*M zR^dx3Wz9X_cx@TiC1F@-roNvau6z*^)kcA#;&P-hF5>QpzMQsdrjU|U4lLF_`08qf zq^H_+?Y4h1Tx?^-v&;+MV%R6WG@q7ti>t)0MCZ+y2?a(+s=s%MoT8p*46QZ9_3CO5 z=s5V)I|y*;4RzM=+1@cD8cDg=@?PYgH)<%a*-lJ~I3~D&uFYNq175q?Q*=~ve^`|1ln#gDKv}eZMz`2gtdW{9zHXZmGU<1nQe|%7BM9F?wCdOJ?M$tmBluP-Df5hj=38l*yRaLCcHE0G>W7v`W*Y;6;X{@3)!QH{v>^FT!XnJrj-%ELdqyJL%t@9IImETXq zx{uk?rTgFkV`pDTZ}!%7G$MNmg~@Ix6PFGD6z;<{wsNAMD+d&H0bYjH)mDZbTr|&8 z9b6N|>Pm()m1XUpg*x{AjMpfe`#9kbd!%&G_ye9Lp0Zty$k4jrHTQ);V_}Fo2aaA#&@|M;R0JwbDYqyEGNe z^y|L6>LP6&h_=rGRg=2YPUm@zBOj_axJL26m4Ag#_Ulor(QVSk)Z@Ia3j9L7v}+N$ z9S&Eovdx(R>_}s(v{EIBNBT==3$6yW*Kv%v&J@>fs$0c?Z7@DV=_*FLTVc24m5EXQ zn6$oJF?KFnH>wUFnbS%?279ynu#);B@MM~I)OZ>9o3Sh>)w@2WnsI_YYfvqsielU zvtUu@f2$q_!o?Dphk_&UIWf-xDnkRPxtCn>*UG26dT^rG^p-zFNdQ zWCf@BVo_!EP;7PO2ww`;@^r^sGQ-TF-kE)qk5Su^RI-H1=EB&Ud^u{NQZ9ECW?)x9 zY-khnx3UEP2Yiv}H+y$^h-yF);+Pgb~MVgNe2Hvp4s8aM^`%=C%S6O|nt;ISi zBeF3a*jFoaeWzV5&5XR^bgleVT%`M;^qDVdE@eefO;6IDa(&};PE)=lJ6eT#NjE#rE^xoV&EWaAHD zb4NJiVk_lc14o#_0Or3UH1}HF@_#n!l(N@n@Nj2C!}5P-V13y)E*iGcDvOr?BL#cv z^FR)FQ7AAi|A&B4axyhiEE%L!%m0DNy_j~wV*V0$p0WI&wn}@dsDHcpiJAv3|7SO> z!uJf`2mkpx0L%YznB9zQb6_w%DuS{6pY^^=x|Z+mbV5~INZ}~rNs=lX5cHf-x6wodIr!knz?od2f zxYE|u2l+pR7|m4DXJz%Ee@d4B^TZs2((Xyz!uUCw<^SB(BT#NN(pv(|F)aUQBfMbj z3(PQ!i$_(<|8Zz>*cs3PC3p|}EdQq;6DK^!25EPwk(%ZIbY!_u38RneDf2RD`9EIg zFmV{1?p!Ii|9}5y=iYJNQ^Eb}XuU0E`9F67#%=?pqkkGFeU|^zHh77gqI|{Ys#grl z|49!{W#3EBxfmwXZTUZ=T+{P<(Py&Oc{#)Ke=4ch4|e6J%MH>CXv_bZ&XsT^8H5)m zmX$64X9D=F9?^I2yvvMYEdM7KAB+Ug4$jufpOoeQ9Hg)NugFEI{(3FD<^RkS7PF1= z9%-X&R~lIU52iL3=Wq?+?)Wsn<^LR@I|hy(?8r0>l;kb{=aX%!@Rg~>tfR}5mj4r% z)l^R8+(>jU#asSQf0QDKCG%+|_=C0lA5`JcqMc>2e;o@+2KhfXe8cgDYVo*%)Q_O$ z|4@3kc^_^B6C4*L%l|2(eg(S@bXL}h9?kN92D$Ep7Rg1(I40j~`9CY;w_%0VO6cj* zGidogf7_t8!K{%z-hT?S{GUW`7L_3FGn)p=(U$)+g`7ckVc!NT?C%(3`9G%eiaAG2 z4Vdc8kmdjMGKNauX_xeat}k2u&n!BE=lt7@!sMnb%l|nK>nfMElgcq6U$y+7&PrR( z#Ae#(Ff~-m{~2OX?hI{-{zmIfSpLrqSju4cUSdxsEDl=!Pp{aOSf@~XctIMb_eQ7SC2C`(Xs8W!n&>AzBrN~Ogd4ePxC7MH zPk1f=C&RlIly;wnCYr#t{GU&1nkx?Pp_B=E7|Z`jl8>{k?U44!VN_?#@_+hrJMvy-AEGu; zojsQSvk5DbSKt<QaxDstMsh0dD%JP43x~ID#-j>^hJ_pPHsi0u;DDuDHYjp>0`9Jr~CMeS}6+bH! z2bTYH#k(c+C2z3yo=(v$|ED%v-~1%&)HLwgX8Au$xk5Ch3R)?YjIjKlbbf&Qf<7*o z9#}wG{?7nU6Z#~)9;~31fR_K`F&1b35H@_(9gNBs4m zkAK3au$KQ*B=o~LOkEJ02YxG-|MN`UuQnlV*d16yu>7Bt=ozk`LSL+lc5Sca|76hf zeY^I0l?v{@d6xgP10EvgCB8R;+6~?Ef2x4?FrJx39#acc%m4Y4{zXqj8w%!ubi3vM z2xd+7x$-V{jkHm*{GZL{UwjvB3z?eSow59%X-TK0k6DQ@j6DS{|0l>eIHcJqN6ku< z<^SX$ulcAWDqORKue^QjSFviSB`>lSEmj9EO-7R}3`O)Zx zH-MJ^;{prh=TxG5JM$l5`9H^5Cv}gHr|W;t^N? zLnu3C`9D376}v}#sZ~>Y0n7h+<3S=AIepg^A!?ms^y^N|18X1=6rzFz*~{6(eI-f zdD8jT^%Eu5W-^uCYG4e#MV&zXDRfc>s7Jgx+!a?U$mULoyEFdHeWFwqR-`Yb3z7|# zIGN$I`uVUB}xWCXm~}O1-3u17Ed$$<>)CTTulU=Aqhf zadWLw813(IgI#1*p&uC_Hln>FiDo4m-J~E`$zNW+fp0WVDjOtPAB_)4XsuPCZ;&2P znYiIPtRANes9vy;V#2mScONDzVp#Mph<1WymZy)nDloD99(V+Alw8X7lw6^_w8emR z>OEz>zeaios2*BCq`Awhoruf&N@`bwU(b^ay7I`8yY+59+@X2cVPtJiebXoQ^}{#lMCF_KYbCFGk#zKgsXu z9;out$DI!t zMN?59h$mQ#y^44V_D|jJYeJp}mDrQU6KIyL)Y=Xt8UHC?aM)oyU zx`yXVOi6Z>HoK98NAb6v@3n38X&|XPGsn4C8mEy!7lyo%ee^t$b#&)`GY;l8*3Wzo zXNA_%G*K!4f>@mWBIUt+Z7St3y9Cq0ve;9;7JLm#c2rkG{7P`iD9mi;kFbSxGStQC z3=T)NmJ#{Q>4|9((krr(<0X4W&tV?J`Fu0~0FAd_;ftVma0^Bcoz=qS54N#*F=ag3 zj~Bz0Gq)W;`)EV&(Ocs*J1eUW9!5m&H_TRb4N@5YQ8eIhYPpjY#5xpHq4n6oSLr^{X zb$f--T~_n1lgrr7(_MpJ&<|@wWe@|BSAcR@B2|ii4KLs?sms{O97wbR(4*@MnRMqA zBUOKAWSN;}5#J1W8*m^Q?FKTu+oR4aof8Kecd#~0w0aMocKXBtz>}Qs+K4SxySjTT z5k5P6hTr4F*qdYqy6-1u_hUO@EORTeFi}Qx2CBLV?h{=oybapp35-_3bB#Jd0c!%c z1@USFaUQ=ZR76UE^Ab;hW}!D+PU7S9sW+M67C3is8qX#@vP4z)WOZcy{)BK5MX@hDN_5&DYH0B1m?kY)S ztmB|kGBD5J*pBWlQg>7vZB`?KF9L5!;bMFIc+#%ihRSsMp!mYkKbV-~(RN{#WRNpT z`A2_Eyb)Tu0btDwN!3GZxEo?sHl7T`ZHJG<@k+iQZKmVKv)%L-QU_*K^6l&#gTjx? z&j|-9fVXA=Y--fwrbOema=y=bAEGO3Z?*0jss8!7m*q$!m91doI6Jo&)!<2Ln*z9S zNuUWNpf1N2h$q;_;Iiu_b)EhsC8-DD2)<4B2~1^LyLZ{&>7|TzczF*nQ@9sNOZ6{k z_tjf_W23QqCR-x6H@ArC&Z$N#x)}b&*rwEC4g(ukP5z|b)t8!I^hoWl|F(V`*TpGy0Z>GL%K|g;bC+fztB_`4VAL`Z&~g9>FX!Pv9TqN4b^AFR3)~G`j)Q$^o4zo`2~v zjB#m!pf;3Snzc!htoHBgSC)Z7Yowau5sa+b(rwMc{O?K!K&p-n_Ll=()KFE_6ndn$L7 z8m(5NllkXV1I?%PLmRu6lC0VZgk0;jH|9dFGd4x_h+~xN@D=w(KaG{iE@hTcE7-GL z)!|@mEZaq5aO!4z2QDNqj(^v7e_W7$?~sG*bQaE=o1r z>n{rb!5yp^`#0W{z6BF9>uT<#^Xe1fyOPP5bbW;RZoofbKO!+XLhD^eL>g~PZaPTw@_me5u&;aei& zX~&86dRsx@iW@;!n&wkK`RWkK(M)hKHP4)?7I&xc_mm1jj(e+IPkLe$rWYMtC9k2{ zn}_Ae*kZJ$V4IMOW_l~iyQFBjNp@NLF+zYR(0015$`Lt7%5i+~wt>9bMj0jc7d~@0 zm@4#e>=+Rqd=Yr2G}7yGbEu-sFwr0jLJeVOx8R$Fsz=A@6yc@*Iv5)pruNmh3rH(4 z|MC9q3}o6_;$UPYsnTt7%@=@dx!UD^f+|w$yFyD5w|Y(Zr$7*U?v+u}lql z5Zws%Q$fP`xU~u8Je|nvriW5VjJRTq7ZTC*dl9V?8=q4}FDFcO->TkUX{#4z6tDwc zXY1h~;U;vdkd0C4L#bD!C%+eSHvUUuxLh*tw4y6vC^7m#DXSlJ{DRxqbl+5UjNFml zB_)bE(M81J-f?&__e7&y{=a-RIAUj4P|0YnG|p{?CUoy2f#0KT&ijQ@uKV*UDBE-o z(hQfoS#<_`I9T8OhprLgvVQB>%%|*1t|L^uQ60{K*~}GoFL#yxn=Fzy(lt6X zj@`sB|PVNrUZ_lx>~X~g&BT)QgA9`h~IR#8ig&fu+a&wLgr#AKr_(DB?; zB#RvdTj+LN3Y?Kpf*$Yc?4Ror8Qt617?sw_m@b;hO#@`lF=sAopRFgi%&O$owRr*#!F;6M3w=>#i=+}t|!;960 zGu7is8@&qE)4r6eY6PSM=B~_N*&N)+T-ACqUO10v?=7^iX``ISjG&&f&z-A{ zwu~K1#?@Ud)Y70me;*&oj=~9Hinb|d0QLaVL!K3ASYAxIH+_ETN7^TfMI3X1*2d4xF5w!o0^J?+N z%l)k*w+r)?9p1{w9WJU2cAp9L)Mc2XEXWaf-sV#)(>HRrm_2d|xqQKI!ioG|uEk_y zqadKDorOnQIfY{vOCMmYxknt1ys;Ju;p8VX#&^>=Ap9nWCAi3hU>DLu>=Np+RQ_QK zCj8U}QddK-v}9O`jiE???}K^ndstp_Y5FbJN!;l8MXgnK>boJWO{NbT3)wsP0Z!B0 z{2FvR)ug*g+reT67u}vLdJx=#Z<3D@dr?oaI$TDbgbi~#QypXphP(d;0+D4FiRhG; zq8wCL!&z!Cn3VaNFK_!rDQt7sS$`h31J$ld@6XVh>9y@9(UrQV>&6pTW%_Sf+tW=f z0!XZ%{Dc{NZ~%3Uo|~1zEZI0kug)|>R$PkuAt6niV5=1>57rZ_^$ETd<&oK3`7`OC z13$3x?uAMcaVIC24XIHfK49Y=u7+-mRr;pR8ewa{7{5}(Md4enzP^8<}| zdO)M|N*w2q*D&L-cjP6r9UaCw9mmL;N>^7=;u7&#`$}Nq==`GYVe}=^^v_a{GGz{R zL_35E+go9iZ9mmo?pkmLJV#umFEGW`L-cqyllbMjz(g_)k{VOn$-C|ve#S5zJ$z4` z^O8#PuN~9%5lS_xICo;-aK4Z_#Hfb0EhW)!2|cNB`ey1_Zc#2?YHiq!rtXQkE$9ol zNDL5CjsD;+cnI2x(ZMj=Hd{&MR3b{>hZcI)P%+t`xmNaVL~S)k!RhT<3jGdmihf01 z%_&SDSOk>Ue)?*0|6yCWwL(*3G*w(^Ozr39DnGEA?!TQ;uo3u^Z=02oxJub8ZGnEh zDcwA&1)qn6O*H8&RmmTz?UO&TOL+%97mO8F5L?KWoRc1_U*I~~&+?<3Ltx05WiEDH z!R>rSG^5;d|4ns}S~2*`m+Wr^M;IsJ7`kn+k`l@1Q4P%XumQY<1mW1`-W|PCvPLDZY1Ap20CGJajnX^m%g5OI@k=JljwIk^c zt(_|&w4EGBpOg$6%f1)(80FdVi8GjA+;&tU5?B}V3eoaF;wa703)!`RP&z8-)$H6G zS~GZ#PE~IPt|YqQ15h4v@SL&|%c<4ETU2dlrqXqj>$z1U#57QQB&Lcn_GU`=_z~O@ z-yXo?2HW0V7rxgxrLmF;*6<6E+1#8BGaDPN1N}oc4~&#g=8kYG0Dbw%SLS}L4j03} z=UdW=#Hr97P2yW2$F;CpoSUJ3WV*P^aiZFST%9>NVSnlx_Nerc3K@;1`Rq-h0yxBM z@n6u}OFgJoUwx@}&E9uJqEf~~OxPEUR(Jha#t%z&kilJ}NC+KWts?w1;XuIdV zAG+b##O2_oR@>DlkgKkMspPrzS*E1y5&I%x zw?O-$d3YLhG96q_pfWZ%JB7JH+|y6EbM1{`EL(;g6#LY(5+jVshEqMCC@Kl& zRk;>FF6jbYLm7t@@x8E``b(HD^>rjOw}fT38Okc%x7Xz!l8{aO4)jsZ>gV7Ys)Bx- z>Lr$!3nTSpnJo@$1hYafb@$TYP$7c;LD^i{;tIXm&R>0o_WAVXHGFBM<$b{_tFd=Yu<) z5m@elb9F{xRbW453MXNE)54e;bYuA&Kh&3L?5F-sP~bfg(@KYCQI(LSHjHaez0T|e z^3f^qJZzv8%YUN2gB96O!uU{&ph86j|MZ31Gw~|HII$!bZcJv!D_xCBVr6W(5UpNE z)3=Mc^XziEJN6LZ-a+(u?_Bz&vLSdmn9O+GzSYa&heXGGBXe z1G|9U2&cuuDF3?DXd_NmHsLkQk@kV0vXp@~jr1k{htH>mAu%pDwi>Eiu4l)}JzV#c zCu9;mTaQ$yvZM1io2T;zVa4e+a$m~BwRYKU9Rd}dkNJmmC+<7(G+{_^g)oIbO0}}D zNxtfh(w*93_9L?*dkpu{{eU~BchP>p5o|Z}DBchJhZi(#t!{N62;NcG!uwR3vmWF1 ze?eOyM~d+of3f}92cg$^BVPq76;#0IsT#|;Gn12)$z~aLo;%;1ZPX+;gR@ElGt$_^ zmBKa!rjaKDUlZDUi%1lfuU!aZPu(aN{am~4xX zo}q7Rqr{QoeuYl_j8fOnT{5aPFX8Oyton!_0%o(*nHaECTZETUBdKgQMD5`P{&#km z>Pe^;Xy~1nTsS*?=hNi>Q8Dhq*h{qgaUQ-N?SPmRI_LeY65LMY{g>c8naH`{VB3$L{HTYAEL|s)UK$4d1dm|Fe^Vku_jgQ2ay0eH$axL3=^^sVhBssUh zOH_m}f_P{+08iJ1!?}%S6K$y8#&HXCWe&`}5BiA%V0>UQ>XW`pZtD(%Pn0vlR@CK~ zDF=+ONY;H$vf1u1|F~PHzF^|?j@%mfP`;V*7oRE(gJTm=2iRRH`M>Ob`Y6ykSQ%!! z#=tkou8jp%=&Q^adbNHd@*&qEBf)l=>XREw1@PZ=D>SdLF@C>&X!LgTIZ=>(7;79g zRY-v`SU4!_e@_({{e07caoj?_lDjjVKvkf_LsM*>oN1tjd<^~%CW2_gASQ9mXeY8g z?_+QB&?VfM7(%oER9N0br5FTkUrm!^5B}7X$!KgmS6Ulnzrc)AOHnp$HoKfN;1%x> zrC#urS*W&$+i$-_Zv%v*E2xHcmGmUrqNLM4m@CJqtI`iMoqX5zvdS|(D|NTyPD34~ zQe)sJ^p>?Zc8eFF6%mXTx4-O_h71v%2(n*1WKT6~7o&RtI4 zLDXmNC|$Mofqvv`sfwHBy4%hMV?zG}!PtdH7RURz=SHpjqFx9~ZA+EtX+ z@glBe?kCbCrYp~}r9eBB8Xb|aS?P%^s0v`N+KSsIZ#Cb+DCMGaA8`bP+Xu6M=}*bh zaz#g?AaOF)oXTKJGrN>crX4hO2jFvQ1Yb3Iikc!_7DiKdwLjo~exf;6+2SuPL>ww0 zE-I?r1216TW?$2b>-oZ+sJAqQ@@x6}DKN)OK+o#i;)CS#^jYj*V@#|byhYKr;_zqk z5Q@cDr5E5!$l|e2=pA5_csswjJlVGyyM!`fKV8pI}+fO6w>*6Fh#&_-DR&2bi8i^)FD-y*rwAvsPwdbuAAI7rJ_S-_nL({iLOf%S_L^* zm_uAqDWPP-ndCLjn!<4BaBz}==B&6Bzs-L1{{rg)6#ltIyzRU^G%`J5Kxn$oIUGD& zzR0pay9vJFIytraJc_qWA*{Fbu&1|pu!tXFK%+W(H~%;dvkWeqy`DPgx<;%uywop_ zy#+38w1zhKxuCXNy}rE_KQKKcyeT-kzWY1hIHa`dxHrA*H)c8LJ!8Exv0gk!vO~HG zv@H-os2w0b&(v?DM>JP)*wxVpO(ysf%dwtT!0JpD34G+;ZvI1oM1 zHr2QfI4?cyIu|m_IX65DxM{hOx#ltJyE42OzGgelv{txMyTm%~uKT*^uCzVQId(ca zx^29^F(^2jvX8y0!A-pLyll3>J^d>9H0vYzmYk7xXCRPyI8eiwo$YgHA%fAto1;RG?X?*w%)qTJV?M; zz3aMiFjT+fFn+#5IoCNbHN&}sIPNnCu^}~%JlVKmx@tgPJ5I2tH|siay)C~CJ@_iu zC$%aSI@3O7x8pn*yw0@2J>WP^sItHgJw!lvG%Y#mIXt)cxrwzJvmrtRHLtoMx!}91 zyo9}CxG^}OK-IvvIat1*JbN`*wc@!Hwb;1LI}fp2v5`9zIEuJUIo3c&Gx9uyE@(J+ Iyk@kmKnw7WQvd(} literal 0 HcmV?d00001 diff --git a/inst/tinytest/test_rope_ltx23.R b/inst/tinytest/test_rope_ltx23.R new file mode 100644 index 0000000..15b76a5 --- /dev/null +++ b/inst/tinytest/test_rope_ltx23.R @@ -0,0 +1,89 @@ +# Parity tests for the LTX-2.3 RoPE port against diffusers reference +# fixtures (generated by tools/gen_fixtures_rope.py, checked in). + +if (!requireNamespace("torch", quietly = TRUE) || !torch::torch_is_installed()) { + exit_file("torch not fully installed") +} +if (!requireNamespace("safetensors", quietly = TRUE)) { + exit_file("safetensors not installed") +} + +library(diffuseR) + +fixture_path <- system.file("tinytest", "fixtures", "rope_ltx23.safetensors", + package = "diffuseR") +if (fixture_path == "") { + # Running from the source tree (e.g. during development) + fixture_path <- "fixtures/rope_ltx23.safetensors" +} +if (!file.exists(fixture_path)) exit_file("rope fixtures missing") + +fx <- safetensors::safe_load_file(fixture_path, framework = "torch") + +max_abs_diff <- function(a, b) { + as.numeric(torch::torch_max(torch::torch_abs( + a$to(dtype = torch::torch_float32()) - b$to(dtype = torch::torch_float32()) + ))) +} + +# --- apply_split_rotary_emb --------------------------------------------------- + +out4 <- ltx23_apply_split_rotary_emb(fx$split4_x, list(fx$split4_cos, fx$split4_sin)) +expect_equal(as.integer(out4$shape), as.integer(fx$split4_out$shape)) +expect_true(max_abs_diff(out4, fx$split4_out) < 1e-5) + +# 3D input with 4D freqs reshapes per-head and back +out3 <- ltx23_apply_split_rotary_emb(fx$split3_x, list(fx$split4_cos, fx$split4_sin)) +expect_equal(as.integer(out3$shape), as.integer(fx$split3_out$shape)) +expect_true(max_abs_diff(out3, fx$split3_out) < 1e-5) + +# --- apply_interleaved_rotary_emb ---------------------------------------------- + +outi <- ltx23_apply_interleaved_rotary_emb(fx$inter_x, list(fx$inter_cos, fx$inter_sin)) +expect_true(max_abs_diff(outi, fx$inter_out) < 1e-5) + +# --- video coords + split freqs ------------------------------------------------- + +rope_v <- ltx23_rotary_pos_embed( + dim = 64L, scale_factors = c(8L, 32L, 32L), modality = "video", + rope_type = "split", num_attention_heads = 4L +) +vc <- rope_v$prepare_video_coords( + batch_size = 2L, num_frames = 3L, height = 4L, width = 6L, + device = "cpu", fps = 24.0 +) +expect_equal(as.integer(vc$shape), as.integer(fx$video_coords$shape)) +expect_true(max_abs_diff(vc, fx$video_coords) < 1e-6) + +vf <- rope_v(vc) +expect_true(max_abs_diff(vf[[1]], fx$video_cos) < 1e-6) +expect_true(max_abs_diff(vf[[2]], fx$video_sin) < 1e-6) + +# fp16 input stays fp16 and matches reference values +outv16 <- ltx23_apply_split_rotary_emb(fx$video_x_f16, vf) +expect_equal(outv16$dtype$.type(), "Half") +expect_true(max_abs_diff(outv16, fx$video_out_f16) < 1e-3) + +# --- audio coords + split freqs ------------------------------------------------- + +rope_a <- ltx23_rotary_pos_embed( + dim = 32L, scale_factors = c(8L, 32L, 32L), modality = "audio", + rope_type = "split", num_attention_heads = 4L +) +ac <- rope_a$prepare_audio_coords(batch_size = 2L, num_frames = 5L, device = "cpu") +expect_equal(as.integer(ac$shape), as.integer(fx$audio_coords$shape)) +expect_true(max_abs_diff(ac, fx$audio_coords) < 1e-6) + +af <- rope_a(ac) +expect_true(max_abs_diff(af[[1]], fx$audio_cos) < 1e-6) +expect_true(max_abs_diff(af[[2]], fx$audio_sin) < 1e-6) + +# --- interleaved freqs ----------------------------------------------------------- + +rope_i <- ltx23_rotary_pos_embed( + dim = 64L, modality = "video", rope_type = "interleaved", + num_attention_heads = 4L +) +ifr <- rope_i(vc) +expect_true(max_abs_diff(ifr[[1]], fx$video_inter_cos) < 1e-6) +expect_true(max_abs_diff(ifr[[2]], fx$video_inter_sin) < 1e-6) diff --git a/man/ltx23_apply_interleaved_rotary_emb.Rd b/man/ltx23_apply_interleaved_rotary_emb.Rd new file mode 100644 index 0000000..d191c6d --- /dev/null +++ b/man/ltx23_apply_interleaved_rotary_emb.Rd @@ -0,0 +1,20 @@ +% tinyrox says don't edit this manually, but it can't stop you! +\name{ltx23_apply_interleaved_rotary_emb} +\alias{ltx23_apply_interleaved_rotary_emb} +\title{Apply interleaved rotary embeddings} +\usage{ +ltx23_apply_interleaved_rotary_emb(x, freqs) +} +\arguments{ +\item{x}{Tensor of shape [B, S, C].} + +\item{freqs}{List of two tensors (cos, sin), each [B, S, C].} +} +\value{ +Tensor with the same shape and dtype as \code{x}. +} +\description{ +Rotates adjacent element pairs of the last dimension: +\code{out = x * cos + rotate_half(x) * sin} with pairs interleaved +(elements 1,2 form the first complex pair). +} diff --git a/man/ltx23_apply_split_rotary_emb.Rd b/man/ltx23_apply_split_rotary_emb.Rd new file mode 100644 index 0000000..a1d9f57 --- /dev/null +++ b/man/ltx23_apply_split_rotary_emb.Rd @@ -0,0 +1,21 @@ +% tinyrox says don't edit this manually, but it can't stop you! +\name{ltx23_apply_split_rotary_emb} +\alias{ltx23_apply_split_rotary_emb} +\title{Apply split rotary embeddings} +\usage{ +ltx23_apply_split_rotary_emb(x, freqs) +} +\arguments{ +\item{x}{Tensor of shape [B, H, T, D] (per-head layout), or [B, T, H*D] +which is reshaped per-head when \code{freqs} is 4D.} + +\item{freqs}{List of two tensors (cos, sin), each [B, H, T, D/2].} +} +\value{ +Tensor with the same shape and dtype as \code{x}. +} +\description{ +Rotates element pairs formed by splitting the last dimension in half: +element i pairs with element i + d/2. The cos/sin tensors carry half +the head dimension. +} diff --git a/man/ltx23_rotary_pos_embed.Rd b/man/ltx23_rotary_pos_embed.Rd new file mode 100644 index 0000000..8d6dcdd --- /dev/null +++ b/man/ltx23_rotary_pos_embed.Rd @@ -0,0 +1,47 @@ +% tinyrox says don't edit this manually, but it can't stop you! +\name{ltx23_rotary_pos_embed} +\alias{ltx23_rotary_pos_embed} +\title{LTX-2.3 audio/video rotary position embedder} +\usage{ +ltx23_rotary_pos_embed(dim, patch_size = 1L, patch_size_t = 1L, + base_num_frames = 20L, base_height = 2048L, + base_width = 2048L, sampling_rate = 16000L, + hop_length = 160L, scale_factors = c(8L, 32L, 32L), + theta = 10000, causal_offset = 1L, modality = "video", + double_precision = TRUE, rope_type = "split", + num_attention_heads = 32L) +} +\arguments{ +\item{dim}{Integer. Rotary dimension (attention head dim x heads for +split type at model level; see reference).} + +\item{scale_factors}{Integer vector. VAE (time, height, width) scale +factors.} + +\item{theta}{Numeric. RoPE theta.} + +\item{causal_offset}{Integer. Temporal offset for the causal VAE +(first frame has stride 1).} + +\item{modality}{"video" or "audio".} + +\item{double_precision}{Logical. Compute base frequencies in float64.} + +\item{rope_type}{"split" (LTX 2.3) or "interleaved".} + +\item{num_attention_heads}{Integer. Needed for the split layout.} + +\item{patch_size,patch_size_t}{Integers. Spatial/temporal patch sizes.} + +\item{base_num_frames,base_height,base_width}{Integers. Base grid the +coordinates are normalized against.} + +\item{sampling_rate,hop_length}{Integers. Audio spectrogram params.} +} +\description{ +Computes RoPE cos/sin frequency tensors from spatiotemporal patch +coordinates. Video coordinates are 3D (frames scaled to seconds via +fps, height, width in pixel space); audio coordinates are 1D +(seconds). Coordinates are patch boundaries [start, end); the midpoint +is used as the position. +} diff --git a/man/rope_ltx23.Rd b/man/rope_ltx23.Rd new file mode 100644 index 0000000..8f0479d --- /dev/null +++ b/man/rope_ltx23.Rd @@ -0,0 +1,12 @@ +% tinyrox says don't edit this manually, but it can't stop you! +\name{rope_ltx23} +\alias{rope_ltx23} +\title{LTX-2.3 Rotary Positional Embeddings} +\description{ +Fresh R port of the LTX rotary positional embedding scheme from the +diffusers reference implementation (Apache-2.0, +src/diffusers/models/transformers/transformer_ltx2.py). LTX 2.3 uses +the "split" RoPE layout everywhere; "interleaved" is kept for +completeness. Frequencies are computed in float64 per the checkpoint +config (frequencies_precision) and applied in float32. +} diff --git a/tools/gen_fixtures.sh b/tools/gen_fixtures.sh new file mode 100755 index 0000000..e3f2f73 --- /dev/null +++ b/tools/gen_fixtures.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Run LTX-2.3 fixture generators against the diffusers reference checkout. +# Usage: tools/gen_fixtures.sh [script.py ...] (default: all gen_fixtures_*.py) +# +# Uses uv with a CPU-only torch index; never touches the system Python. +set -euo pipefail +cd "$(dirname "$0")/.." + +scripts=("$@") +if [ ${#scripts[@]} -eq 0 ]; then + scripts=(tools/gen_fixtures_*.py) +fi + +for s in "${scripts[@]}"; do + echo "== $s" + uv run --no-project \ + --index https://download.pytorch.org/whl/cpu \ + --index-strategy unsafe-best-match \ + --with torch \ + --with numpy \ + --with safetensors \ + --with huggingface_hub \ + --with packaging \ + --with filelock \ + --with regex \ + --with requests \ + --with tqdm \ + --with pillow \ + python "$s" +done diff --git a/tools/gen_fixtures_rope.py b/tools/gen_fixtures_rope.py new file mode 100644 index 0000000..c6d164f --- /dev/null +++ b/tools/gen_fixtures_rope.py @@ -0,0 +1,109 @@ +# Generate RoPE parity fixtures for the LTX-2.3 R port. +# +# Runs the diffusers reference implementation (Apache-2.0) on small fixed +# inputs and saves {inputs, expected outputs} as safetensors fixtures that +# the R tinytest suite compares against. Run via tools/gen_fixtures.sh; +# never executed at package test/run time. + +import os +import sys + +import torch +from safetensors.torch import save_file + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "ref", "upstream", "diffusers", "src")) + +from diffusers.models.transformers.transformer_ltx2 import ( # noqa: E402 + LTX2AudioVideoRotaryPosEmbed, + apply_interleaved_rotary_emb, + apply_split_rotary_emb, +) + +OUT_DIR = os.path.join(os.path.dirname(__file__), "..", "inst", "tinytest", "fixtures") +os.makedirs(OUT_DIR, exist_ok=True) + +torch.manual_seed(42) +fx = {} + +# --- apply_split_rotary_emb: 4D per-head input ------------------------------- +B, H, T, D = 2, 4, 6, 16 +x4 = torch.randn(B, H, T, D) +cos4 = torch.randn(B, H, T, D // 2) +sin4 = torch.randn(B, H, T, D // 2) +fx["split4_x"] = x4 +fx["split4_cos"] = cos4 +fx["split4_sin"] = sin4 +fx["split4_out"] = apply_split_rotary_emb(x4, (cos4, sin4)) + +# --- apply_split_rotary_emb: 3D input reshaped per-head ---------------------- +x3 = torch.randn(B, T, H * D) +fx["split3_x"] = x3 +fx["split3_out"] = apply_split_rotary_emb(x3, (cos4, sin4)) + +# --- apply_interleaved_rotary_emb --------------------------------------------- +S, C = 6, 16 +xi = torch.randn(B, S, C) +cosi = torch.randn(B, S, C) +sini = torch.randn(B, S, C) +fx["inter_x"] = xi +fx["inter_cos"] = cosi +fx["inter_sin"] = sini +fx["inter_out"] = apply_interleaved_rotary_emb(xi, (cosi, sini)) + +# --- embedder: video coords + split freqs ------------------------------------- +rope_v = LTX2AudioVideoRotaryPosEmbed( + dim=64, + base_num_frames=20, + base_height=2048, + base_width=2048, + scale_factors=(8, 32, 32), + theta=10000.0, + modality="video", + double_precision=True, + rope_type="split", + num_attention_heads=4, +) +vc = rope_v.prepare_video_coords(batch_size=2, num_frames=3, height=4, width=6, device="cpu", fps=24.0) +fx["video_coords"] = vc +vcos, vsin = rope_v(vc) +fx["video_cos"] = vcos +fx["video_sin"] = vsin + +# fp16 dtype preservation through apply (values compared in the R test) +xv = torch.randn(2, 4, 3 * 4 * 6, 16, dtype=torch.float16) +fx["video_x_f16"] = xv +fx["video_out_f16"] = apply_split_rotary_emb(xv, (vcos, vsin)) + +# --- embedder: audio coords + split freqs -------------------------------------- +rope_a = LTX2AudioVideoRotaryPosEmbed( + dim=32, + base_num_frames=20, + sampling_rate=16000, + hop_length=160, + scale_factors=(8, 32, 32), + modality="audio", + double_precision=True, + rope_type="split", + num_attention_heads=4, +) +ac = rope_a.prepare_audio_coords(batch_size=2, num_frames=5, device="cpu", shift=0) +fx["audio_coords"] = ac +acos, asin = rope_a(ac) +fx["audio_cos"] = acos +fx["audio_sin"] = asin + +# --- embedder: interleaved variant --------------------------------------------- +rope_i = LTX2AudioVideoRotaryPosEmbed( + dim=64, + modality="video", + double_precision=True, + rope_type="interleaved", + num_attention_heads=4, +) +icos, isin = rope_i(vc) +fx["video_inter_cos"] = icos +fx["video_inter_sin"] = isin + +fx = {k: v.contiguous() for k, v in fx.items()} +save_file(fx, os.path.join(OUT_DIR, "rope_ltx23.safetensors")) +print(f"wrote {len(fx)} tensors to {OUT_DIR}/rope_ltx23.safetensors") From 6a72a8888923cd1b816ac7b3f3dee2d4d8ba2c07 Mon Sep 17 00:00:00 2001 From: TroyHernandez Date: Fri, 3 Jul 2026 11:12:34 -0500 Subject: [PATCH 05/83] Port LTX-2.3 audio-video DiT from diffusers Dual-stream transformer with gated attention, 9-param cross-attention modulation, prompt AdaLN, split RoPE, a2v/v2a cross attention with global+per-block modulation, STG perturbation, and optional query chunking for the manual SDPA. Includes the official-checkpoint key mapper. Parity-tested against a tiny reference model at 1e-4. --- NAMESPACE | 8 + R/dit_ltx23.R | 405 ++++++++++++ R/dit_ltx23_modules.R | 641 +++++++++++++++++++ inst/tinytest/fixtures/dit_ltx23.safetensors | Bin 0 -> 251372 bytes inst/tinytest/test_dit_ltx23.R | 183 ++++++ man/dit_ltx23.Rd | 11 + man/dit_ltx23_modules.Rd | 10 + man/ltx23_ada_layer_norm_single.Rd | 20 + man/ltx23_attention.Rd | 36 ++ man/ltx23_feed_forward.Rd | 17 + man/ltx23_get_timestep_embedding.Rd | 26 + man/ltx23_map_dit_key.Rd | 19 + man/ltx23_rms_norm.Rd | 18 + man/ltx23_transformer.Rd | 65 ++ man/ltx23_transformer_block.Rd | 44 ++ tools/gen_fixtures_dit.py | 167 +++++ 16 files changed, 1670 insertions(+) create mode 100644 R/dit_ltx23.R create mode 100644 R/dit_ltx23_modules.R create mode 100644 inst/tinytest/fixtures/dit_ltx23.safetensors create mode 100644 inst/tinytest/test_dit_ltx23.R create mode 100644 man/dit_ltx23.Rd create mode 100644 man/dit_ltx23_modules.Rd create mode 100644 man/ltx23_ada_layer_norm_single.Rd create mode 100644 man/ltx23_attention.Rd create mode 100644 man/ltx23_feed_forward.Rd create mode 100644 man/ltx23_get_timestep_embedding.Rd create mode 100644 man/ltx23_map_dit_key.Rd create mode 100644 man/ltx23_rms_norm.Rd create mode 100644 man/ltx23_transformer.Rd create mode 100644 man/ltx23_transformer_block.Rd create mode 100644 tools/gen_fixtures_dit.py diff --git a/NAMESPACE b/NAMESPACE index 8cea91a..de614cf 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -32,13 +32,21 @@ export(load_text_encoder2_weights) export(load_to_gpu) export(load_unet_sdxl_weights) export(load_unet_weights) +export(ltx23_ada_layer_norm_single) export(ltx23_apply_interleaved_rotary_emb) export(ltx23_apply_split_rotary_emb) +export(ltx23_attention) export(ltx23_census) +export(ltx23_feed_forward) +export(ltx23_get_timestep_embedding) export(ltx23_load_group) +export(ltx23_map_dit_key) export(ltx23_open_checkpoint) +export(ltx23_rms_norm) export(ltx23_rotary_pos_embed) export(ltx23_split_keys) +export(ltx23_transformer) +export(ltx23_transformer_block) export(models2devices) export(offload_to_cpu) export(post_quant_conv) diff --git a/R/dit_ltx23.R b/R/dit_ltx23.R new file mode 100644 index 0000000..9e1d70b --- /dev/null +++ b/R/dit_ltx23.R @@ -0,0 +1,405 @@ +#' LTX-2.3 Audio-Video Diffusion Transformer +#' +#' Fresh R port of the LTX-2 transformer from the diffusers reference +#' (Apache-2.0, src/diffusers/models/transformers/transformer_ltx2.py), +#' configured for LTX 2.3: gated attention, cross-attention modulation, +#' prompt AdaLN, split RoPE, and connector-projected text embeddings +#' (no in-model caption projection). +#' +#' @name dit_ltx23 +NULL + +#' LTX-2.3 video transformer model +#' +#' Dual-stream audio/video DiT. Text embeddings arrive already projected +#' to the video (\code{inner_dim}) and audio (\code{audio_inner_dim}) +#' dimensions by the connector modules. +#' +#' @param in_channels,out_channels Integers. Video latent channels. +#' @param patch_size,patch_size_t Integers. Video patch sizes. +#' @param num_attention_heads,attention_head_dim Video attention shape. +#' @param cross_attention_dim Integer. Video text embedding dimension. +#' @param vae_scale_factors Integer vector. VAE (time, height, width) scales. +#' @param pos_embed_max_pos,base_height,base_width RoPE base grid. +#' @param audio_in_channels,audio_out_channels Integers. Audio latent channels. +#' @param audio_patch_size,audio_patch_size_t Integers. Audio patch sizes. +#' @param audio_num_attention_heads,audio_attention_head_dim Audio attention shape. +#' @param audio_cross_attention_dim Integer. Audio text embedding dimension. +#' @param audio_scale_factor,audio_pos_embed_max_pos,audio_sampling_rate,audio_hop_length +#' Audio latent grid parameters. +#' @param num_layers Integer. Transformer block count. +#' @param norm_eps Numeric. Norm epsilon. +#' @param rope_theta,rope_double_precision,causal_offset RoPE parameters. +#' @param timestep_scale_multiplier,cross_attn_timestep_scale_multiplier +#' Timestep scaling (inputs arrive already scaled; the ratio modulates +#' the a2v/v2a gates). +#' @param rope_type "split" (LTX-2.3) or "interleaved". +#' @param gated_attn,cross_attn_mod,audio_gated_attn,audio_cross_attn_mod,perturbed_attn +#' LTX-2.3 feature flags (all TRUE for the 2.3 checkpoints). +#' +#' @export +ltx23_transformer <- torch::nn_module( + "ltx23_transformer", + initialize = function( + in_channels = 128L, + out_channels = 128L, + patch_size = 1L, + patch_size_t = 1L, + num_attention_heads = 32L, + attention_head_dim = 128L, + cross_attention_dim = 4096L, + vae_scale_factors = c(8L, 32L, 32L), + pos_embed_max_pos = 20L, + base_height = 2048L, + base_width = 2048L, + gated_attn = TRUE, + cross_attn_mod = TRUE, + audio_in_channels = 128L, + audio_out_channels = 128L, + audio_patch_size = 1L, + audio_patch_size_t = 1L, + audio_num_attention_heads = 32L, + audio_attention_head_dim = 64L, + audio_cross_attention_dim = 2048L, + audio_scale_factor = 4L, + audio_pos_embed_max_pos = 20L, + audio_sampling_rate = 16000L, + audio_hop_length = 160L, + audio_gated_attn = TRUE, + audio_cross_attn_mod = TRUE, + num_layers = 48L, + norm_eps = 1e-6, + rope_theta = 10000.0, + rope_double_precision = TRUE, + causal_offset = 1L, + timestep_scale_multiplier = 1000, + cross_attn_timestep_scale_multiplier = 1000, + rope_type = "split", + perturbed_attn = TRUE + ) { + inner_dim <- num_attention_heads * attention_head_dim + audio_inner_dim <- audio_num_attention_heads * audio_attention_head_dim + self$inner_dim <- inner_dim + self$audio_inner_dim <- audio_inner_dim + self$timestep_scale_multiplier <- timestep_scale_multiplier + self$cross_attn_timestep_scale_multiplier <- cross_attn_timestep_scale_multiplier + self$prompt_modulation <- cross_attn_mod || audio_cross_attn_mod + + # 1. Patchified input projections + self$proj_in <- torch::nn_linear(in_channels, inner_dim) + self$audio_proj_in <- torch::nn_linear(audio_in_channels, audio_inner_dim) + + # 2. Global timestep modulation (9 params each with cross-attn mod) + video_mod_params <- if (cross_attn_mod) 9L else 6L + audio_mod_params <- if (audio_cross_attn_mod) 9L else 6L + self$time_embed <- ltx23_ada_layer_norm_single(inner_dim, video_mod_params) + self$audio_time_embed <- ltx23_ada_layer_norm_single(audio_inner_dim, audio_mod_params) + + # Global a2v/v2a cross-attention modulation + self$av_cross_attn_video_scale_shift <- ltx23_ada_layer_norm_single(inner_dim, 4L) + self$av_cross_attn_audio_scale_shift <- ltx23_ada_layer_norm_single(audio_inner_dim, 4L) + self$av_cross_attn_video_a2v_gate <- ltx23_ada_layer_norm_single(inner_dim, 1L) + self$av_cross_attn_audio_v2a_gate <- ltx23_ada_layer_norm_single(audio_inner_dim, 1L) + + # Output layer modulation + self$scale_shift_table <- torch::nn_parameter( + torch::torch_randn(2L, inner_dim) / sqrt(inner_dim) + ) + self$audio_scale_shift_table <- torch::nn_parameter( + torch::torch_randn(2L, audio_inner_dim) / sqrt(audio_inner_dim) + ) + + # Prompt modulation (LTX-2.3) + if (self$prompt_modulation) { + self$prompt_adaln <- ltx23_ada_layer_norm_single(inner_dim, 2L) + self$audio_prompt_adaln <- ltx23_ada_layer_norm_single(audio_inner_dim, 2L) + } + + # 3. RoPE embedders (self-attention per modality + a2v/v2a cross) + self$rope <- ltx23_rotary_pos_embed( + dim = inner_dim, patch_size = patch_size, patch_size_t = patch_size_t, + base_num_frames = pos_embed_max_pos, base_height = base_height, + base_width = base_width, scale_factors = vae_scale_factors, + theta = rope_theta, causal_offset = causal_offset, modality = "video", + double_precision = rope_double_precision, rope_type = rope_type, + num_attention_heads = num_attention_heads + ) + self$audio_rope <- ltx23_rotary_pos_embed( + dim = audio_inner_dim, patch_size = audio_patch_size, + patch_size_t = audio_patch_size_t, base_num_frames = audio_pos_embed_max_pos, + sampling_rate = audio_sampling_rate, hop_length = audio_hop_length, + scale_factors = audio_scale_factor, + theta = rope_theta, causal_offset = causal_offset, modality = "audio", + double_precision = rope_double_precision, rope_type = rope_type, + num_attention_heads = audio_num_attention_heads + ) + ca_max_pos <- max(pos_embed_max_pos, audio_pos_embed_max_pos) + self$cross_attn_rope <- ltx23_rotary_pos_embed( + dim = audio_cross_attention_dim, patch_size = patch_size, + patch_size_t = patch_size_t, base_num_frames = ca_max_pos, + base_height = base_height, base_width = base_width, + theta = rope_theta, causal_offset = causal_offset, modality = "video", + double_precision = rope_double_precision, rope_type = rope_type, + num_attention_heads = num_attention_heads + ) + self$cross_attn_audio_rope <- ltx23_rotary_pos_embed( + dim = audio_cross_attention_dim, patch_size = audio_patch_size, + patch_size_t = audio_patch_size_t, base_num_frames = ca_max_pos, + sampling_rate = audio_sampling_rate, hop_length = audio_hop_length, + theta = rope_theta, causal_offset = causal_offset, modality = "audio", + double_precision = rope_double_precision, rope_type = rope_type, + num_attention_heads = audio_num_attention_heads + ) + + # 4. Transformer blocks + self$transformer_blocks <- torch::nn_module_list(lapply(seq_len(num_layers), function(i) { + ltx23_transformer_block( + dim = inner_dim, + num_attention_heads = num_attention_heads, + attention_head_dim = attention_head_dim, + cross_attention_dim = cross_attention_dim, + audio_dim = audio_inner_dim, + audio_num_attention_heads = audio_num_attention_heads, + audio_attention_head_dim = audio_attention_head_dim, + audio_cross_attention_dim = audio_cross_attention_dim, + video_gated_attn = gated_attn, + video_cross_attn_adaln = cross_attn_mod, + audio_gated_attn = audio_gated_attn, + audio_cross_attn_adaln = audio_cross_attn_mod, + eps = norm_eps, + elementwise_affine = FALSE, + rope_type = rope_type, + perturbed_attn = perturbed_attn + ) + })) + + # 5. Output layers + self$norm_out <- torch::nn_layer_norm(inner_dim, eps = 1e-6, elementwise_affine = FALSE) + self$proj_out <- torch::nn_linear(inner_dim, out_channels) + self$audio_norm_out <- torch::nn_layer_norm(audio_inner_dim, eps = 1e-6, + elementwise_affine = FALSE) + self$audio_proj_out <- torch::nn_linear(audio_inner_dim, audio_out_channels) + }, + forward = function( + hidden_states, + audio_hidden_states, + encoder_hidden_states, + audio_encoder_hidden_states, + timestep, + audio_timestep = NULL, + sigma = NULL, + audio_sigma = NULL, + encoder_attention_mask = NULL, + audio_encoder_attention_mask = NULL, + num_frames = NULL, + height = NULL, + width = NULL, + fps = 24.0, + audio_num_frames = NULL, + video_coords = NULL, + audio_coords = NULL, + isolate_modalities = FALSE, + spatio_temporal_guidance_blocks = NULL, + perturbation_mask = NULL, + use_cross_timestep = FALSE, + video_self_attention_mask = NULL + ) { + audio_timestep <- audio_timestep %||% timestep + audio_sigma <- audio_sigma %||% sigma + if (self$prompt_modulation && is.null(sigma)) { + stop("sigma is required for LTX-2.3 prompt modulation") + } + + # Multiplicative [B, S] masks -> additive bias [B, 1, S] + if (!is.null(encoder_attention_mask) && encoder_attention_mask$ndim == 2L) { + encoder_attention_mask <- ((1 - encoder_attention_mask$to( + dtype = hidden_states$dtype + )) * -10000.0)$unsqueeze(2L) + } + if (!is.null(audio_encoder_attention_mask) && audio_encoder_attention_mask$ndim == 2L) { + audio_encoder_attention_mask <- ((1 - audio_encoder_attention_mask$to( + dtype = audio_hidden_states$dtype + )) * -10000.0)$unsqueeze(2L) + } + if (!is.null(video_self_attention_mask)) { + video_self_attention_mask <- (1 - video_self_attention_mask$to( + dtype = hidden_states$dtype + )) * -10000.0 + } + + batch_size <- hidden_states$shape[1] + + # 1. RoPE frequencies + if (is.null(video_coords)) { + video_coords <- self$rope$prepare_video_coords( + batch_size, num_frames, height, width, hidden_states$device, fps = fps + ) + } + if (is.null(audio_coords)) { + audio_coords <- self$audio_rope$prepare_audio_coords( + batch_size, audio_num_frames, audio_hidden_states$device + ) + } + video_rotary_emb <- self$rope(video_coords, device = hidden_states$device) + audio_rotary_emb <- self$audio_rope(audio_coords, device = audio_hidden_states$device) + # Cross-modal attention uses the temporal axis only + video_ca_rotary_emb <- self$cross_attn_rope( + video_coords$narrow(2L, 1L, 1L), device = hidden_states$device + ) + audio_ca_rotary_emb <- self$cross_attn_audio_rope( + audio_coords$narrow(2L, 1L, 1L), device = audio_hidden_states$device + ) + + # 2. Input projections + hidden_states <- self$proj_in(hidden_states) + audio_hidden_states <- self$audio_proj_in(audio_hidden_states) + + # 3. Timestep embeddings and global modulation parameters + gate_scale_factor <- self$cross_attn_timestep_scale_multiplier / + self$timestep_scale_multiplier + + te <- self$time_embed(timestep$flatten(), hidden_dtype = hidden_states$dtype) + temb <- te[[1]]$view(c(batch_size, -1L, te[[1]]$shape[2])) + embedded_timestep <- te[[2]]$view(c(batch_size, -1L, te[[2]]$shape[2])) + + ate <- self$audio_time_embed(audio_timestep$flatten(), + hidden_dtype = audio_hidden_states$dtype) + temb_audio <- ate[[1]]$view(c(batch_size, -1L, ate[[1]]$shape[2])) + audio_embedded_timestep <- ate[[2]]$view(c(batch_size, -1L, ate[[2]]$shape[2])) + + if (self$prompt_modulation) { + pe <- self$prompt_adaln(sigma$flatten(), hidden_dtype = hidden_states$dtype) + temb_prompt <- pe[[1]]$view(c(batch_size, -1L, pe[[1]]$shape[2])) + ape <- self$audio_prompt_adaln(audio_sigma$flatten(), + hidden_dtype = audio_hidden_states$dtype) + temb_prompt_audio <- ape[[1]]$view(c(batch_size, -1L, ape[[1]]$shape[2])) + } else { + temb_prompt <- NULL + temb_prompt_audio <- NULL + } + + # a2v/v2a modulation; 2.3 modulates each modality by the *other* + # modality's sigma (use_cross_timestep) + video_ca_timestep <- if (use_cross_timestep) audio_sigma$flatten() else timestep$flatten() + vcss <- self$av_cross_attn_video_scale_shift( + video_ca_timestep, hidden_dtype = hidden_states$dtype + )[[1]] + vcg <- self$av_cross_attn_video_a2v_gate( + video_ca_timestep * gate_scale_factor, hidden_dtype = hidden_states$dtype + )[[1]] + video_ca_scale_shift <- vcss$view(c(batch_size, -1L, vcss$shape[2])) + video_ca_a2v_gate <- vcg$view(c(batch_size, -1L, vcg$shape[2])) + + audio_ca_timestep <- if (use_cross_timestep) sigma$flatten() else audio_timestep$flatten() + acss <- self$av_cross_attn_audio_scale_shift( + audio_ca_timestep, hidden_dtype = audio_hidden_states$dtype + )[[1]] + acg <- self$av_cross_attn_audio_v2a_gate( + audio_ca_timestep * gate_scale_factor, hidden_dtype = audio_hidden_states$dtype + )[[1]] + audio_ca_scale_shift <- acss$view(c(batch_size, -1L, acss$shape[2])) + audio_ca_v2a_gate <- acg$view(c(batch_size, -1L, acg$shape[2])) + + # 4. STG perturbation setup + stg_blocks <- spatio_temporal_guidance_blocks %||% integer(0) + if (length(stg_blocks) > 0L && is.null(perturbation_mask)) { + perturbation_mask <- torch::torch_zeros(batch_size, device = hidden_states$device) + } + if (!is.null(perturbation_mask) && perturbation_mask$ndim == 1L) { + perturbation_mask <- perturbation_mask$unsqueeze(2L)$unsqueeze(3L) + } + all_perturbed <- if (!is.null(perturbation_mask)) { + as.logical((perturbation_mask == 0)$all()$item()) + } else { + FALSE + } + + # 5. Transformer blocks + fp8_mode <- isTRUE(getOption("diffuseR.use_fp8")) + for (block_idx in seq_along(self$transformer_blocks)) { + is_stg_block <- (block_idx - 1L) %in% stg_blocks# reference indices are 0-based + res <- self$transformer_blocks[[block_idx]]( + hidden_states = hidden_states, + audio_hidden_states = audio_hidden_states, + encoder_hidden_states = encoder_hidden_states, + audio_encoder_hidden_states = audio_encoder_hidden_states, + temb = temb, + temb_audio = temb_audio, + temb_ca_scale_shift = video_ca_scale_shift, + temb_ca_audio_scale_shift = audio_ca_scale_shift, + temb_ca_gate = video_ca_a2v_gate, + temb_ca_audio_gate = audio_ca_v2a_gate, + temb_prompt = temb_prompt, + temb_prompt_audio = temb_prompt_audio, + video_rotary_emb = video_rotary_emb, + audio_rotary_emb = audio_rotary_emb, + ca_video_rotary_emb = video_ca_rotary_emb, + ca_audio_rotary_emb = audio_ca_rotary_emb, + encoder_attention_mask = encoder_attention_mask, + audio_encoder_attention_mask = audio_encoder_attention_mask, + self_attention_mask = video_self_attention_mask, + use_a2v_cross_attention = !isolate_modalities, + use_v2a_cross_attention = !isolate_modalities, + perturbation_mask = if (is_stg_block) perturbation_mask else NULL, + all_perturbed = if (is_stg_block) all_perturbed else FALSE + ) + hidden_states <- res[[1]] + audio_hidden_states <- res[[2]] + if (fp8_mode) { + # Dequantized fp8 temporaries only free once R's GC finalizes them + gc(verbose = FALSE) + } + } + + # 6. Output layers + scale_shift_values <- self$scale_shift_table$unsqueeze(1L)$unsqueeze(1L) + + embedded_timestep$unsqueeze(3L) + shift <- scale_shift_values[, , 1, ] + scale <- scale_shift_values[, , 2, ] + hidden_states <- self$norm_out(hidden_states) + hidden_states <- hidden_states * (1 + scale) + shift + output <- self$proj_out(hidden_states) + + audio_scale_shift_values <- self$audio_scale_shift_table$unsqueeze(1L)$unsqueeze(1L) + + audio_embedded_timestep$unsqueeze(3L) + audio_shift <- audio_scale_shift_values[, , 1, ] + audio_scale <- audio_scale_shift_values[, , 2, ] + audio_hidden_states <- self$audio_norm_out(audio_hidden_states) + audio_hidden_states <- audio_hidden_states * (1 + audio_scale) + audio_shift + audio_output <- self$audio_proj_out(audio_hidden_states) + + list(sample = output, audio_sample = audio_output) + } +) + +#' Map an official DiT checkpoint key to the R module name +#' +#' Applies the official-to-diffusers renames for the LTX-2.3 transformer +#' (cf. diffusers scripts/convert_ltx2_to_diffusers.py). Our module tree +#' matches the diffusers names, so this is the full mapping. +#' +#' @param key Character. Checkpoint key (with or without the +#' \code{model.diffusion_model.} prefix). +#' +#' @return Character. Module parameter/buffer name. +#' +#' @export +ltx23_map_dit_key <- function(key) { + key <- sub("^model\\.diffusion_model\\.", "", key) + + # Substring renames; *_adaln_single variants must precede the bare + # adaln_single prefix renames below + key <- gsub("av_ca_video_scale_shift_adaln_single", "av_cross_attn_video_scale_shift", key, fixed = TRUE) + key <- gsub("av_ca_a2v_gate_adaln_single", "av_cross_attn_video_a2v_gate", key, fixed = TRUE) + key <- gsub("av_ca_audio_scale_shift_adaln_single", "av_cross_attn_audio_scale_shift", key, fixed = TRUE) + key <- gsub("av_ca_v2a_gate_adaln_single", "av_cross_attn_audio_v2a_gate", key, fixed = TRUE) + key <- gsub("scale_shift_table_a2v_ca_video", "video_a2v_cross_attn_scale_shift_table", key, fixed = TRUE) + key <- gsub("scale_shift_table_a2v_ca_audio", "audio_a2v_cross_attn_scale_shift_table", key, fixed = TRUE) + key <- gsub("prompt_adaln_single", "prompt_adaln", key, fixed = TRUE) + key <- sub("^audio_adaln_single\\.", "audio_time_embed.", key) + key <- sub("^adaln_single\\.", "time_embed.", key) + key <- gsub("patchify_proj", "proj_in", key, fixed = TRUE) + key <- gsub("q_norm", "norm_q", key, fixed = TRUE) + key <- gsub("k_norm", "norm_k", key, fixed = TRUE) + key +} diff --git a/R/dit_ltx23_modules.R b/R/dit_ltx23_modules.R new file mode 100644 index 0000000..637e384 --- /dev/null +++ b/R/dit_ltx23_modules.R @@ -0,0 +1,641 @@ +#' LTX-2.3 Transformer Building Blocks +#' +#' Fresh R port of the LTX-2 transformer components from the diffusers +#' reference (Apache-2.0, src/diffusers/models/transformers/ +#' transformer_ltx2.py and shared modules). Field names mirror the +#' diffusers module tree so checkpoint keys map 1:1. +#' +#' @name dit_ltx23_modules +NULL + +#' RMS normalization +#' +#' Variance is computed in float32; the result is cast back to the input +#' dtype (or the weight dtype when elementwise affine). +#' +#' @param dim Integer. Normalized dimension size. +#' @param eps Numeric. Stability epsilon. +#' @param elementwise_affine Logical. Learn a scale weight. +#' +#' @export +ltx23_rms_norm <- torch::nn_module( + "ltx23_rms_norm", + initialize = function(dim, eps = 1e-6, elementwise_affine = TRUE) { + self$eps <- eps + self$elementwise_affine <- elementwise_affine + if (elementwise_affine) { + self$weight <- torch::nn_parameter(torch::torch_ones(dim)) + } + }, + forward = function(x) { + input_dtype <- x$dtype + variance <- x$to(dtype = torch::torch_float32())$pow(2)$mean(dim = -1L, keepdim = TRUE) + x <- x * torch::torch_rsqrt(variance + self$eps) + if (self$elementwise_affine) { + x <- x$to(dtype = self$weight$dtype) * self$weight + } else { + x <- x$to(dtype = input_dtype) + } + x + } +) + +#' Sinusoidal timestep embedding +#' +#' DDPM-style sinusoidal embedding. LTX uses \code{flip_sin_to_cos=TRUE} +#' (cos first) and \code{downscale_freq_shift=0}. +#' +#' @param timesteps 1D tensor of timestep values. +#' @param embedding_dim Integer. Output embedding size. +#' @param flip_sin_to_cos Logical. Put cos before sin. +#' @param downscale_freq_shift Numeric. Frequency delta control. +#' @param max_period Numeric. Maximum embedding frequency period. +#' +#' @return Tensor [N, embedding_dim]. +#' +#' @export +ltx23_get_timestep_embedding <- function( + timesteps, + embedding_dim, + flip_sin_to_cos = TRUE, + downscale_freq_shift = 0, + max_period = 10000 +) { + stopifnot(timesteps$ndim == 1L) + half_dim <- embedding_dim %/% 2L + exponent <- -log(max_period) * torch::torch_arange( + start = 0, end = half_dim - 1, + dtype = torch::torch_float32(), device = timesteps$device + ) + exponent <- exponent / (half_dim - downscale_freq_shift) + + emb <- torch::torch_exp(exponent) + emb <- timesteps$unsqueeze(2L)$to(dtype = torch::torch_float32()) * emb$unsqueeze(1L) + emb <- torch::torch_cat(list(torch::torch_sin(emb), torch::torch_cos(emb)), dim = -1L) + + if (flip_sin_to_cos) { + emb <- torch::torch_cat( + list(emb[, (half_dim + 1):(2 * half_dim)], emb[, 1:half_dim]), + dim = -1L + ) + } + if (embedding_dim %% 2L == 1L) { + emb <- torch::nnf_pad(emb, c(0L, 1L, 0L, 0L)) + } + emb +} + +# Two-layer timestep MLP (linear_1 -> silu -> linear_2), matching +# diffusers TimestepEmbedding state-dict names. +ltx23_timestep_embedding <- torch::nn_module( + "ltx23_timestep_embedding", + initialize = function(in_channels, time_embed_dim) { + self$linear_1 <- torch::nn_linear(in_channels, time_embed_dim) + self$linear_2 <- torch::nn_linear(time_embed_dim, time_embed_dim) + }, + forward = function(sample) { + self$linear_2(torch::nnf_silu(self$linear_1(sample))) + } +) + +# Sinusoidal projection (256 channels) + MLP, matching diffusers +# PixArtAlphaCombinedTimestepSizeEmbeddings without additional conditions. +ltx23_combined_timestep_embed <- torch::nn_module( + "ltx23_combined_timestep_embed", + initialize = function(embedding_dim) { + self$timestep_embedder <- ltx23_timestep_embedding(256L, embedding_dim) + }, + forward = function(timestep, hidden_dtype) { + proj <- ltx23_get_timestep_embedding( + timestep, 256L, + flip_sin_to_cos = TRUE, downscale_freq_shift = 0 + ) + self$timestep_embedder(proj$to(dtype = hidden_dtype)) + } +) + +#' Adaptive layer norm single (adaLN-single) +#' +#' Embeds a timestep/sigma and projects it to a configurable number of +#' modulation parameter vectors. +#' +#' @param embedding_dim Integer. Model dimension. +#' @param num_mod_params Integer. Number of modulation parameter vectors. +#' +#' @return Module whose forward returns +#' \code{list(mod_params [N, num_mod_params * dim], embedded_timestep [N, dim])}. +#' +#' @export +ltx23_ada_layer_norm_single <- torch::nn_module( + "ltx23_ada_layer_norm_single", + initialize = function(embedding_dim, num_mod_params = 6L) { + self$num_mod_params <- num_mod_params + self$emb <- ltx23_combined_timestep_embed(embedding_dim) + self$linear <- torch::nn_linear(embedding_dim, num_mod_params * embedding_dim, bias = TRUE) + }, + forward = function(timestep, hidden_dtype) { + embedded_timestep <- self$emb(timestep, hidden_dtype = hidden_dtype) + list(self$linear(torch::nnf_silu(embedded_timestep)), embedded_timestep) + } +) + +# Scaled dot-product attention on [B, H, S, D] tensors. R torch has no +# fused SDPA, so the [B, H, Sq, Sk] attention matrix materializes; at high +# resolution chunk the queries to bound peak memory. +.ltx23_sdpa <- function(query, key, value, attention_mask = NULL, chunk_size = NULL) { + head_dim <- query$shape[length(query$shape)] + scale <- 1.0 / sqrt(head_dim) + key_t <- key$transpose(-2L, -1L) + + attend <- function(q, mask) { + attn <- torch::torch_matmul(q * scale, key_t) + if (!is.null(mask)) attn <- attn + mask + attn <- torch::nnf_softmax(attn, dim = -1L) + torch::torch_matmul(attn, value) + } + + n_q <- query$shape[3] + if (is.null(chunk_size) || n_q <= chunk_size) { + return(attend(query, attention_mask)) + } + + outs <- list() + start <- 1L + while (start <= n_q) { + len <- min(chunk_size, n_q - start + 1L) + q_chunk <- query$narrow(3L, start, len) + mask_chunk <- attention_mask + if (!is.null(mask_chunk) && mask_chunk$shape[3] > 1L) { + mask_chunk <- mask_chunk$narrow(3L, start, len) + } + outs[[length(outs) + 1L]] <- attend(q_chunk, mask_chunk) + start <- start + len + } + torch::torch_cat(outs, dim = 3L) +} + +#' LTX-2 attention layer +#' +#' Attention with RMS q/k norms across heads, optional per-head output +#' gating (LTX-2.3), separate query/key RoPE (for a2v/v2a cross +#' attention), and optional STG perturbation (skip attention, use the +#' value projection). +#' +#' @param query_dim Integer. Query feature dimension. +#' @param heads,kv_heads Integers. Attention head counts. +#' @param dim_head Integer. Per-head dimension. +#' @param bias,out_bias Logicals. Projection biases. +#' @param cross_attention_dim Integer or NULL. Key/value input dimension +#' (NULL for self-attention). +#' @param norm_eps Numeric. RMS norm epsilon. +#' @param norm_elementwise_affine Logical. RMS norms carry weights. +#' @param rope_type "split" or "interleaved". +#' @param apply_gated_attention Logical. Add per-head sigmoid output gates. +#' +#' @export +ltx23_attention <- torch::nn_module( + "ltx23_attention", + initialize = function( + query_dim, + heads = 8L, + kv_heads = NULL, + dim_head = 64L, + bias = TRUE, + cross_attention_dim = NULL, + out_bias = TRUE, + norm_eps = 1e-6, + norm_elementwise_affine = TRUE, + rope_type = "split", + apply_gated_attention = FALSE + ) { + kv_heads <- kv_heads %||% heads + self$heads <- as.integer(heads) + self$head_dim <- as.integer(dim_head) + inner_dim <- dim_head * heads + inner_kv_dim <- dim_head * kv_heads + cross_dim <- cross_attention_dim %||% query_dim + self$rope_type <- rope_type + self$attn_chunk <- NULL# set by memory profiles; NULL = unchunked + + self$norm_q <- ltx23_rms_norm(inner_dim, eps = norm_eps, + elementwise_affine = norm_elementwise_affine) + self$norm_k <- ltx23_rms_norm(inner_kv_dim, eps = norm_eps, + elementwise_affine = norm_elementwise_affine) + self$to_q <- torch::nn_linear(query_dim, inner_dim, bias = bias) + self$to_k <- torch::nn_linear(cross_dim, inner_kv_dim, bias = bias) + self$to_v <- torch::nn_linear(cross_dim, inner_kv_dim, bias = bias) + self$to_out <- torch::nn_module_list(list( + torch::nn_linear(inner_dim, query_dim, bias = out_bias) + )) + + self$apply_gated_attention <- apply_gated_attention + if (apply_gated_attention) { + # Per-head gate logits, computed on the raw block input (pre-Q) + self$to_gate_logits <- torch::nn_linear(query_dim, heads, bias = TRUE) + } + }, + forward = function( + hidden_states, + encoder_hidden_states = NULL, + attention_mask = NULL, + query_rotary_emb = NULL, + key_rotary_emb = NULL, + perturbation_mask = NULL, + all_perturbed = FALSE + ) { + if (is.null(encoder_hidden_states)) { + encoder_hidden_states <- hidden_states + } + if (!is.null(attention_mask)) { + # [B, 1, S] or [B, Tq, S] additive bias -> [B, 1, *, S] + if (attention_mask$ndim == 3L) { + attention_mask <- attention_mask$unsqueeze(2L) + } + } + + gate_logits <- NULL + if (self$apply_gated_attention) { + gate_logits <- self$to_gate_logits(hidden_states)# [B, T, H] + } + + value_flat <- self$to_v(encoder_hidden_states) + + if (isTRUE(all_perturbed)) { + # STG: skip attention entirely, use the value projection + out <- value_flat + } else { + query <- self$norm_q(self$to_q(hidden_states)) + key <- self$norm_k(self$to_k(encoder_hidden_states)) + + key_rope <- key_rotary_emb %||% query_rotary_emb + if (!is.null(query_rotary_emb)) { + if (self$rope_type == "interleaved") { + query <- ltx23_apply_interleaved_rotary_emb(query, query_rotary_emb) + key <- ltx23_apply_interleaved_rotary_emb(key, key_rope) + } else { + query <- ltx23_apply_split_rotary_emb(query, query_rotary_emb) + key <- ltx23_apply_split_rotary_emb(key, key_rope) + } + } + + # [B, S, H*D] -> [B, H, S, D] + query <- query$unflatten(3L, c(self$heads, -1L))$transpose(2L, 3L) + key <- key$unflatten(3L, c(self$heads, -1L))$transpose(2L, 3L) + value <- value_flat$unflatten(3L, c(self$heads, -1L))$transpose(2L, 3L) + + out <- .ltx23_sdpa(query, key, value, attention_mask, self$attn_chunk) + out <- out$transpose(2L, 3L)$flatten(start_dim = 3L) + out <- out$to(dtype = hidden_states$dtype) + + if (!is.null(perturbation_mask)) { + # Interpolate between the perturbed (value) and full attention paths + out <- torch::torch_lerp(value_flat, out, perturbation_mask) + } + } + + if (self$apply_gated_attention) { + out <- out$unflatten(3L, c(self$heads, -1L))# [B, T, H, D] + # Factor 2 so zero-initialized gate logits give unit gates + gates <- gate_logits$sigmoid()$mul(2.0)# [B, T, H] + out <- out * gates$unsqueeze(-1L) + out <- out$flatten(start_dim = 3L) + } + + self$to_out[[1]](out) + } +) + +#' LTX feed-forward layer +#' +#' Linear -> GELU (tanh approximation) -> Linear with 4x hidden dim, +#' matching diffusers \code{FeedForward(activation_fn="gelu-approximate")} +#' state-dict names (net.0.proj, net.2). +#' +#' @param dim Integer. Input/output dimension. +#' @param mult Integer. Hidden dimension multiplier. +#' +#' @export +ltx23_feed_forward <- torch::nn_module( + "ltx23_feed_forward", + initialize = function(dim, mult = 4L) { + inner_dim <- as.integer(dim * mult) + gelu_proj <- torch::nn_module( + "ltx23_gelu", + initialize = function(dim_in, dim_out) { + self$proj <- torch::nn_linear(dim_in, dim_out) + }, + forward = function(x) { + torch::nnf_gelu(self$proj(x), approximate = "tanh") + } + ) + self$net <- torch::nn_module_list(list( + gelu_proj(dim, inner_dim), + torch::nn_identity(), # dropout slot in the reference; inference no-op + torch::nn_linear(inner_dim, dim) + )) + }, + forward = function(x) { + self$net[[3]](self$net[[1]](x)) + } +) + +# Combine a per-block scale/shift table with global modulation params: +# ada_values = table[None, None] + temb.reshape(B, temb_tokens, num, -1), +# returned as a list split along the parameter axis. +ltx23_get_mod_params <- function(scale_shift_table, temb, batch_size) { + num_params <- scale_shift_table$shape[1] + ada_values <- scale_shift_table$unsqueeze(1L)$unsqueeze(1L)$to(device = temb$device) + + temb$reshape(c(batch_size, temb$shape[2], num_params, -1L)) + torch::torch_unbind(ada_values, dim = 3L) +} + +#' LTX-2 transformer block +#' +#' Dual-stream (video + audio) block: modulated self-attention per +#' modality, text cross-attention per modality (with LTX-2.3 query and +#' key/value modulation), bidirectional audio-video cross-attention with +#' global+per-block modulation, and modulated feed-forward. +#' +#' @param dim,audio_dim Integers. Video/audio stream dimensions. +#' @param num_attention_heads,attention_head_dim Video attention shape. +#' @param cross_attention_dim Integer. Text embedding dim for video. +#' @param audio_num_attention_heads,audio_attention_head_dim Audio attention shape. +#' @param audio_cross_attention_dim Integer. Text embedding dim for audio. +#' @param video_gated_attn,audio_gated_attn Logicals. Per-head output gates. +#' @param video_cross_attn_adaln,audio_cross_attn_adaln Logicals. LTX-2.3 +#' text cross-attention modulation (9 mod params instead of 6). +#' @param eps Numeric. Norm epsilon. +#' @param elementwise_affine Logical. Block norms carry weights (FALSE for LTX). +#' @param rope_type "split" or "interleaved". +#' @param perturbed_attn Logical. Enable the STG perturbation arguments. +#' +#' @export +ltx23_transformer_block <- torch::nn_module( + "ltx23_transformer_block", + initialize = function( + dim, + num_attention_heads, + attention_head_dim, + cross_attention_dim, + audio_dim, + audio_num_attention_heads, + audio_attention_head_dim, + audio_cross_attention_dim, + video_gated_attn = TRUE, + video_cross_attn_adaln = TRUE, + audio_gated_attn = TRUE, + audio_cross_attn_adaln = TRUE, + eps = 1e-6, + elementwise_affine = FALSE, + rope_type = "split", + perturbed_attn = TRUE + ) { + self$perturbed_attn <- perturbed_attn + + # 1. Self-attention (video and audio) + self$norm1 <- ltx23_rms_norm(dim, eps = eps, elementwise_affine = elementwise_affine) + self$attn1 <- ltx23_attention( + query_dim = dim, heads = num_attention_heads, dim_head = attention_head_dim, + rope_type = rope_type, apply_gated_attention = video_gated_attn + ) + self$audio_norm1 <- ltx23_rms_norm(audio_dim, eps = eps, elementwise_affine = elementwise_affine) + self$audio_attn1 <- ltx23_attention( + query_dim = audio_dim, heads = audio_num_attention_heads, + dim_head = audio_attention_head_dim, + rope_type = rope_type, apply_gated_attention = audio_gated_attn + ) + + # 2. Prompt cross-attention + self$norm2 <- ltx23_rms_norm(dim, eps = eps, elementwise_affine = elementwise_affine) + self$attn2 <- ltx23_attention( + query_dim = dim, cross_attention_dim = cross_attention_dim, + heads = num_attention_heads, dim_head = attention_head_dim, + rope_type = rope_type, apply_gated_attention = video_gated_attn + ) + self$audio_norm2 <- ltx23_rms_norm(audio_dim, eps = eps, elementwise_affine = elementwise_affine) + self$audio_attn2 <- ltx23_attention( + query_dim = audio_dim, cross_attention_dim = audio_cross_attention_dim, + heads = audio_num_attention_heads, dim_head = audio_attention_head_dim, + rope_type = rope_type, apply_gated_attention = audio_gated_attn + ) + + # 3. Audio-to-video (Q: video) and video-to-audio (Q: audio) cross-attention + self$audio_to_video_norm <- ltx23_rms_norm(dim, eps = eps, elementwise_affine = elementwise_affine) + self$audio_to_video_attn <- ltx23_attention( + query_dim = dim, cross_attention_dim = audio_dim, + heads = audio_num_attention_heads, dim_head = audio_attention_head_dim, + rope_type = rope_type, apply_gated_attention = video_gated_attn + ) + self$video_to_audio_norm <- ltx23_rms_norm(audio_dim, eps = eps, elementwise_affine = elementwise_affine) + self$video_to_audio_attn <- ltx23_attention( + query_dim = audio_dim, cross_attention_dim = dim, + heads = audio_num_attention_heads, dim_head = audio_attention_head_dim, + rope_type = rope_type, apply_gated_attention = audio_gated_attn + ) + + # 4. Feed-forward + self$norm3 <- ltx23_rms_norm(dim, eps = eps, elementwise_affine = elementwise_affine) + self$ff <- ltx23_feed_forward(dim) + self$audio_norm3 <- ltx23_rms_norm(audio_dim, eps = eps, elementwise_affine = elementwise_affine) + self$audio_ff <- ltx23_feed_forward(audio_dim) + + # 5. Per-block modulation tables + self$video_cross_attn_adaln <- video_cross_attn_adaln + self$audio_cross_attn_adaln <- audio_cross_attn_adaln + self$cross_attn_adaln <- video_cross_attn_adaln || audio_cross_attn_adaln + video_mod_params <- if (video_cross_attn_adaln) 9L else 6L + audio_mod_params <- if (audio_cross_attn_adaln) 9L else 6L + self$scale_shift_table <- torch::nn_parameter( + torch::torch_randn(video_mod_params, dim) / sqrt(dim) + ) + self$audio_scale_shift_table <- torch::nn_parameter( + torch::torch_randn(audio_mod_params, audio_dim) / sqrt(audio_dim) + ) + if (self$cross_attn_adaln) { + self$prompt_scale_shift_table <- torch::nn_parameter(torch::torch_randn(2L, dim)) + self$audio_prompt_scale_shift_table <- torch::nn_parameter(torch::torch_randn(2L, audio_dim)) + } + self$video_a2v_cross_attn_scale_shift_table <- torch::nn_parameter(torch::torch_randn(5L, dim)) + self$audio_a2v_cross_attn_scale_shift_table <- torch::nn_parameter(torch::torch_randn(5L, audio_dim)) + }, + forward = function( + hidden_states, + audio_hidden_states, + encoder_hidden_states, + audio_encoder_hidden_states, + temb, + temb_audio, + temb_ca_scale_shift, + temb_ca_audio_scale_shift, + temb_ca_gate, + temb_ca_audio_gate, + temb_prompt = NULL, + temb_prompt_audio = NULL, + video_rotary_emb = NULL, + audio_rotary_emb = NULL, + ca_video_rotary_emb = NULL, + ca_audio_rotary_emb = NULL, + encoder_attention_mask = NULL, + audio_encoder_attention_mask = NULL, + self_attention_mask = NULL, + use_a2v_cross_attention = TRUE, + use_v2a_cross_attention = TRUE, + perturbation_mask = NULL, + all_perturbed = FALSE + ) { + batch_size <- hidden_states$shape[1] + + # 1.1 Video self-attention + video_ada <- ltx23_get_mod_params(self$scale_shift_table, temb, batch_size) + shift_msa <- video_ada[[1]]; scale_msa <- video_ada[[2]]; gate_msa <- video_ada[[3]] + shift_mlp <- video_ada[[4]]; scale_mlp <- video_ada[[5]]; gate_mlp <- video_ada[[6]] + + norm_hidden_states <- self$norm1(hidden_states) + norm_hidden_states <- norm_hidden_states * (1 + scale_msa) + shift_msa + + attn_hidden_states <- self$attn1( + norm_hidden_states, + query_rotary_emb = video_rotary_emb, + attention_mask = self_attention_mask, + perturbation_mask = if (self$perturbed_attn) perturbation_mask else NULL, + all_perturbed = if (self$perturbed_attn) all_perturbed else FALSE + ) + hidden_states <- hidden_states + attn_hidden_states * gate_msa + + # 1.2 Audio self-attention + audio_ada <- ltx23_get_mod_params(self$audio_scale_shift_table, temb_audio, batch_size) + audio_shift_msa <- audio_ada[[1]]; audio_scale_msa <- audio_ada[[2]] + audio_gate_msa <- audio_ada[[3]]; audio_shift_mlp <- audio_ada[[4]] + audio_scale_mlp <- audio_ada[[5]]; audio_gate_mlp <- audio_ada[[6]] + + norm_audio_hidden_states <- self$audio_norm1(audio_hidden_states) + norm_audio_hidden_states <- norm_audio_hidden_states * (1 + audio_scale_msa) + audio_shift_msa + + attn_audio_hidden_states <- self$audio_attn1( + norm_audio_hidden_states, + query_rotary_emb = audio_rotary_emb, + perturbation_mask = if (self$perturbed_attn) perturbation_mask else NULL, + all_perturbed = if (self$perturbed_attn) all_perturbed else FALSE + ) + audio_hidden_states <- audio_hidden_states + attn_audio_hidden_states * audio_gate_msa + + # 2. Text cross-attention modulation params (LTX-2.3) + if (self$cross_attn_adaln) { + prompt_ada <- ltx23_get_mod_params(self$prompt_scale_shift_table, temb_prompt, batch_size) + shift_text_kv <- prompt_ada[[1]]; scale_text_kv <- prompt_ada[[2]] + audio_prompt_ada <- ltx23_get_mod_params( + self$audio_prompt_scale_shift_table, temb_prompt_audio, batch_size + ) + audio_shift_text_kv <- audio_prompt_ada[[1]]; audio_scale_text_kv <- audio_prompt_ada[[2]] + } + + # 2.1 Video-text cross-attention + norm_hidden_states <- self$norm2(hidden_states) + if (self$video_cross_attn_adaln) { + shift_text_q <- video_ada[[7]]; scale_text_q <- video_ada[[8]]; gate_text_q <- video_ada[[9]] + norm_hidden_states <- norm_hidden_states * (1 + scale_text_q) + shift_text_q + } + enc_states <- encoder_hidden_states + if (self$cross_attn_adaln) { + enc_states <- enc_states * (1 + scale_text_kv) + shift_text_kv + } + attn_hidden_states <- self$attn2( + norm_hidden_states, + encoder_hidden_states = enc_states, + attention_mask = encoder_attention_mask + ) + if (self$video_cross_attn_adaln) { + attn_hidden_states <- attn_hidden_states * gate_text_q + } + hidden_states <- hidden_states + attn_hidden_states + + # 2.2 Audio-text cross-attention + norm_audio_hidden_states <- self$audio_norm2(audio_hidden_states) + if (self$audio_cross_attn_adaln) { + audio_shift_text_q <- audio_ada[[7]]; audio_scale_text_q <- audio_ada[[8]] + audio_gate_text_q <- audio_ada[[9]] + norm_audio_hidden_states <- norm_audio_hidden_states * (1 + audio_scale_text_q) + + audio_shift_text_q + } + audio_enc_states <- audio_encoder_hidden_states + if (self$cross_attn_adaln) { + audio_enc_states <- audio_enc_states * (1 + audio_scale_text_kv) + audio_shift_text_kv + } + attn_audio_hidden_states <- self$audio_attn2( + norm_audio_hidden_states, + encoder_hidden_states = audio_enc_states, + attention_mask = audio_encoder_attention_mask + ) + if (self$audio_cross_attn_adaln) { + attn_audio_hidden_states <- attn_audio_hidden_states * audio_gate_text_q + } + audio_hidden_states <- audio_hidden_states + attn_audio_hidden_states + + # 3. Audio-to-video and video-to-audio cross-attention + if (use_a2v_cross_attention || use_v2a_cross_attention) { + norm_hidden_states <- self$audio_to_video_norm(hidden_states) + norm_audio_hidden_states <- self$video_to_audio_norm(audio_hidden_states) + + video_ca_ada <- ltx23_get_mod_params( + self$video_a2v_cross_attn_scale_shift_table$narrow(1L, 1L, 4L), + temb_ca_scale_shift, batch_size + ) + video_ca_gate <- ltx23_get_mod_params( + self$video_a2v_cross_attn_scale_shift_table$narrow(1L, 5L, 1L), + temb_ca_gate, batch_size + ) + a2v_gate <- video_ca_gate[[1]]$squeeze(3L) + + audio_ca_ada <- ltx23_get_mod_params( + self$audio_a2v_cross_attn_scale_shift_table$narrow(1L, 1L, 4L), + temb_ca_audio_scale_shift, batch_size + ) + audio_ca_gate <- ltx23_get_mod_params( + self$audio_a2v_cross_attn_scale_shift_table$narrow(1L, 5L, 1L), + temb_ca_audio_gate, batch_size + ) + v2a_gate <- audio_ca_gate[[1]]$squeeze(3L) + + if (use_a2v_cross_attention) { + mod_norm_hidden <- norm_hidden_states * + (1 + video_ca_ada[[1]]$squeeze(3L)) + video_ca_ada[[2]]$squeeze(3L) + mod_norm_audio <- norm_audio_hidden_states * + (1 + audio_ca_ada[[1]]$squeeze(3L)) + audio_ca_ada[[2]]$squeeze(3L) + + a2v_attn <- self$audio_to_video_attn( + mod_norm_hidden, + encoder_hidden_states = mod_norm_audio, + query_rotary_emb = ca_video_rotary_emb, + key_rotary_emb = ca_audio_rotary_emb + ) + hidden_states <- hidden_states + a2v_gate * a2v_attn + } + + if (use_v2a_cross_attention) { + mod_norm_hidden <- norm_hidden_states * + (1 + video_ca_ada[[3]]$squeeze(3L)) + video_ca_ada[[4]]$squeeze(3L) + mod_norm_audio <- norm_audio_hidden_states * + (1 + audio_ca_ada[[3]]$squeeze(3L)) + audio_ca_ada[[4]]$squeeze(3L) + + v2a_attn <- self$video_to_audio_attn( + mod_norm_audio, + encoder_hidden_states = mod_norm_hidden, + query_rotary_emb = ca_audio_rotary_emb, + key_rotary_emb = ca_video_rotary_emb + ) + audio_hidden_states <- audio_hidden_states + v2a_gate * v2a_attn + } + } + + # 4. Feed-forward + norm_hidden_states <- self$norm3(hidden_states) * (1 + scale_mlp) + shift_mlp + hidden_states <- hidden_states + self$ff(norm_hidden_states) * gate_mlp + + norm_audio_hidden_states <- self$audio_norm3(audio_hidden_states) * + (1 + audio_scale_mlp) + audio_shift_mlp + audio_hidden_states <- audio_hidden_states + self$audio_ff(norm_audio_hidden_states) * + audio_gate_mlp + + list(hidden_states, audio_hidden_states) + } +) diff --git a/inst/tinytest/fixtures/dit_ltx23.safetensors b/inst/tinytest/fixtures/dit_ltx23.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4cc5b8b5aa1f52d7fe03a27e4a866f97fb8205af GIT binary patch literal 251372 zcmb@Pe_YMy`^OJLk{pDP97&QRr{AaZzAp}vB$gIJ`lS+*Bnjb+KIG>b%^HnHXx7N@ z^u907@+;OXtq~d_)+{t@)+~JQ_sRCD?@rE<&-Wh>kI&cjys!Jd?(2Tt*L}Uu)C)wK z|66Jq5*A_?J~z}ZZr0rJ1##gqLF_N#VPWC1c2TpU!$V?&?CnBlg)FdCEVT@aTO1Q^ zsj&2Lbg-0KE{F^f{x;3tWxCWdEF>-@XkJ9bg77%@|2j=3b#a=$OsbFZMd7n%M#jC# zywt&23&O=oYVYFYB-ab$&9U9m;$?4xS zuQ4OFm)rk$)US={9G#OeqgpHqwxkZOjt+m%wGbjrFPMW?E1G}LtT?f?tPhN!c?;jL z?97Tb_cR8i4ssXSKZALbk$-!m`KN*b=BhA`xVUJ$=y|bogXY-joFFZeqcJ3ylgk>L zo8Me>av3w{%6$KObaC^7W`@Lt2Sv@BIV)~~j+k)J<}tG-*mSOsW5MJ9XH%EwO)RNk zk{Oma@SPc+qhpBLVoYO%nR09Fw?1p4J=z3IQU^CTm-;Yy?K6uwJ6UsvCm51C%4Cj> zrm)_euF11^(KgXWszy?tm z-<+WdMwlsw*ZcLgm+^Y+4(&q|EJ+<59sau#K{MvP;dEjFDs_=+Nt7BJ%u<6y5HvSr z!JIcHHmQ~rpfMpBdVSn!CL%&GA$4?eYTz3Sv7qRAy2jOnJ;9RH(b+-vPgn#k zm=*md2b-EJ!3MMBEdM7Uf)d_jr-=ax2BeNIY)C>}%?%1!7&dEOP5e_A<>ZX{_i7 zDdwFP-vkSSo%%SK8>HJS2Pds=2_~2!=lZj5t`L^>bN;6J);>dm5y6z(>-ql2RM4yi z^O|GMO)%&3_wxPM+=957%`nFdF>|heub*HpepXoc8%NgEq%kW}N4dR|X4lkDn#Ptc zPn)nOSYq~^9bQMpKVu8h;c8R%m?dVkd+nU(+({;^=9UL(#8`djnW`owO!5!6Y-x&N@w=Y;1UL4BNzEAyLuqc5Tx( z`vOtmzRu6hvtd2B2F}>1;FdP-PEM&dq;H9?gcY1b&@;2PDu?Q3uc5wg$FH&oD~rl6c-X2 z#R>*Z)e#)Dnih--rd|IvK|<21FIN1|H-d)8s7|j>@&*p+Pi^Kl&`_n$E`Oi2&0sfU z6KHsh>QaBnZc?5+5`pcHV*oK|+P$wa{&_8UE)XtHHBDI5*mZ?>2~LMzCfOQWsa(`g_Og6@8bX zG$WujcwrEYGV8kyLic=VIbJkFU;}ZJYf2IN&4ym#&OG~9xNA647gn-sD9UuWt!MPJ zyM#9X1eVmr?QgwYpRcqK;~hdYg{=R1=xf-*2pVnicNl^0b=F>y7Dgat5m#RbXnLjf zn~7!wy@oA}pwVgH?3rk0M%VD!C|nxJ$Ty8buW;8Adm4s7YIJt!LDId#+G(tO!KBpd zo1vCSufHdTIK=-CB3)aFK?9e%%4Lm&Nh62fZV0+2f^BWBv91|{)XmAU>8$oP1JM=h z+U7z7XG3smI?Uc~2ztzg_HnQgNZp*@aQy$jap)E9T2TUop`7)OKs{4CTW0M%Mv}U* zvRK0_nU-+-pD7!yqY(IP6pgj%yNyDRwbmNsCAZfQ9P*RV9mhSwkck$U*5by*pgl|Sv7NNHHFZ@|?n*0r{~)EgCSvknBU)2L^t z3$O6n2QOG=|C5VDLqQW48xp-Bg0;nj#|DK)&6_c20qdI(Dp~gL(kxR?(ZCw27F{rX z`WUnv*vutCmAJS#HdM=Rfc?Lwm^XL5X{o)`qw0wbYCZZBbp>D(=}PdBr7PAvpy#YM zvq`6(Ab=X(Ki=$Lvs-xTF^tA?W`oux@71B)3s<#iDnIo5#^1_ zByT1*r#YyeAb?z10$9e{E;Kq*7pU282^dCmbJc&>HK(zto*;na4QnB9cB|QKNA(!i zqbZX&d?0C(=I~BwQbWT# zpcv87p}v(kP-AiWMz@-u zCN&sFb8ghcq1`P_@1IaAZ}3j~7I!RXZI4m)WMKd?^HKMfXh3}Tye$AVT-`deX|ApvqxF$da8?lWuFiVMipFeR?daNP zm|--uhE=pTg|Q}zGzW$*|Frf2368P1Lwi>jnQnd2*gt)8xmF(ql!oZipvz1jAM1(D z^wUM=BFwCdqi!+U*iT(aUAvDAqoFnIC)CGBJ$9z{{8N7yb9H&U{KFzgS9aI#AHxX# zxx7{G(a@T^eheiLnP}9dZT?w5z16UGAM5q0i`(l4+BaNZ{y$NVRn$^Aa1~I3k9sKL z2E=#IuL4lR)vb%0=IUD4Ci1IGy*PG#oBZ=yxxTqjqIFFL4D-~LC4;w!qIb!!0!kph zP2SP?*X;Z%pftp{$UO~+y4JCYbHr6s3t}x1Z=ru)YuV0IYcr*XvUu`{*j zR{f7a?=H^!cBhcP1_qks{=Eygi3>4 z)~Gw?4O9C1W$Z0(KiadnhA4m=y-~jb{1!wvt@}}f5{UZha$#=>ALeNKV)d<(XUhWQXm)Temb2fu&o&HD>7w-lp2N zG$aOO??k#5^Pgk?&3iO1WfaC@5Wl=bQKx?+gaP(DsmGeU})y9L!)mBv@U1;2v`cJ|Mc~3 zeo#bPw$&g7vV+bGT$5xyGM|>MsX=PUIy0^|GMj;SlQoPlq+Rxh&bK8!__{Wib^)z{ z3uDlI8ERq-%~3TPoQA6N&AdsfE`M2a(k9~=TBvlpIq1998V5T2n?ABed!WDGVvU>O zv^`8U;q<%Ip_#+%TMm0S$3~wm>Ak_3-^~#<8k~lzQ@%FgvmRSptB7M{LB6^ANosQg+tEoV__;YU+5W}@%l?1R*QTK`n3GIL>TCqly_VXytUwW9 zjk@i%gzn$4chAfMs-E~}4b=ou*AlY8)S8Cw4CAwS>%3UFo+8Q2oi{#D0@W ze~>P(gzB$$w-|=GXT%(!vu;3rbGVz3)XD+oLSY25nY^gR3{n`b>&j+ zt_cJUNQWeC1ZuWhLU$#jxjO3~Bf3(icDDqA2Bf1}*}yH`H@_x>QRuE@G`X{0@uBxi zo_5y+iU8Bq@>uWMYglvBq(Bj1y4oK9hP`{5tZ#i}M7=h)=GCR4WY!g$O{GbJCpfB0 z!ZmQTd1+E02%tv4_|xRUqw!C7xvj~_)ch2Xy_=h^0%HTWnx7^Gij@7xS)&hY-|U)x z^0Nu|>QRhnuj|3JfqTtKll24*NJsUs5lB}oHtCkoUCC$;jedaO%@f(2G+9s3fOM40 z8o1T$G%3)8-{oY#|ES|u&S;wA-2b?*t)46lpwW*DXrI^m0lZ^hTRlocZ1m;Hzwc-N zxcpzD?v$D@2N{#H}+4TT&~?m;nPw^lsEbsOAjCQ zOXyAc$#9J7)aV!HbnZ_!v$~SHRv!h7Kx_0RtA4a*uA}BI#)xm1f0%!|vb%Qwga>qi z_Exz^Lu;;o^|{AM4{7>Ooa(KHwfiXOQ|4o%A1~3dhI&ilCWx%&#fW++;s!plS#79N z-~2eDr5&OksNw3?#Z7Z{t!oqc)kzS?!Vumf|GZYNZ}wFC4Ak|zzNj1$v+xuv+}Ee(a_!^|GcI(*T4Gw!+wWP zXSuTR9%)v76)=K-Z|-=gkJOO$)aY8s&Aflf*|k7J)?Inj?9s+#_P2M` z338zpDNKOwCmY>-T`{h`npRI22GQu(E8if?{xu2Bx|j(#4YgT|&c>j-(ALDkDrX-) z3S?dVF@JBZfBCDAkkD2Ru4gi|o35T29cW!E&1zLzx46I+@Vb6qphpU1eR|7)|jO$cv-M$-8>o+1F~Nb*R`1c9Q$wfOFKreFJM^rsgC;1KNwvu zvHhrh4h0Bf{qvtTXzEKZd*NdT;g`bR;+StD-D_Zs>8AVT{w-i`O{|$f5nwJlC-b$I zc!#f}wJ)cD7DnKv^PJJvkfG;)C`((m)gU!wo#LWNvL2c5-+gU@vqHudUXpaR5ojZ` zB=~N!Mgx|zem>oAOM38iZ7%I)5Di>22A!9oCdSYlRl}(4gK_4$?oO2^sk;2t-q%*o z6wvx^HMYzR8aUbC^pQ2%0$qdDkae^%+S@hZ^t;rd8D01fp9!K$S1U^s^yY{f4NgOC zR$5`Rt;g2ZD&iPfkgx2yQD=Rx34fcTX*4*2$|hRJsn{g7xq&S(1+?ae?U#vveoSH{ zYq@~#*T}XrGa%9U7%exprJ{$9>F!QiV7rvKx9bOKU&=6N z%y61>E|q&~;{&tLjRVWTAy6SYNk#rea@Fdzyj`PQ(J)<(Hp@oS%Aoet>C8bA-7f;u zgMX*y#2X@G?Z|=o&M4_#LxNXcpiY?`F~_Ewo@L{5=wgH!o-&f*_$SFQY6I$TSHNLk zA6OFI7WaQY8A_MVr_r-ILt$vB7HnjZ*C$yA#A$u~qD_8|%DbCjwT z`xL>m{Xljmk|;KCkaXlFkyTa`^J_P#dUgn{F0=rD8ckNFwSxTyBO&Uy8zi&*Taw~^ zl_q_5W-fRBQ<;bU|`2SP?;*CRh=%9(kb_8>XBePdhG&A}xf63j{? zP#bTAH8U*nc+2)!o|w!9M<&o3ud(b^8WAb4yF`koLDGY>B*u9h*=phs z>dMi?(B@AXoa+Z>!+)X1E2Q}R%qXyUGy`r#b%LzYSh{2MJlGM|8nS-sO2mG9Idi)| zNMhD+)I9PqRa_B6QF?2VQ5MXL|Fe@<@3H41aT%n469CU$=HsRDqp|#Kul)G|(4^qeYtNskgG^PM>E%6 zq~12$N$v3S#9-nK$aNb>y;=^3{f88o^{fra=w{CaUR(&`4y_>nMr*8@TSrtITPhOI zjmAl5x}eWCH(c%G3sJ_Cpk(K8j5GO@__kaCML%5S9d^rrOdbW|ZfziI^lkch`(!Yi zm_$r_KO?JC!ZBd_NurGI3Hz<>aqk01OiDRNtEJC~|5ul2d3QrjlQ*;kbm% z1+_)4!e@Onu1hq5;>?jC+Gjx%+D4;I4`*08LX7#F0F|>op`tx|@?u8!BPyR<5_d6^ zaL2KZknzA9<0iZ$x%rc^ zV0}ClV<|6tVn=S6nt*M#9BVSBV(rJh;bD#)8uS^1RW7Tj^6GZ#6Y7q^8*F%+>1kw1 zdI-e%eL@OLwv&_rT}ZY|Cscp_C!bI`m!Qg@=ygFttA9Zdm9bdv> zT;yHqx8Dqv)R3&~WCQuLP0_b7400oXAmT0aXzoTkkmQEZ>h?#dV%98_e7>J5AALom zKL3`;_e5aq%-Nv+p3jq?o^JHPOB!R;dZEWc$BAd>%%XZ9)QX(Zm_jd^P=xt4el4Z``#NKrZgU}R^%zw4awX~FJH&h6G%PhN zB`SPLN~~9r%YV%Un@LqPLLG%q4a~66aW+=j>?JYnmJnlmHoqi?9v@>0qTn`emMo`i z=yaA=O52mHfEA=<(=mGXi(z1OZ8N3rpOUz#n)z&~*O_D9Zeli|V-u z>JIgXl{eZ$#+PGx(_`jj*_cEb`b5i0@fu>gG+S*SlC_x(cL4!sEYygG_-^^vz)-X zwvDSn@Gp zD@%L^mvM1>xkVd-SKcNX4D3dnjapMiZy?b13@7u`}QRt{GJJ4 z=zf_*?lz(79=?z;=yMuqVFkwWxp>(j5mHX}C)O3M@cdmHNcm(B4g0PumYyl0$;As` z)2Mmi`DZjy)m8CrKJAO5ynXbt(;`f~oq&>UlepZrN>KMd$OVpi57Rb&MjfnsLy`9k zk}}DKIP{r?waU%(s`VrgZO*4QGk&4cD}ETZw>?zbq!ZDHFZh=S<#6@)?(qB+(+SaC zKtx*6jM&38U{y<08Hk9@+Oa4KJxP=PvO=?62S`Pib0p)M1ZyAcBjr~rIP0Ec;8M;A zFnMMMQRagoqh&r<+b0A(4VGj427u#pM*x9JQnKz-YPfJ7^*-DNL_a+^>9N)Y{DU(| zaUXQB5GlaSnPjhRR#Ug4ygWnqWW`5%C}zKLUbZoaa1) zAJV6*+QCtS(YQ<92NLfN#pNbm=;IgxFaH>dP8H9o&C-EHnX!Ra7VO{@kBu=dDUaHC zCBUfHD{L*jFwDyyRKIss#Fn%l|{O2NwI#*7Tt>dYos|l&g^TxAVqR`MW zo#q9(qharOntVA1Vsggejgj`)@~h6^WLZlSCg+p#iKet>{Zvd?^$DriwT{}}FhiHG zhcTUejkm&2iTd7&JkLi~C^{F&Wu+p?aK6l|vUYGO)4m|7A_=I+T5&}cb^O&q5=^Q( zLDN6KL#r;|r|tpvs2bQK&$f*P)&{g9rFH8_uKhR&e%O|a+3pNf-UrPJQt5^@=c)MS zQCd8}6McW5ip5Wy(RX_w)INVmJ)Q3pr-yrpEUYb2x&u*-bEB1p`)JN5e{hAX=Hik( zrsFQ<@W*A&nC7$)ET4E{o#Gf(Mp_Z^M|~ip|200Q_d|Z&pKdT!{T_%qui}eHcf5LL z7}~TsPL%C!@#vL3L}ittNb6ffE?X)vs$~co*Obwtx2{mTAD2MJoMc`#V;=2%q$ifo zSxcQxY$j=EI-Y2O5_Q%sQTnE;!g)a4yPt(U)-h^Yv*IknGsZ# zG6iNowFh~d-lQaTI|)6|5;ughJd<)w;oGVow0yo4qJ6tV>K6%U^F=RE5A47>Z8E@! z-xg!y@}0zkj>95FZ!#%74y?NmLF3_vNo=tj7XFq_Ye%?)=c97sj~@{tGsVK8vmt(x z8CFC(L+O~2cv#_%tJfRBwp}u)kPJiXg>687p*6|AMaj|8Ye=v!%js(_6XS4#lIv|j z6*q@I{lOGz-A!^fdK^4U84Xoa-h+Ls3ds z?xCX-1bY|oYKsVT`eq8+NH&w;{VTb|{$7x}btQ=!`v*}+593NZ=F*|d{LyQl7$YnI zb3V7kvXGrL%H{wq@Ek_Y9$JXeLs~<1>Q$1qUm@Xgo%q2*C1{J^8W`U(-^@e(2<8je$jTAVnU626d}Q z0*m=c$IZZW;Vg)}VA$^&Ad)T_e=9G&+HEY_s`PeaxbWSG?My%6Gh4b|D=&CTY&nzfh5>lMZAo=LCUX6 zN=L0gr_6QKu+K7@x+MrC*0;Ib-LFW7q8(HiZlsk~J!zK3I5 zei#kMSGNGM+ih-Lt|fY=v?AqyJm6ERRa{b^8KA0_D+(Ts!VU5cklt-RmUP}jGA_R& z?k3Y=rNkXq{uK$58Y#wHJV$mqv%F-XXO2f01WVoRF3j^@BbcN(zS818zFmeBOJDplR1`4YB6T1Qn z@Y%$3=`)4+HwJpl7?m+2$r88GP~NqUru;UXC5*SZ@HzupZYJZOerW`gZjA-; zzFbb?G>7JXK9l4)deO9J^Uzkk3>?PVfy~DXqrbO;@|@qu^ZxB&Z}DuV0r`{4`TcPA zr;BjIe+FT;R4u6Aw?%^YJ663-dDVbE~M{O$^^o<0H%qX*!&K^Aa+ zy(vcCJxkK_27!<32h_(hlZ$lujk;`E04WFBlAig#7!y1My@RT#*=KGLuwH^`>ul-s zVSrPk`eEprQ5f}#K-!27cxzP%`qXCgszp9%dB+QrOfqOi-ec-Jc@$>G_eAeYL(%hE zI5E4gptdEGF>T5RB(*|;=f^ukwefO{dZEC`RdQN4f?(p=R$zWdjOyO^6jLMTfO2J9 zF8g6yNE~s!Q1#RS>Wle^ny}$5k$sqE)`yfn~)lu4gQ=QO&*8j{naG&x4tNT)sggcTmcEA4KUl_F)7WGLUq|3QhqLkw@I8tl*tja zGI}ZcTp0rX?dDUTU*^Nh(CLudp$8?0SgakGO+^=`=RNoI$9SW~;OXBMI**@%KI=}B z$gQiWSBrj7{dESHoTs80GuqPf8FHFB^lMr>b^w|V+)Xn)I}wvxGths=cu>8v<0E!* z(CuhPtb0;N(&ncTAMa(fWI-{_&fZ0;=e9%R+sm-cjkl!fgf98!K-Ujwxxm= z>|8-apJbhk-|UW+#fwo~ahi{6yBH;>XH!Y}N?s(h&x?N?jWv6o5Xm`T5Nk2;^H@doD^~NH;)NDBhoVh{b+^^BI@;g)^8VYf@XTa*ct)blV zAUE~C1cL8aLtN|XgFqW+>gsmnKmBC$7GeR7(3hEF0fA1#J#lPA<>`9xM@@dX>d z$HeA#8MQnbjwL3nmaq7hn3g2K-TjNec!UAwPqjsZ56waSq=?@pae!{pQJ7nNkjOqy zrD?gw)aL_Zy!`DlT=$hVW^`Z0uWK;@YoCTw;}L08_Un31VKN!2FP-64EkEaMHzuNC zk_B-Z9)U9Y0cn}zfn`i5s#`sxOJojEo&B7cE(--!-xhq`rpvT^cMvIfBtfgoY)||( z2bO#*#`F84AhXjY>Ki8lpPh%O^7buSkh_4S_T5R;+3enR*qPoco(k0uA5z1Xt%>>m z@5qfmmta)<_ayakFIH!{MILt_j6Pe>@KeitQCT*H^n4i!iMOmEIMDbAr^M(VN5gS$1eDhK5>KmLWVM|ks$bkAl5W;ql-m;;&~_uq{xSws z)hUGJJD_@GJ1`gRCw{L|=!Pf9=u4~C5NI?3O0Qm`=3WEvLH}qd`jzSMq1P3XPIsx< z5qomDmnGH>XJ^Gh7r4>U4W);(NQ$EoQB^Dem#mTKeueF0`%pUD8!%aYfR0z2peVnL z^B!}O)Jn~0k^dR8q@ph*@esEJ}JKB zs&f=3oHNIkyO)AZoC{`-o`KTrP*i@GZ03A;tXOaOP?qh%tW7@)j zj(wq^?I3cLdr2}Ri9|hl5N7%Y7X8 zzLV5?gGZXBf;_EtEd8`kLHx7eaW7!+n z(C<-ksQPo4BLA^7s*=0GwgU-}tr`GvPu39C zk$#xF-;SD&4MiWTSTcM2AP}$njE`}leCEkIQkB*ccio8uwQ)69I5(Fb4b3M`Cs@tk zLJw*vejgrBS%!&;i^0Xi7j443kiss>RP^{duR4B&-@n=syyrinMR)pudW8iilV9hO zFZCvsofAn$$_*~Md?Dud^})h!wh$296)JL4NzBfTT)M+idU?Y*Fqs_$)$ex!@wg6X zbMq__Up&OQ|Fwrm^4VM6-nn4dF_!d{bc2%2R(M(38$_BWX$#FRwhJ(lD%&j zCiW)fW8QBm>%9T8VXM&QPI2@%ypO#X`c)sHqU~He2zT{Udn2K zTL+=vnmDHaTTmajy}YdDT-+ZS162pRqE(AUnA2h#m(gMgjqEa)F5mbb-jx``=29&Ub*Jcb&muYb3uGY*yGPVm8J#*(K2?4A~s0JLvk*wEo;8rG!^s-@fcXMXH0wF^g0 z5?RdbX^raa-Fa<_ra<*m6-oC0iuj)^rJ{4zc@=qcKs00-_34pHca=mzd6YA6a6BB+ z4oxIk9j=hV%iD>>G!crwGsR5P0@tRy zeefO_D(z@x?*k;p`y}U=>kRU}n@CMX9WhVNCPvA2P-@YIiaaZ5d9MnhzSR$n@2tS8 z&y4Xw*O_R0#TFyuys5IkE$41|ho-FFMuQ(1gSz%PpQSiL>h8->Ib|tU_H#sahBH^9 zxImI*>7*#!nAWu)1(pu);jVGkD61%>mvcqnwCP(KQ)Wm)r7N)9I*3MXm#*@_xh&L8d>#MUMea9o}Y1D%T7d_?0JN-${p=g@7XE-E0PA1BhcEsmc zd#Y+*nkS0cdLndy8Cd!($H+qpDsp;|r_O(vr@HRI>V0EqOzS=*>yLBf)ZLMwI=FSTuL5qU+8^V1{}h**)C{Rl86;w94E`S zy5Q5`aVQ#^#BVbkjB@lP2KIqa@kt1ZN|w82yzI)UyNseSUM66r_=-fXW%F^X6&4v4 z^SQtNKrOCxfYMErnl9}L$y*jedhkay*y9_{Y-u*B{&_Jz{nZZ5Uk!pV#}cA`xd=S- zyMbZmG$=ZCk**HvfH%fl;iUgeM4yAJsbuR2tR9k26w}A!q`mEM+j?(Q#~1L$--JQV z?vHqDhu)azUrBwQd`(vmNI)aLFBmz`gIxv#k^0(!sG>Wm%Ic5V1IIwEksOnkW>8bR zzL_R0g|nGT&T$iOxQ7#R-Ic2r>v)g&!csmd)Zym`IRd^KSn^ZBpNIi zcf?fpd}5Ol33U&?CsCiy#f%FkXfyUYiL)7ihN^j_jOFQs2Q8@gFC`=+?E%d$On`tj zRpiFP_aOLqTWTt=C20jE;`^Bv(hg z2N`LBn00JfF0e^7DaZoXXBmYaRpAC?Sl%W9h$ z98Fq*BHwN#c(o1ZXVC{d=OmM}BKDTIcqQ?^?1rkqRIYaUebTLk2u1uY60?QpR0|0N zfA$mS&>|Gl^TuPhXd{R?tnDa z`w21X*B;axq%?W(bo98}4`L@;qcSN0mds-M=iFIh-XfEhIvgUT#|=WRKEm`^45`$0d=VJEEM9Bm@Iy<8k1Y`|`!#}ph zG=3SB{~m=kgEGj`hovND*fG9z@J3p5?;P22V-^^99EF!>%FwIzL?H7Qpi@#VG3xFC z`+E<@%zY-PmdxdY1LpGGmOI0lpV_E>oD zBu(5Of)$_Lr2!8KKJbcw=bNRN(RmoHzPX2wyfGOpHXGph?|h+b_s`^LQ2?uVpI4-q zm_kN*AorlFA1wLH1w&7GgXhsLG-Y@X;uC*|7IL4F@r*5rokhfPcSz3*t*~00K{9(> zq-9?xlXU+sC@~EJQBGG-z373qZe8K|=T0Cin+63x7ZaP|ooQ)?Jvzm>f#gu8U8QF~n+s3lr=!fejz;ft!s=QNVxF`dvTeR0Moa@4 zf9MLwSGYmtiGIZ5R0K+QcLvXbeq>u>IP8+M9+}yD6`>JM)tEnGHMB_a)^fL zQ_)Z$b|I?veH8^e4pCXw4}4WrG4-kNCfgnlgiPy`H2+#_xcXafFdQTzB8Rq&cee(Sgqb=BdN2pK-t$hH1VT5)E7p9tjiVt=v?l5BYL2hBM@gaoG%lJhu$Ca%w+XMaCMGtU3W z7dzTx5L9n0+T4_s?yO zwYF1e% z1NS;g)q!33>Y}gtoNH%E-O6~#e#~mJi9@)+kX|4QJI4k02tsA1lvjTe!KrQTf9s1n)nyQ^1W*n3(c7cqQhE(-w=R8SjC#dq_N%i?_@`N?!5!rpw+ic9QXXaoC~_Jy3fIC=GJ!RW0LQ}WW>bl$92BYn? z^Fu(@id;S?Kmw&k7Q|{#CKY`xR>ZU!2=U3DSU&P5mB15vcg zd_WV%OF`B9k-{@0l?FRSfl)7SD0tya6a4=}qMc(wwfqHdwciBNP7j33f5_P#(HAQo z4+HC%)=)R@AeHA1BH}+qB(`P@THP^)+Mgzol?9!FW+Y(R=8L4{i~|IpmdTgDvan@>#x)mq&b0eLkzEJ_ACb#CjJsNiYMeno@dp_;RdS9EAoC|0K!- z37DI+omM9vws;LQAnNWNdK&(SjUK zxvGZq8{86O*3^-FrWxa=4?$VW;ezn5YtwTV(aXl}3 zJQ9N2j^;AMvPhkD4GA!6jZXI@C~FsrnNt){(}ky}+K$2G-tH$8HH9d{5JF zvG;F22D3gl!LpIBh~a84>gSeAM0MLwra3Hy@?SsT9Y`?v_cJ2_#%Z+Z!Y-n?Qxi9v)0^MfHjxy!DAClzeuTyY%@3%Hmlu>=Dx|XGTeuu|3z`v~O3l7Gz-kJ^$=RNB;l|JH(0gSJ#vN=0UN5ac zKB1IcofZq_h8=j{p(9b$MXpeOTB`6qe~~;N;R*ix2SJ77c3Lj6;kMpnYdS#2yKlcm zC0Vvu^2uaK-Mooj_F*-`VpB|N^#_%e$#JUFa6Efcg6>z-h?$WwbSpCi`Q}iRe3(UL zoo}1W&R?X-KNAp(z@hc=Hj;vo{`T%~C zcO(R#cVzo49<0n~!PYJ<(J;fG?&>Fln)m9c`u&b5J}{DOJ?n_%QZULl{6>$u)Q}p5 z0yE>7QYxeKh<=(7Njgrw1jxxKOK{#P`$ zob`*=&82a%KM~7shGW{x1R@&agfV$fxYEGKwDSA`68xe+Z+`J3YV}Db>oJ{<@mpA5 zN*>2=?X(P3MJtH82h+aRZ6PXTG5T-1NIh?LpneyhQg7RrG%o!C*-_mQO5h49Wq0u0 zqJboBog3Qx+?VXQ*9L1!?NPm*^^ctM0t3}wH1ZY8-5Ywq@p13t;dP6!V$mYF{ACcl z>}ZKu>QE>b&xFtl_MY;6ah@z~DbPb9Algtxy|?Zmo~Nf1)zHiILD(czRaVh9{d+?7 zs*@zn`~)@Z>WQY-6JP^-!+PmSpeAwyu~eBucBk=>VQR^%ZoS9*bTWdKKX(SpJAH9o zj4k@BmV-~fmT2@a5Ii0Xg{iH=AzEY&M`wRbI`{B|fT>%kOmdq`GQ3I7@Ark-$w3%U zeTk?=U*u)o%p=vYd#K!|gqD%@BrAF_Ef|zW4D&A2mSe+k^-f<zp;$!s% z5uI6kQq_4iC;sgiC(ZbZ#;p93)@+-D;@+pYqaGfRR4&7+^YgIi$R^%+3tNM=PFULa zKV)6|NPL*m6@&XN$GuPdAaa(JJiZ}eDDj(0GwO9LNM7#&ojqeH6%@-#{vi`r3NBFGPtZz|S%cWbecfFsA z_{tq&uz$J()ZUvwWi!6!^=L;M4G8SO;Ub8 z&Bd~w)m#sM;?YtBspG=1;O;!Cu5hJh*9Kzs?*UXfFgGvKZ~!fL-pHlx>VlV|W@3ip z6ey30povo-kks@#lAa!e$FH!O>!@F-_mXdDnr$%p@Mme^lMhJ58Y9de$?CxA(}_Cd zSA{5g3}1BbIf!W2=Q$K(qRgrP)lQ_%ybe$vq@n@oeL&VWly2KL3{_R* zAZ_~;k`ehe|CrqY?0UPv0do^fdtnFVfknjhd?Gj~{T=0U*lg`jf_&z55`H043 z_QtdK3rN-8leF86Zm0}AOxh>|(BPx@A?IU1cK2DuKe*Tjl+{&4HSrKp-8`1}(jWjm z4vhx#TM-Fp9fI4w9tonTM|tY@ZOEy$tPgLW0s2VhfGnUBW?i=+>K^_1yHz8wRJMwi zhi>8?uN{GA{jF){)GdToT%b{`N6ANSg++nIoM=iET>710S#LkvleK$cR0vQ!bnWC z5qk7@V)fBlGJDn_$a&HYa(kF!>iFF>?XRUY`}l`6<@ZWH$7(x`GpVI^&pV++9!uNA zcg5^4SkIgIQ|dGM2QG5ZKrr9-h~&LL3=R(*41NidK)HJwr+oENq3j(5#%sQyqSNXIK_Z-uaT4UHOR^{xE^m9Amlf@MvmiYY&#vWsu}F z8;=`CprkH{#&ohFkNb^Nq|ItU ztCk`uf1F5)dIYgNv5MrJmSItq7mY01MI#GcFy?3lRoSLm3!~g#*@0Siy-AE zBa+?j3|Y5Ifp$-4L&mz{Ab&yWd6^Y#h!>&3>gRNwMK35=HjQ4MXoXAIw>xpIcakhu z_PyTNwrKO46R59*a{lc7y?WgvE=szRDA(U3YG+^0CzN0=X-y2LS^OVy-~Sg=`u;yi zl4K+#m60S#Mv^q=zD!AyWTcWxMv^H-k|bf!CL|#Vy=-gJo7%1l>dWBWt?;)Nh= z3-ZR8feV@LkPFK=wS$w#Z7r)fAB&uw4@&MRxY~tPyh;C67{7BfXSZxPSB&Q9{Z_;k zx*On8&wglZ6~j8R=CXpCSWU{yUYHkXjh+i4FndoquO2>;wd@b&)cOIuv$u$;dIWQs z(O0>QwOO2eWKX{SelUwXQ_B^uAH}ZiG-heV4|u1ZW3lz*Y9?PiTUN4X2B)YKadj0+ zR0;V9^|EjcRogJ>nGRWiEP0W>qqR zi7aMo3O--glnWY8)-=Hf+*u&=x)gwtr}l!Myq-_I9LCZI`!k21=2%&$#}(Kc@^EG* z<{Tf%#AUBJy>as}@!Ac(-t`!NbLvd?q(sD8NBCj!x;`lVUMA}tlg?Y$hhpi`6I{x2 z!7J3hkI(o|@I5Y^fx&JjyxMf9MscQCW^wQ#-@N)ZSNK-Lw?(<3dYdCExBKx%-SYX& zX*)RS+9iCrg(bH4c0ld*nVi?(u~_30%Gb6VvApXH&EAgSI`7`*+Km^pHi19X^()q7 z4l3td^QGL^cg{>DAVOMy&iY`ofWEJl)_dI{I#vTtBGM46>&y{Ze%w^_;pv3JK z=XG~6mUbW~`t6RaB13qF?GJNWn=fSrx2DPRjC%1l>xJ+7uNYROOy~4I1B>`m;B_tv zoYFWq)*{R@l}0iyJ1&ex&fCMue-6-8uA9yUy!p=8cnsp>dmiN!Kh|*~;{k5oXYO)s zv7C?{*vISc3;E+mxqQTOAp>kEVv?n4oIERmtEeA|#e<_*>D&}9XWa%~y8bs#bb4Ev zS!6pu;9f65Bk1HLiQP2fe(}6PNEHu)7MON%K69BlAKQ~(@vbvDc7CYfk=0kQ_WV9v zv3n%?P8`XaM*ifqm%U})TRiyMM=osts{zdQ+$Bz7+n2{Cap|Te?A{+^nPPL5tbLOu*EXwwk7^a>jLG$U zyW>_^nR28$i@jtF*!W~(#)hmv7`;RYvSI%WW`BUb4T*JG*ImL0#J2<}*fftJM zXJ(^?jPcEFTx!_we7k`PJ3Y2>`pz>2?)3xT_+l+)?2&R?TZW)S*TO}Pb;9CT z+zCGU@J+sALSHlxyh!)98Do6ST;52Ygx1+p@#w^5sH_fQRmp{X!KG1*>n&u7HxFiJ zpHf)UofW*y*{cJTS5(|PBO`*~YkZ``=5 zH>TF~7C!G>mNvA3(?^2PhhZ%9Jv{`w?qpcn+?96`@)z;_?{j8W zN}hAR$BQ~l(X{9jAG2*TmJIo!d1Y_TN`>5~y4!B9(qtANnf;4*R@CtiZS7crwJ|5J zf96)Y^Dx&|x*F^MiA0fa2pZN4x_Ik!u6oyE0Xq-i)X}xF&IrNZv09H+)(ak;Jt{ny zGmDuHP2(cnl+5OI32!s8j8_}X)Ocq0Vp94kE10;3(;e%rDaf`%{RA;ft^SQqo>amo zs&?|3tM_nu!B(uedM2xTvjVIA7Gc(rS=d=n%Gn3X(P+GYdv+MOnb2_7`PWB2b;NG& z?ffJ(y0C@QY9G3ZX54gBw|++DCvQCO_CGvvF9Nm}rcmYH0nGb_DV+cM8+$mcKUMwn z9VL|&fPo{>XN#W^gfd&(Ss%?926AZohOlj{3~ zQohM}IG>cw8n%YhsauR(JvGerPBw;obOw+4fw14R7pPj-qv}X3mCj#44p;M;f1e>> zCGi9GI+09dX)BYw^kR*3hruqKO6d-saPf#2X{Wzp?MF@UriTd3R#;)mZ_}x2i~|+L zJ3~;4m~sqckO)eIsyKzKT>E>IA-53X%85kfGEBsm2?hi~NwqdH8Hf^{lpv#ogua6Pn5(Cr@X&r zQRIadw2d7>^$+Gy<)Du!{hvPSo(fs-*nKEzolDNP;ozDlNA35o*zmQTl^k$@jG6hE z?f8j5JiZSVCJD~pnlmUJ*TjcUHlk$jnM@vhLQ}shjaAv-W%0)5EV)+|GszN>>%!BR zoTvc1%TX-q#ZcO4GMUt$)A{CyBL@(#ns4u&>XAYhHZ51V7 zFvch4MkKK%UO7mQRKYJ^a3cYa_Z^KYCY?mAZO@B;D2TwXNH9BO7QfzspM zupuXk8ltCzq%NIby4DMtea6H7AI88pdqdXgIIv!54IN_?WbvjQ+v9}1g5aTwycR=x zv(|uVVkYxdDWSRgFl!$iiDf54NV0eZwy%mtE7f>VzO>~;X^pb{oq?3PBM^))Flz8x zLv>nz3R`)KHO;uomU@O!OJ^kJ7_SD=w70UvvJqrc`Wq{&6rRDhw`CO@jG%2Lkk3;o zi2L<`*wA4liP*r4AAaRhhtH*;cs=6GexbI+iK%x^=jGZwnRmo+;n{f20*bpqw)l|1 zbJpXBtC3K!K8Y2bT>&EB*KSVw38Zb4a{7A1$j>Ai)V-E#v@fp9>ffiMR?(Z2t{e^p z_mseOo@PG!v6TGJnz#WY{?J^sHkK`qk2*N+yAiasXf$w_5qdW64sgYlhsak zhKBxcv2)i+%-)y`$^Dez`6dy1)c%Q@pm7r@Ctl6s(*mfo zz7#9I$C8(ag6erUs@fd}IUo9wQRfjXDQVzKX8KUwj40^r{{u}w$-#K!da5rU&BESP zV%W23z{O-BeUU=;Qvq|#HYC%$iFDpSgbHIHGP7gHuqGsjE3a4pb}r{p5AI_dC!y4+ zTGZ>`i1G&wvede7Ea1N=%K01*b&=i3!(N_9_$k)oApk1L%}5z{4FP z${p1Yp4bkcUE4gsGi@{KEwlmA#-EzPBR`q9s|xCmhNF}=^5Qvvaq@pQ%MOcH(%G9{ zaCnq6=?Y%(lKzvy_*@{!%OA?xmUN}0nBLSrXb36}|G}MV?FpX$SdiP$!4zD00)6B5 zv95PUf(jaLRys+5a|WMr!KMD6YHRShB-A4wu3|i6*bK2Lrw2~;qxw&z-H1RmhC-|b^h}= z79H^*pA};u=$H?fjhIMvx#1-CnE=n2L8fdm^h3B9%q7h-XNw5 zibSJv6&RNK6L*z3Qn2VZ)*%Tc#gEmf9{*XBcdG}axUYcB-l@cU>}8_7#hSENZ?S1< zKdNn?L=B6_faFRS!7txK)^x@TJNvkSbRuB>@u8$V7zTNZ&$6tKv*6Xjfwa+5N=5Jb zvy%0@1)V(vs+x^JJAM*VjUR)`$m3k)$_bG6a{y(po66!9Pgt^^JHhht5HTf;MEiqD z>$#IFd9WXAE_s7CJVRE*_F>Vcb#UtSGKk$@ik|khkEVAWamUr_x=B;+8n%^tg*@rzr@=c0v>UC5X8B7W73m{p3nWYyS zlYg{1DFTC8{pGn>e=vpY_WsMnX5W~PX%rX@e91D*)`9xOHBGivx-f@ckflc5K*{>i zob`tRkQlI>;%}t$O6_Q}?`H-&OA(y+--Q065!A5$DMqy?Qt-=e^wu|q;y-TU)yFf~ zMoy1JNzXX-n8UI%{Uvn%sS+YauAz+Nm8{{I2m>V{|RlR~T zJu*>sWenL>pJTB-{V1<&grJ?Zv8a=iDC}VyYWJj)Xvia3(&DucE_{db|4rv=rg~7( z@LZu!Wh*B>_8oPhr+MYy&XlJNfvhX5AbC9l@#(ABCh7)>llqYIy*_-XF^3FEmN2WQ zQmXX0fSr&RyZVvD^&3lxm=7&^Q&9QeRVLc7gll>?2&I$m@umJlnfUVo)O)ayF3N^d zfbK1pw1r|?gcw9?hjT4fH@FIkJDGMH0KN(JEWLUmSxA)Ruy88XZ!Zx#BnENuw?~3& z&uz?CuMEYLjaWh9I+8wc2l?YF-e$vltn4?N>55LWu0#4l-J?(v--x4-Wj2ucU?(PL z45JQ9cRFZ3AC!p=s9thV)^+tTh<*ABU1OKB`18Lw_2YQR2o7Nvf9plLPD??RnTbJ> zVW1e;6SelYQ2s}|EZuSx-JBf*;;VkBTQC|Wx4W~HmkyLQ+<*+!-Ghvk~XV=H9Z~=H8byF#_O(N__9B#HrvqB zehv_SCy#d-F@YNGA7I3xzEt3Ml?9L5%Ze6X#IQmC;HzKZQ1oUTy%KPSHZ}%=-|6s` zDx7@F`$Eb1gUBZ-NF5zY&YN1%-pdm7OBp@+I{-vv?3Vd7pNTJ(al6(m0moMh=wPBB zl>1qbNTPN#Pfwx*)2T!stU)niDiiPCz>4*pAk6YEQ}5o*H@b(C(#evilfIxIKN~jI zj-r5pmzhgt-DEOaSg-(+wC>^tdJ#h%5qP23Y*;({|UP#E>e-fI-7j`ucDiYczz zk>p1#Ygq6JO&jVl=d(Lx%VMEY`h|(ag;>nXK`tN8$6J*O^Lrx6^WMv>ZpB0Lj#Vt= zp(|9RPNCSm8Z25f6BF+aqRIp>th+mrN?whm@bVDis)fBvCY+Ji?Wxsc4#{PY`SM=F zA!PY9&~4HKyU-X?*9g37{b=yq^$*%D5i-8Fy0PRI4U7SOe5K{h+HSkBj3Mm=xjlE%CH}cqS!7h=FkZ8 ztQko1=m)Gp^O~tDYf(9K3CGK9sN(xru(7*}>VN%sQO%Tz>Y!<|?5a+e1xgFUEg!0QG@sGKY+*Ao`yH=lD1raxPV4)Rj=;Lw2)_xni^J6-3n81OO63WpUY@hufobJ-j-Z0vzq!I=b7}wYmL$%7!G~GgLziX&U&?;6%A3Yi z^|LqVmIysPUzZ4Y_N_+gU`%Bvyuk5C5_uOtWX%sgqu0KJE zDBw*C)79O_t7rYPn`hDzaWizK3GL;p%@ZuTc!I222CHYirTQIf0VbPlDpi za#rwA!Cz}NC%vEZN&aOrcQ#uD+SzY8X9HI_ziJva%OfE2q$wCr4JOrjU)EYOgqn{C zvr2p?D|8is)n#EH{&oPHB*n-V9A<_3S6Qy`Fb_B}6m$&=uEI%0+1rySrI!pSuMar? zGy>N@t5_s_#gfhwOzqYcbGGE*07oyn8QT+-7fktO zsJRhHzR~X(|G0%2%oe(s%tP6EQ)9|2EoZ(B!Juuz7!WC<488v3JNz=9FRo_o_xcI^Oe$5(?F!08=Xv$ml|1j`1s9_n zsbE|elAFHMm^%N4Pu9<*8o@IWzABNl?m4m&qtk5bsc_I`HK4CSI*K>Wg;TE-;MIeJ z4>QJre8U=EjvqCz+^j+ToLIzsXHXmcl$H3M$9viF6no<{iq=awGm#SoJM_k~L~p3< z?8*|49K{0Dc)A&#KzU(}DB8W(tte)&;4OAwuzL-~9O_B0#Pccm%1tzwXv?liCsKvA zjd0(_LgavMsnKZQ*86l{@wMn(S;{+@vmoOrPo8dUC(U$l^ld;ZNX z2FjuJ_z0@`&yUKdNh$yPBNn`?4Hds8VQ1YnHsF^(tTR`F%2d#aHkh*D*!kd}I}|dP z&nJt0@0cX&7gPVdqDk~y3<>>=$-p`mYU+DqKx{io0-tmFr=JTMwdY)|aDQ@tb_cth zbriN_I%Qi%Q>OC((mh$si=Q0iYpVX_r0X=iI&ckd!}&tMld~)>b3YRe84kJ&7Q#Ba zf-Rema<#G`(z#FKVpGe|=DaO=7#UMRzCXvOU&4~7M|jngY1D};q3Fr>S zTviUHf|~!aw303CY}Qp4(s(v}5a~q0r)ViEF9f!m_H9NeW+in;B=Yh_j)hUNLOI zb#HR^;88y1JukU>hHLK}!xCe=lD6grEB@4ln%t+MT|om&-uDy}-iMNIm=CA-_zLP| z?wqc#kj1dRj9W)LQS6&yRIu@6YSD{I6a$%4qz@=V|Kl9`TY*u@Dk|y>VWK_3nD?g_ zTdEfciN^<#tJ4vT2ugtG%a>Do_fO0#s5fPAm<1c#2hzn9CDf3HH~sr5W_=kz@^_;_ z;Qx7PUiMXu-NvQHN~Z)eD`Mdd@@IAw#TvbYzLN(UrSc2$B^2mgl)9%MYbP( zNYu}rOUpKbAgw;QYPyrzxvoOCZzn7GZx5^gDtOpS{9xlJ58?d!#g(Rg$NJK8v>q~o zI(2dq&GF$2o~|SR>@m=JJ)V5mE1}@qO6I)cE*6Xl6?!K}@LA(#LCVTlN<0t?qNE*e z<-dZ+bH!vZZ1IBlqB~f#!5=Dv%s!-in1(AAFHn9GXM? z(}zs;!JZ`#*uXL_C18yMVflG;P|JF7HvdiniXR5j51)DQ#Zh#yM;Iw;c5{s;5|HTk zfROECP&MgOp`$l7)r%k}tPh=SaiYqz70j?P5v~by8ZX?VjghNKZ@q9nj#|K^_XjZb zscD#WU?qiw&7vHM5maoQLpfV!f$rr(X3+H~Dgs9^vHkCSbMIlKt`K43&d;a~n9N=+ zGJ@(q;;Asv3>4>Xa}_T)Fx|mcENT40yMA3Ncq#>qxZFtNkT3=uj5xTtQ9%vf{VBNO z7hBgnlZ*z>hk{9VaBx|7vJIR@!87_X=S&Cq`R@!W%;{j9@cV;D0zl1w!ngKO5cXw* z&|mS2S!}Juk}=ci?V&}KVJrA(Up!+u^KW3o&^+e4zzED5wz0Agt4Iu2v5E}_kH{fp zb7l(VPt>O;_uVNmsRG6RU07`W2y(Lur>1eeNE~(&Q~HLHsQJ%{+4nb~>2^6}jM|3E z3WeZb>cy#gcNI8K6RgR)%h~QTBuSl=4-oj6gPoR8kQxZ;5(BQp!V5z3WuVUP!HVn- zumX8+KK8&1W^?BuR$Qj{GtA+zwZLy5`i+xTE{0bxR|VxMIsP7D_E z*Fwi!c?AaCeT>bzsT6kE0&1MUGrJM%LAgJH@3d$@#fH%=bC(u9-+aKs4r?jEJdzY^ zjGLg&;4wDqPZsW>x;A2*P^{eqzBkeKDMZ%q15z@K{sjyP;088{2w zJ^@0`>-T}6+V2h$%Ts)7^n5bid<_ZcH@(P~O-5}YwgXP7Em~=oT>V&Q$-?cZ{&){fk-8&CT zhu>iZ^K!95D(Deb!+<+@1gk!}Q+9I{24oAIu+wRcw$ERT@7RW6t+v#@?H*G*tP-yM zGKwYVS;f=cOtI0BY#vl&Q%@T-TDBb^7MJlw5MlsVg~L{s#-jrc(2)4Op>fI_V_V%+C3DEIp=UwnMsr)48$2_nHJp zhns@V-3%i#jiET?I(l7QEO1YUIKwlxkU8r*rUjm4zR4W4YzV`=(e0Rgvp)nUOvYCl z706w$@yV}E=~|x^AiY0}WiN}wjM#zjVZs1{Rw+0*MUrZzus?1%qiI$#uumQco-5w5 zjM!UD^7#sS23!-aTg&VoMS(?93W=mYU9~xxvPfnEIikPNf2S3x?PtjX#@!Kc)?)C? z)}eZV2!mt;g*oD;z(ubFzpZoO#CU)L!=aqSD-i7JCSs{dBxnt%F``UN*=ydg#*jgz z{$DPyo7)qDpQWOiT8mGjj3ICMI4Zj3LY3hY>EMQ$AZm?6U(5GG7ll1|b`wGME}Z zn5DIvQ1j^)%v|XW2I*Z{Su+PV^PVx)l~pYHcpp}>w4QlwnnxgGbIShbs#Ec?t1CRkEa)@p3X-^gzEOoi+-dDAX+pBeB&o)Xpg7 z44!gK<&@7{4|Qi!*%#jF?qo`M;Ya+97^sp3f%UJs^n5!p_7bXm_6XajrwLx_-`Tyjb4k7?h-+BT!dlMoyxs3! zq`t61$j;1V;;(9<+n|D%9~;4?=Y)``Y!G;B`cv)y;^4%viKI1e);Oa3K=H97T+FuNR5>ahcLj0OdHoq0*rYNyg)3!W?BLR- z6{2>^0xsL^0_Lvph4|H`EPk#(IQE)DxxH3Xz^zk4A8ID5Cq!~5rbSV~yNP^7bTm{P zih_u~UQl@1l!EpuN&cjhtGZLcidH4iy?d)6Vccx8?_)`dC69#;ns(m7Lr(G^otogw z8m2cjo0*O81@(snt>KA)xz?WGx9+hgi;8_L_~r_Vx)T6e=P|sCY#fLz@ABGCCnn#} z1zb11K?Cu9tQoijyag>r^>Qx0IwJw~Lp_c9>%YR;_!k>6rZ*G}EMwCDwsYCNggxos zN9en26nKu<&AivVMb*d@rb=G|CAOigt><=(Bx?x1H4l>CKgTq)I7ojRLsfH)>1bFY z8CWE-{SPB3J33SF1Kq>mzD6MWo}_th=tA+^7m(cP21^@wQqVs8W6t1hO!RvsO83+< zul}YG+5a8hY)v6Oz)iIE6Wws9;z>R zy*QHE=I}nJ!{Jq@A(ie{;$bsOI6v7G;#FCEdY?f+LspVrx4T%~e<`%vRkN1qDa>JE zv(R^vk2b+w$mW^@wXg0?=Wl8;zKa{$ep^ZM(k_tG{+tC(Sw-p*NwUa=D$*Uj!P$MD zOyX6c;CX8))sE&UvCBcGc%}z_)4d_hE*d=F6`{N(l04@jlNuf54iC11t!={DH2oRV zRW8Ehw(cx_wE=-54kU6Zn$H)sIKThl!KM@l|1$+L*ZhVxVeU-#qZbuEHx>4n!KCt# zfI-PebeLAg8fPq|jlBcFXqGKB&wU|ex|%fgLoAt&%i_&4;#j8FF!=he8`PXyLt(#e zVbL8uCV4%R+Fq<7-OxNt1T|mmU?t1vfx6#&nXR)o$S;IaXOrN$=Sg(pGdMzBHIT zXFfx1Vw%k9e?uYu-b~h{T*69jgmb&geYYD7lqQCBFTGm9O z!`|-H)T)Gz=n)XnC6*NZnsMu=IUu*ap*gzq|IUx8eBCL1IQ)l{(!Ga)ujx7Tm^lR$ zyX_z~zYlm0ju5!7Fh2It9C8I=CKLCiBHz`N{cI;|mah^x(0v>%G9qR929_Nj%lvl^ zA+<378{Qm5x~S3MwlbU=Z2!kR66es-m~oKn;6jpF-?^mjg8^bD!>-j%RMcF@TIX0n z<{v{za>1T6u^a@&mkU{?_fb}MIRMnK5>VUWazgTie#h)!PAD4hPng#1-h?debAtqoXr(H?A; z3-=*+Ecqsff$ycc(0)vhTJ^?*>B%`Hy13RYV~-ZKno3r=K)ALu$q?MH4;nZvAkmsq zzW&28rrQw8Y>v#N_&!H?wYMuJDtEE`d7i|tK7n@K^4Swz0*SWEWi3M&PE*CsXC?T`cq6Jkb9! z466ICA(hifCUnq2L%lzFFPKI(9~PsTek|sbi0qPnF+2M}22&?d8x*2+z!|P$?q{Zp z7>(J+Gcnn=JIL?5QqtB{pv`{4`MvTa<^Fumpm`k0>wocwSIwoSkH;}8ESk!@L_*Q8 zNg#hYfN#!?1~c_4mR=PA@rP!BWcd<^uk^<3i(k-TgrRW#$1&@SF_=vc1Cv2L!S9A0 zRZGLj?%7tXEV#@>Z)#+jEq0W@-W{Ht7D4u{W6ox%z)& zVvilD)!)ELd;3F~g}}MHErX_q!$9@hNERWP1nVLqLF6vyoV>ac)J!6;sHGr2=MI8L zTe!Ew1Uzg>-0Fc8&{oMrKRWo7f~9mgIu;Dp_%PT%j{-Crq1WA#5?cMqwbBQx|L#r= zW|z=pLl8Vkk&>?3oeGz1W;HG)f`=uQNkeQfEzW`XYYVARw;aSHTKJ$X<|I9Po0Z%i z2o44@WR!P=nN8`-w6joV(KMDKZVV^wF?UWi_zTw7*+apmsZ6w5!ldKAG4-4$oU>&$3!5+y^z2oJCy zJ`zg$Z{+hLpD=Cua5D6=1#O!-$^#lT%5_TSePj{pVz=-qJ7<#e4@Z!Hi(rZOez5G@ z8O%uFxg>vg@L|`lq4Rue@NSw)27*UR)bxI0!Fgx)^VwKP4qO9U7X?C1{zukgGeO`g zJuzXe5tJ_!Q?j55WacfQ_*cuZWZl2aOz+3{W^`8k%ebDRKvd%A)8a9_~s3!d~U4S%hE8da{)N9V|Hz^m?K>3@^3_>_Q$ z$C`s`(L#b;q3fp$)Df(fPKCX%|E!75s6ozxtx{?KiA_neZM1duQCQSpkZ^ z$9bEvgP{07uA=xw zSxl1I9ZdXOAbUv+w7kF1e`pQ?*9Yq%*?Isptq}P3wZaUde*m4kG&0Hb8+`LF9}u6F zVqRA_wAj+j68-M8w(wU1hKNVo^}V3QP0Xr%=7Q^}YSfzPaRJxeD8suesJ0Dabv>t& zqKe6aqmT_9q7XG^2t$kNCX0 zYAmu;p(<}QByEfY^A0H~*B;;s_gX`F)jC>tYz>L`2{_YdC|X}$0OlV3z;tRhny`Lw zIAalTb0pBYw-5`Q(wMsHljcNmB;?&+4(jEp+$p=+v@~`#wM;#Px_fyW$h>SvE*u@KlMi=J8yUsOC6obBC}k`I8@3?b|#_zpFITq+y-h^ zra+F}8I}me==j)_yk_)-rQJBn^s2+=zcNu2DP-QekAkge0N1=0Km*+nx-*2X?4v6* zEpbP=&Qt$mb$`|hg$a)^yGI-tZ5Dc}LfW}|F|$bf%8z?~CI_!53PHE0lJD@mh{lEe zApG=Q8XtBIBYGTU((V)a@WD=3rpNzD%@r58Q=@aS@cVfBI=)`$3t11_rftTk(st&c zw8wQ<#AJPDId$!IoIIj82>$7Myt-!!h(?66b(4!_y2}2t23Z-Ga90l7o-YF%`*JQY zavN#u4x`;mGpc&n7reI~X45B4V^y6JeEZi1oOjU<3>Gc`)uCE$jz<6#3piuCs2fRL z8#ULncY$SgFnFzWhtzmAHl+?>JwJsDv+HWkb59cAce9KdHe15C>0jB%g=5G<|1Y-O z&jS_GID)s)(50V{rQB$Qzuwyse}MyqR~etzI|%Lw`Ou)BqbM#`*h5dpQHy0CzG{h@ zY%N!jRmVzNY&#cn``VJTVI^uwkIybmCiAxzV9=+ZfZr`aYb++;lulL?(XN@eZxA?K z3F6D621CA(#mav21s}iTab8voIlex_t_4=`Fk?HXdTR{gI5`!D7T^~TbJE3*Ca0Gi zW?V{Sfs^*L(AFqAFlab@t?5OvEt6^LQYUcwz5&{+qPZ5k+g!ZhbMYy!VAlht(3+0^ zFlkdB{&A>3Jh<`)y4NN~LiL>2 zn3c4PGZbdJ7w&UmMa*V?-faUiIp$8`V|Vi*wYJ#RWHQwB=}WHZ{U}$_jz-5mFzF~O z7$fkWnP&#G?BBxinYWB8;uNsv%m;3ap^{S79NXiy625+!%YPm87C-l~2D7v~OmXtI zTg~Zpl(+u?b6sHo$pMpLiLg&8dt}hr?C)I0qgo7@=mZ+iY0Rca1>Vd1!uv<)v8T^X z;B4Stc<{#q7Pxi?EpaaAWo!G>#eaj~Xx1E7FhZDzI^LnSw7%?;Y79&h&g{2NrfiVF z&u-!^fs9*G?0t^+%*utMzI$Qkf16;z)C$zA+RM?3OHgS!mFqa7VAN?R@S(b4P0VF#@7pEzl&xAG*0sVJ~J+AiYaR znWM)Nsvc{J)-~&C>*`@Nxy6h0PHzUR`i1+l&fy2Y^*ANMo}xG-nEt_xE^KIP?=rs)MX-PCW8Q4rWbRb? z7}kKW3+r3_?;~9{#F1-zDU=6^b>@emue+<>kwcwB4jssE3_RDd# z0*)F#2@HO9;kBa@_;!93Me6xM3fIAgby+H`4>5^EV?0Ho+{q%5deZ;TpDO)a%ssFI zqK5SVtB!7@d)y1;%D$Xwl^IFQXK>je!rtN*ho zlV*@4MxR>lwc^pZAykk)8!OuwRJ05w^+r!-+V?5TYwbe$KKGb!8ERTWF2^go|I?Wh1w2SfXjIci1%n> z;&ug?c-RX*zKvzBqh^xGPgGV^6+(Kuk|=7@P_Wzm9F?R0;IrT6Vr#GI)S{^3lRXDA z(RN=bt@;BGK zSwZ&4MhJR%5;;mi@RUwxwc%DE|2GjzRtxi)uRm;6j)TgU5mdbLKFie4#tP+bmVPLL zZ2mJL2VZMp)}M-^uKjtrcZsIeZ6L)z>ts)Qb%o5GO5r<=Beh`|y2fR(!#_Mok|%T` ztx|DO`IG7B~A)@&_f5A0Gqi_k*|-u?~XfebG(sa-AichM>{s z)#My%25QMuH0V8!>Q;3XuzQlwhvo*#n|{dqT0y(b3XEvAB8S<6AMM#1h*|Ce$*;4p zX5mwm7>sAZI+5U|@5&T84_Lwe$$Z2(;f!r;W$||fKVHy2KEK4DjCV+=Xv;6OQ3r!^ zvmalX))Q^`>!|!SiZ|Tu03xl*HTBYa=C;oSbOEcW$oL#*rf(vrtd;8^ci$*74q5^B3kCdpHJm*RoJW#wdVEpW zfAD-;zK~N9gOKxt_P9b8KGKtD_8gKp>}ASAC34e0vYb1~;GMsp`PiA0>A$=2uyE!S ztS#rNvmHUSU?r(TJNc?J-%+~71Eg(FS+Y-0;!jXJ9#!~K| zKG5~vvj;$p^*CG_sep`l zLeY>$&BeKXAnnzMxczG(Z^c(E{H-f3HJbwd784+6t}|8jXk`Uwc5^<9Jt;qJ54Nm1 z!=={5Qb}$)U(!wR&HVz%nc)Il#(hlt(#q@>%w_8RD*mAPWbzK($wEw)K=_t9l~InBG2N975gTDc7ZXP9v3u~bHelXdJtt8d*D{-5Ks=P<2T$GC+cuz`aow*Ot z_smI**(>Cp=HJGeT}N2e-w&CteiYYUI*zU#GbPp2C6vALC=+EQkV}-9^eUHw-1(Iz zVC6ej^v6#uy|5fJ;(ucBJS{U>Ka3n!H(_Z?3RA8W`iD-pVL?d|rwtvCG0T(aL#(hC zde0&qe~hJf6+?zthRzqaW0rm_=+fk%KIVi?br-PpUjy1N@SVENy{KjSWo97V&Z@Tx zxI6VtS=y>@aBXjIiY>m0Rp#Zm(J_f?dPb3G+)oy;rcvmJ-+=btqUp(o(Qy8+RZteU z9x7^Uu{~xU*)JFl4d1(fOQ1KD3`l1!Cq_ek(Q~|6CG78~hOiUyAy9pJ2t9P^4sFs0 z_;r#MiTt-?$on;5y-~<`ES&{@v=Stvd-D0Ob|fdfT3CWE23qafo@GW)2ls%iCa!$Mw}j&{5g`u&ctFberDQbfw-|`At;Le zWO`@ULfCy36va=%sNZK%%fyjnuStTpQ37`&y0yl?#PESJLguUpt^b%u%~2=Wd6Q`J7~72^)$5_rYZaLm3OwLcPYSoTq2dER zpp7mR-fJ_341>h*{A)PmyZmI@%~!aB4qz?M{$_gfBfz1?1lm)llFddh0eN*MpQbx z1B-%ZP?(P)WryD8M5Pu~`Bdly$tze9S0Vg@d77T!|4RB)o#hX~$-5$`^!%Ijsxj>6c2`eetz}i>#W~O_-vv{i| zAb+$FcO5h)hr4Q&^su02VZJo#{U53e49N3&n1F|3AUpFNmpyAN);Q{+IJ1@67%ig@ zF0<&=I$yY^Nri&NEBN@SKBTREDwB`>fTGSHmqiCUnEdE-VMc4=TUI~g5(Brfq9Sj~ z&ixbB^CetDKNBiE@D%OZPqKS~{Xo5QkxVr5I;PkgL2H1Ku*V5q?ds)Fx|b8OGTB_f z{$i#(zZ$)IgaTA4sI4v&?exZCPVqf#k6(+nMONUl$_Uijg`BOvpetT+Cc|T9lpOGy z>FmOU98)tkXwUZUeK8f6tY+F1M5bRWY8lnc#;pA%pOYVVQx@s zaRf8P0@tK&X4=nN(9C%lGzstZ(VE|sd4CIHAHw9M`ffmNhg;C*k(lCh#xvb+Usj^2 z#hB-|U=VLi$-Eyk9k&b1`h-)#8Cw$D+-1f2H(9)|73CHEjz<2=C^o+*1cl85e%UeR zA7w@Rd-QsR>N&NK-Z0iOExGoz>(%b2n4aMB4pMB}t{5VqC z=$N{%utw9=nD(Iu75^p&@uewD8vmZPg^Z>e%^Z;YKf2C7DyA=d`y)w6k|b27gx;n1 zX7=7^B$lc3 zYxPg7)tWhH&ffRF@9Vlg@mIi}!O>LwYbQ3CV?!llA%62s6=fKAkcJR{RP3D#<7QeB zjh+)Q?KB{9f7g+NeKTl|@p#l+IuYeBya20KPe2jZx1r$C>(LdJSn86r21)}iV4tVw zVcTJ667fD0hNZ9N=2%@YyLS_6nqg1W3fw7b^r8OiorvUNtgLNKJMd57!Kw$G7vL7h z+tSX0>Z-~#@=gM3-%Bo*!!n?dQ%dzc)i^%yt|Wltm0~ zi_zyln`qXUu}Hj@%kAeIpcNN3Qc+Yrw^sdK@*TgdJxrq^9vT=ybC)`IlapJN?+|6T%X>P%@{{~Ij1KZH8n*GKY!x46ft7#Iq*(90kj+U+!w@}iv) zFR~uCt-p$$;WqS{^TrxRz93SaeWbAC1*|TZLVdR+QEU4lv}jd0Eq}>rK(n_)pGdB@ z-Ndmn-8Z30W#efVU!S&(e-67ITT{*Zk<|2o8sdj25SI(XQO`Rr8&V#PLfsyaX6l0u z9v@5nzh5GKWG>?W;LfHu1XMgwkM;E?pf==8+iFFiyMe30W9QJ8n>S$p_E@SI9z`W4 zmBM(x)l{U?P0F@%`UU+tq}IWN`hW67Rc7(DpT7v^@0>x)(^YByj3Qt>BN1tQ+C>x= zBvD>n5R~4lC%wZ>XsJsb=liRJ-An%vdCN|q9IpXA=N|_4X1XM&cN~4=$N8J1XQS@n z3sK)uA0*!DjK$LnflnUiscP9k4m$3D()Fi+;!Zp6zT43l=TFeobq*47bMB;7+Njih z97;R(2x>@8QG?qKqP18Trj#{8i&_3qT2hYr=}J^=U`yko$I<95w*aq&^ED?`lSnDI z7Jm(^q?T34z}yFU ze!l?CYo?>BO>vyQh^u$PL*dAhZB$#imBeix4+B11p+v8P90&IrR7|`HC11;dCL%Z+~z;cRlDd^EQ{qnGAhLnQ{6MKUlu%B@uP3fL@0d(I6W?v}(Z!G_b)AMZ|4~ z)t{1R=nos(k{L!jQ@H-SMjYzOnT}LeHj*s)WZLMUL7%hHsAlqJYU&qHElOIT^x1V# z*fx|-R~tjU^fuF+7w4h2^#GUglu0(v;pW`+3utL|0y_Aoh$se5MbRBIh(U-AG9If+ zTg$}+U;RVE#D*xiFqSH37$IKm1_$RK`n09_04aaci&TRIfTa?>jI>Aqpi@^^?i zZUL#@zJ;uQwmOS%U16TR>L=C;R*Ywv1ijf;gcjPxfEOmukiYXoX7Ow%COdH^yV`g- zGeLb1$u2nsmp<}_vQ6LFEj_!Kg%2>w>N~{xnrO(^xP7975wF3?xpVRM$RO5QU`%3s zRnUyxEpYu3Z6=__9p>7VGau~l(myv|vaa=^?D++1%=>R(ZINghY0U-zB2%rOlLG|BuP`nab`idPz5gS&3ekuR`WK zZI~m9dCXV0eeC^%q0ENBE=JY4A7yylfbZMY81==a%(JErR4`A%IG!niUP=y({o>Qi zWgjv7Wzi7kLlmMK>s>fTIYCa5n3*4(&dBq5a!?+Lw&7$(cI`PcKil&(>C0t)B z72U3{r7aHnuwJOi+*%!(A znM+F@&taw5Ye=y-l}yv#ibxNOoF_~~7Yc5}?>h%jqTO678n%Z-94LWxf!@S^s1lvO zsRR|YKc&^}`dnB40q__agup^hyF2y-%9C@N#?M!fdDR&bT3m?wC5OrPDV)aq#Ap1t zBpk^v&7dQ{yhZ&QEKSa}Vy(>p^SQ+VwO-L;fBQ{F{hw;l&7o@;XVZ8j7&K)H=YBvr z>Jij+*AtX8<{#pHw18)REny0d)5ct!K$w&Xtw1*== z_?Eehw)8F6Cp(Bzcv@(^pq=C66(ATil<8UR%vv7Aw0Z4Zq?_{vX4M^moBt^=#j1+b z_uMSTBU?hPr|J^Ef-70P zqB&e>|dLyaA3Hs#^A7yalQQUrew&wL!?^dihit!rKcQcq2=p1zr^}+u#@xDT)NmuKKEC{ zFx^kYg7*{nMn!SCC1Z$Vs}s#vKZSk9mm_S`g`&e9;66Vk7U`*>r|$!(r*tnJa|=-K z7ayqCsDm`Y+WnE~<1|CbPBU)9$XJ$j~zr-Fj*c&)_|1E3<@# zR(lh+>6i(~NADH9Rh@{67FW}Ik6O?-iHvTab{3hoM$%QA-BD1^U8HI7AGO(g z4v7Pr(X*5?dNNkZ_!j!n`H^2~^zV9@uc?Dv%onrkB)-goJIB}?xx<*AV+?YB8&5rU zZA3Y;qi}@TXw<7coAb)43NChDN7d4syNygFBQW^5B}-TWP`N^wQM?)6e>m@-Owti?1(=`cm=#weX<%tV;|L*+V? zkjPFEjj*>t%h!%XKVqjdgGp{|=ec2Q!hcUu9G59>e(oN0=~HLohqjX77jqb%W23F6 zvjMf#T!K=@?4~uoMHK72LC3VN(?U%lqrJ+Vs@-ZJwdOCA4#4A#S-?lW+k<^SjbB`#~HXn`Bet2hF4H-06xuUf$K&xeR~R1Q&c zdxch4Ek%pAnsS*MSwwYkFS;sZNU_r_#BAv2^6C0bk1Wbr){n_%p%uY zYw3;S@8GzI9JG0)1C=ePLG#X(L-eQt#hB>|`#IclbOWAaCuU-$@4Jv@_!9$8NP zf?4z{;R8{56H1)t7}Bp(Kcbq`3e2sfqbMVwjIQ43h$sqhnCFG`_jfBGqYT9E+e}BJ$96wKna=!&IHT@m& z;niTn{i|s~m^;#FwWYU=x1$@rci}qr2#vJ6MA!eQL*2W>m?1kO=`yEcW<~r?$~*9h zTsRiSl#I=xJ;v^g*P#J=pfrd1HTZ%W<8*p1)1FN@e*rzboy<7K&SKeP``H~|Z!`OM z>|t9{Vp*4xHEcuR3g(gU7?V9!hW6Hfq(_z}vj2Acg69`0F#9JjW&aHy#%8!BF~5^D zndKcqQS>BNW~EL(z1+5!X$~}$8#tb3ho0?5^8NeSw;JEsvrd>^ooL6NGTMQvHBYkB z9_(SNR;EMm8*k~swFi-Q^Kg-^fj#3eNx;sTvXWi$x(fLXSh6!sYv>{CG-k7W9TT=7 zkGYwU%<9zUu_KEXv2V+ppx^k>%#pPRVb8*cY<-?3y2H#tFCL4i`|K3<)OQIpe+|nr z)_gX5`PpAX5V{2B9pigH{Wl|37FymkEXJdB7GEH1hzWZA^JG1Am;JQGM0vI-2@qp`o|vHEMz_g9z|<* zFK1U~S22m<0yc_wj6ZjpJBYif*JOE8?!h{6Mb`* z(x%-LoHkL)V*$^yrpg#{b8Za zo(X!$@EJ3@d#^ewvB+Xdbko@bCP|Dz-D&2}bbq#e+ji*s{T))cJ(IQFFzegR)>y+FT3{b3(G zdJ1L_`mqJE2bi+s+U!5=d|GaOO-)}fecO4r$o@E!bd ze;a!?PJl-I$w0bC^VpJwTiDN5jX?TqFT2jNm3c!6GF=|bj?MQ+)iTOnbL^(Wwzbj` zQ@xm}sYhv)Ru-(@DP)vwccA~){Xw#lq4ICuPg%=fbJ?*cWvoHnQ*?XbHa7pD7$uAs zQ7hY}%ntno^g^U45166HT$HOZu0wUGPgfp&aQPLpGToO}TEAhkc^BBr!)&Qe=Q)^N zSB=JxjbkPWCNZkn44OORG21uJiCJ6lmvsHx&8VFo!nRfVu&;z?sE@}8X1S^bqx*6m z{dYNyF8$O^2f$v&`qCtkkJlEun?H&@5uMKN4p_%%`9#u%--7A-T@pt6NrJ*eJjO*& z#*FB$W=u#OnH6ZxdY8SWz1gFf_K#obj&<*lMVElp*U-hMrxqa`o=O9)F>(w(&xoBh z8AVDFy#lzq~eKr1a~^zX;)ZFVE0b3;`WXTFhjy9d!W<}Fy0pUD<} zJj5cab13Ha^cN9tIvzy__q&(`1hi*|cGp^q*JnETE#Y(SDHlVO*I zNO1;}U}eNSkH5xvWesDW#j4W9*JV(r--)@ndJ*IC?GY+DJ)e0n`aZn7Q%CgJX%8yO zQy1;)4q+OyU1|6I8T8DAy+~>P6z05{8I$^9Gu^T06gpnghZpWmVPBp|L&JwHWWVIw zv+|N6Myan94zw*{->)cTE!W;+Pn8SVT)lMGqi_Q)+!PkZw zG6#z5=;QK1mhKtO^tZ02You1RciBN~zjPG4R-MOqS{pIA*Oq=wc4hvS>|$KsWYXcH zJoc%>7}57{4y?RUmGztOA4>AGU8)VQKdd)`hF-RDnZhYSkP^`tfbAC1KSJ)B)GI6Ad=S(}Zg*^gv6Ro~uDXI)C+c7}L%<^&V@c(*60m)F3EGp;cT=R47$T{oS(K9(k^ zg`yJml?<~ni>Y3CjTv;xrlmtlnEJ*!3>Z^IJ%-iOx@$ujeAkjX-8E;GKjpGfZq`iX ze^TaHXbv4W?J~16Xd*S=sVWMpSi$TW>BRKb{A9ws9#HweWcFHOHJQm z>L^ZPeoa1v^ut`}ggSRN|J+znX32g={ar0vI@J}C>dWAVl?`+ExgHymN!fp*Wl(Ne z&cx{PU^C}0svPx+2?)+)7tUJ9&NzFCUO1s8zdgo-jofvFaU6A+IrZHgHMO6h6SQA5 zW%ti8@QW&Ic9DmgcRYagEzj5?N+T$nHJ>i}Uc~BeJb{$%uVL$p=S=y{7W&HKFuVDg5>vk5FCDj}5TyeXh6b9k>7VRSQ=x>5 zg|TAJuJWhBUiPSEgF8HE_Y7$mW`UMv1)w|l8THlHXVxkYVV$kqQP|H>%)fVwna$Zc zG|!G8rdYrT5-ymqpEB^3gwLw?ulmD z?D$+{RPdd4Wll$a0YW4i>c)I<%OKZRC@_1*Su%cd2gdNhYP2-zrEF><#+ zniXJ=a#|KZpZ*s}adt7-P&pt}8Y8DQj>e4pjj^ox%jsy9IiG24^k-_f+0e$FSJ2^Q z&de>jCH>9YfO_0dk(X()H1X9Tuy)aFlzR6xYW2HJUSCV5NT$KqKU5-3RpBTm@*nN# znorNB=`r-~cDg99fd2KaL6y&~(A-;=Y@Ep*I&$Mo_D5(J(rV=RTL~LzZ_g-nFli8t zw9BWxW3!0uu5iS+>cxiJUh+GbsTquaX#Zm_tA8! zmLX<3+@~>b+N@dkO)PC5t;>iOYB8niQ`i*8Ei_tY#@s1}jCZ0pyTRlnU72tLl_q7; zcisl9d}9{wm^GD&Nzp<$)PjAm?mFe|G-946&SFM!TFCyDp`_#MDin9DkxFE%s+F8t z=)szkH0wB@i5BXjnsFzo{IEV--Z-A|ZOcHO$2i^Px4De@wu>n9bpfqTilSydrbux0 zCav3Wl`1~nOoNVjx0%837nBx+}>?d5VZyYdY>xN9|RZxo>(`%@HQHP&sZ zfZ5dEO*M-S5c^$I;M2W}kn(>LwqbG*jSntl7aY}Lz9sllu}!rwLw25Kc~570Eg#XB z-(JJojeChi@4Ikf##LGpJCyR{hJ%*1dCa`+9aR3ZnP@)BW!vvvV4mKQQLSf-$-wOy z%+)X#=I-^SEDq@>*Oweaz7^ckax8?EURNYv%wD4N<5x4n@yV>CT#K6cdLsTVuC}_C z!lv>8s=e5OUS3Q`%kRu*#YY_J`ImRm@0`WVmA`+e^1q!hv&arsdOW3%*yB)x%Qs6} z>Bj1F-_6Qw6|!fqe~IgIZ*L{{}fxmr831$7+fA&G~sVs5tW@ zmmTv5$`#e{pYO-$1*2*@Z|p7fR5_J7&f80U9gkBmaRxg^n$GxY6w>QmQkuPa1*06L z$9&bD$u3W?r4ffFGl{aXjKAU}w!vg8ZQN(U3_i7CYMs2SK`t0wzH zc@Z%-)MUI)&p;7R4A_9nm_{p@F-fD17{_Tl>GaWD@7`P$nSZ=a8;4fIxOw%Iq#dL; zPEBBV52|2xX)bj!n#S^`k6|jT0+8jY10>nUnn?|&=;Q7!6se@qRW^#q^0}OjsLY|c z^$Tg(vRaBg?Wi=dnU0S*LEEmXF|wD@jLZ1_G;4V&{qkZcGG>p_=HJEC%F2q{4`(`s zcRi-OAv(~o{T>{QzD$dfbEr$v2P)|-q*Bd~usrh$%9wDErpn8yV(|!S5&oSxEu4bN zzP+aQ#G( zX3Khtu5dYmQRM8-UKDnwhIZ{p<@{}xv?1FWISm}75xc!nO{FcBWE$dTg=F;CL6a;< zKPEez;@j6!?=9EFqV3KX|90C{=U;GYg@C~tZi@$_3n?>H_-laA~sEo&-C-`6-g zzH=+=8SV<4XAPkX9Zn#Z)sI22Pdb@&ZZhJ+&|zAEKAQb@8m-X1#`U>x!v4d@P1U#z0%{B}~$)uANk^a#?7HWA*(Qv&@_c2G21gQ!Uou*+aD zQ4Z_^SFk(jDI=WTbT*f<&Gq}*2e5kn3K&<+aXjW~z`6ydq~hvC*zs}&^fb95^Zlzp zia#9y{9Ef_^v?|-C;Ac4iZXyP`*es)oi3C{gai9mKXF*~Eie!>gG^6xAQ86%NY8{U z0Qv>6vL=L7YKlq!eHW-(m_P)z3y9kJdhCB%L{uk?gncXRV88<-qRLE$l1H}%p>H|g zM*VM@vY$Uu5X^)MX93}@t`izJEFc>~=Mkr3+Fl{3b$2*!lCL4fmI1ian%J;|80S_hirlU=26h=lP(d({;cj3KLPC;i(!)$;yf--K&f>gY~OJT7{1sI zN{zgrOTk8LE`X%^b1*av=9t1i3xv@&tjwdLfa85mB)L*KRvtS5{EPL#W*Y&ineYKL zwXY$iA$(%Wm*KRX5cV{dVad=-pz*IW>3Y11i1yumF$D%Eq;0rjvuUu|!dGIY|_C;0o3L%!fr8*_Nj zbpXpI&372o~J{~)z@=>`w_ov9Ro|1f8o|G z9FzHuBP{>#0!SIx4r<-PL5g5HDc^SkH-7LYo}Mz9oXfh?6-Pt+uSp=RXB{XN4~02? zeH=TJPf;(kIe_bedlW}8V-JRtI5PvUd;A#MWKaqIWX z*zJEK*z`mZYOF8C(OdZ>XTKG;IO-35E{}r!+O9ZP;7BaqgyA~1x!hhf4^Gz&g~bEP zK+Q-3at&-CFEmir>uv$tCUf8SaYsSZh&{4Zi4jn;SCym$>k z;l5gqWi|POOsvlbNprPe#y4L=ovgW;`3jKz*_80FqzRi2ec~dy#(_fn^Vqsf0?I)< z*3Z#{(rL9ae*9RF>1RNqFJUbDJxSR7cN9z`H-Kp2by;Xx7sjd&fcDgDz>#ArH5+Mh zEcAwI!-a7;qTh+jbS}iE6JCS120fwVu@w-XeSrh!B*L)q=RsT74G{YMCRS}<0rzl> z@X~-)uxZ*XkgvTEG{xN&UTKRb#T<{#MMaBa329a5=8Psk-fKeMIae+lGn|Nh5@1`s zrEu`d7HB$z#d3=SGABs}$p6|XdwOCew7%~KBOY>`7W*sJp3G_>4qc0*S97`gTaE~7 zKQ&{Iu_44_p*9$twT^TUIM(#dX~ZBq1=qgv20^!T0Iz>kRo{r=(Dt$|#0F8MXWa=|j&2CS0h*+0wi!&i zYDCKC%4H+B>l4lHa9GOBA(bI|#Pn|kE>}pF@z!}@wUt|;bV{bMleZA3MXO<6Rvi{S z4uee_&H#%&!@=}$HzF}vEojO0ChhuXL2ckfEZHrr=5N_4OVQqk4R4>v{RPH2?Z0O5 zvPS^5ZSR4UmE3c<*Im}=03iNcbgM7R5vXe&vAg0)dv<)-E(!GY5-{gvI zS57BVw`^hN)F7g@^aps}e-iM%&j+I43PN$4y)g0kNs#Hfikq2bS2v|Q5Y;9jspY@H zg(n+v^QAZ_KClssivomI2Un1s152Rw)Fnjp;fSm^Xebm_pOclPDv(TlPsq!)!$EJX ziPo3VP_niaq$K6yRfI=0MB@l=!uD#x&E>?fekA!Z-w?jM&oL2ht`PF?X>j~o1{Mx) z2RX+N1Lehhu;&%G5ANr29IRim9Aj6k|9Atje>DqMC;LF&?Iommj}`W1bYSJ*&9Kd7 zDNr2yAK>p%g|&Ibe}gkdCrI^b3Dh~?5|i9Vl9lDRf=(75E<;Xg27tG zP?2AZTlw)I+HMhSSg23pK1{=f%Q&VnP9))tx}^NfT#zk_6vqM{C1l(q@+;#_PRJ&EwTOzGo0UJ*XIBie~$Po15@-QJ( z%X%-{Fc<_o|GWT_<$hoinLzx<>;-Boxc_heQK2dgFx1Nlp4Vo?+pK3Y2-N_XjEMT_;Yta%VRkM0OR&rLkay3{_>}hw36evGPVXt4?hOcF}^_Zs8bl! z`UfkgEQHaMPsq-OhQmmyE9oA)k}TN236_o*vVU-7`PAfhoCOUib-!P3q= zaP-kiPB(K7ds<8;BBf?QpX*#Qa$q<#b=(Z)OZN(sPOT@6p&H~$b1+dUALKNh+;!7t zvF^R$1Zb*4@smx$GN4D4T7#ke88H}$NFq5MkmC-`0$~%BaPSQaE+groaB#gc>AxL< zBeLCL&EYAKCyWsWfimprtit8(n-j@>H<^pVL>S|E3KuTm)&&S)};B6-naGy1e$o)x7mJ99u3?aIhM};3@Nn z=)N=jF|Yu3-%x_m{=+iy+KJfziz5kZNC#D;DgbV=gwNyLV2}L)p~9RwBwAk;S{_b< z$|L-UsB2J=nDLThD!!;z=X0OSF#t>6Et8e+JuWNlxP_zlvqDjRh3vVEW2H>+H-zLB@ zp@sT&p+vieV~igy!@Sj(WYq@SVB6Fof)UWqr6GbUnLiG8tucL2A^;((Sdv&Yj9!w;0mxdtRB234fbGL%_`#w<3X;603YZbR%XF6!j7{RUa6G_j^ zGZ;&cg4|B7hICzs#iJ(5f+ijW;<&l8zZQJx|HlRf;Rak4Ig?au)PZF`mlKJCmcTI5 zh}7(s0j+a>FxShBD9{wXd>Pud#bKJqSo49nj9&|qFObUHI0?FEZ zVa*g3*!Fm`OsUl$s?3Y$_OtbXmr&_YoO&FHx~i&6Kh%S)#Y;&3shJSJ-;Z<6Ut`7P zfy6?28<0;IU~`ValbWJTTy`(ReLv5EeC`Quo1*~w^PhpZ91R@)Z5l~ohmoMEQAByw z7}9ZVBP^HB;5fn~iD6wHo@DD!ns0?e>zy7@A|4{Js6B(FKgJNH6^=y1>M-~_GYJNN z)rQ*JjETIvP-r@RCDC}03Bru`1JfN-NwiZsr?c*r9raQmT0RqCQ|88$t9B(s>~K{jZ*L6y{pSWRaW@KrJ)Ugcvr zuGA+73wTE+TE*2Z>T3x9VGe$?VgXT3myoU!yF;4;~X55E@wcxD8HMjJwd zF)4t5)ER1w@P`^Y8%TfMdeZh`zOZ)Z7M!6I9h{E9I|e&Q8) zI!A}Z?d||?#)J^lDH7o1cLzuB;26I@F93t4xup2cI?l(`j3bKfVUJ-F%nK1gjeU`D zpvs9z&0Y!ZU%GMitQwaQVI`E`&BlcZn@H237K~iuMWQVqfUq|ffcJTmjB{9U*>2x~ zbn-f!)-;-^J=!aj^G6W=fs3-1KaM14Tr0>KSVTIO$3wnMialb!;*KMeNEydBYIPb# z6c25M{1A?Bv2rYF444NCP4mH7jyJ8bYc>?OI|EV96@kC`70h2$Cj^UKpkZn}zQyeu zVf9}@tMOUfmsSRRmY9*h2j&sAd?zfo)BqXNwMhTzF}O{Yd%oRYf>ymrxOmlSAep}i zD*jC*O%)%3=Dj#*k;rilrQ>1OIeX$#8wxe*J_8qbCzv|mP27ZDBu5`X`PC|czx7j~ z`0ynMpUbUNE6Z{7H)U2GO(mtU(2JZ30z85F%ZtSS=3Z5hMWkpigL z_!!9fy`cMaGq#rO!0H*nkk|hnSjPnuHR}^Xaa<3!I5Yry-uMFRKNyJGu5csEl~6_T z4oGu7G?tN@ym#^0))@s-Nl1B9HUH}2`n_5 zWvvn&9M`-Ie_k{e_B8C`GB!Css>frH>lx1J)wU5yp@lHN%vV-A@h8Ynb|d{C7GgMIn*#1hjgLvBy*z%jpSKwry8kT;QmE-}HNee6Y$zj2iem5m|(C(NMSemThf{1@Oo zwy^z$4V)ghjc8??!WPTtAfLAw=Z~EwG`-5P1Ke_fm!1*S3TwwsBng(+Hp>)-7{az) zwL-5+u|%7hN4k%Vgr+ZU;o{8;3BPp>sChjSe%=L%+#>{o<7=SL$#S5rkOr=Z*T87T ziL?}ZkkIcy&7i$K| zc)grdW#~cmWou#L@_qQtwUv<1uabdzO;|FIPK~(lw8aJiC?@I&UH3BPl=)9v0>Y7zp$B%pf`D#h~qJzOe5)=i&Q0 z0z{^4A-!BbRepITmVS5-;)7?vN$xx-{&zrTf8-pl5}yXiQygH0l@Q8@Mq)$%7$Op# zuC{O9ND5s&h+B`4IK|0;rh@{M+*m5Sfm~qj(GjFI#|Lzo269>CBSA-o59#(fg=HR6i#{8 z0GyW_!Td@un7@0EEG^xb;3SUE(Vzh0rljFVtAn}c+YR)Ftc4mA_TpwuYoal33~cIC z0wOdPwuW+N!(k)AhKwj;XmJpyCEmx%@|R4HS1xV*8{A$YZPo)F#?J|f0NZ! zE5O2$aZofo7HZk1;EMTcpl6i;@(mBjZZS$kGhqzOex?e;1jl~?uqP1o(8Nc z84u)HD`2#(BEIs~9gb`r2Mr!Fa8Hjrsnr&N4WW7@H`1FNym<*bs)Z8$3tI@!uCZEq z%VNNHG$WFv2%*2o9oBISU{95E!j$vwq+F!Iv52_!dQ7$OWyB(q+O-(!2%2M-$g5OI^bEPr>7j92;B0UR-h+LJP|=Q=OY z;bB9v#t$RivW+k;sUCY8)e4(d7s|8;9m&b6K*EdBaL5f@Osoy_fm(Y44qI!D?Xe%s z8#xlHDSyTptDb>=b3dG##my{sE6C)6nX>$aTpx4v7?M9tAoP6c2g+AO!Q>AUiC5n| z68GpK@G&dHapxU@+E+ywKA$1Ar=4+2>;^dUjuI@oph8AA3W&XDE3UGt2d&HFaq|XM zk~V5Lu(C+%7rfT{^5$38)1*dOQG_g%Q#Fu zh?L*8h5kGJVEzGLP`+e9)@d*msJVzhYM2*n{pAcLB_beR&E*SQJq5-0tg-qRMIvE0 zVddirFvlP zIaSNI`NO|gcqA+tV;4aPHVttgaT;PMF5p75|E_?2iG1SN5)8}tPXfHubU})6GjWRb zff}QRl9YE@xMBYttim*43p5$}ulode?t>1A;0<4^xj1IqxMWbZ0 z=44Xn+k|%@y zu$53Zb~X%^&4#vB1zM^4D%=#s8-7L|JN?Zy5~2L+zYDE ztcI1pB`n27xX13TP(Gqb7UZ@NZZ6RzIX`XSK?haR|K$a4-ep2+ zN7(~o8Uhvmu7)lj8Cc;Wf+~MqNxxbL=sV>Jn@ZBK$mA6Oc@D7hzhIJkdMAE<-FpcU%b1#z~N}DiG>UO@Q4diKK7k4P1P8I9#!K8^Jz) zB-iEu)_8mlaN1KSdhk{j=c|vkR?OknfORB#xi$8``yS+5a&rM+Q(5`dKeEV=3B>ma z*Gs4vK`QF`q1P$Lwc|O6TZpYi5l%Lpy?YH(vyeFP}`TWXzt8p zTDUz7x@a7wq-sAZSFa|oo}5I!N50|Iii33P-YnEra0#kAo99Qda9}=sZYHE zwxSf9NR&A1FqQ0BjlS&)qs>2eBJt@&XsKgOM*nvbj`CR#KJ}O|`CPQ!!4a?{-AcgX$F5PwRj@)z`u;Ym?|2PMew~ z*iMgM7zR^1mJ$7;DmeDRVQO|nMBB&Yq5?q$QnOKq&s>erM=})+3%>=!zG3jtSCIzi z3`5KGR8X{_2x*-=Np)`zk|gE3@TqkhY2F(`s}GeSMM*aN^~jtsuhUWKQYoD;RzP;z zv8dkB0^RmLMbj-b=zuhk+>alFwnvrHsd_?MzV;GF|j!5=raXPYIHAw1LrP2QF zb7|LUF7JPA8tRS>5*7(^ul-#`nARtHTFUh?l+9i-E9c#eY4Tr>ACbD zn1#%Z-RV=0NXlEBMrQQ*pry8|v_LU~PPE%Ye_bqwN&n5HcSf5L+esFve#~V0DJYD# zP9BR$%|(=Q-xNJEEyOc#T|zf|)==J)74VyJB1-NoqX#cJX&0#K^Br{O`IUIqkeTS2 z_jWq;=^?mCV1)92d_{?x_4IjcDRsAALeA{IgihI|Qq#;X@{V$O|0gQwq2cb3|J0cC zq>p4Z?`;AhtU6oMbPg-t=dv-QE@IL7bXoc2E4a_H5oEsBAXs}Q$wEn_^12$dF7Sfg z>%6%7F&|f5xDCRLCgAqPQ=p-a4pggH4diDGNvoy{$4|b9E$u~Qg=r`;yyriwaqocBq#3NX%j^76n9u_VGC>p zW1t0IMp`#_0mSKh-n^a< zHJtQPlXSoHC8DTjz(dOyS{_se!mAXO$Jb;Cl$2i zTSHajC7d>4AJ(!3(0`f)ztCD>spt(G00M z+li&W_TZkJiNG3M0?F?zp~Mx-q7OhUE`Ek3=2yU@2o++uB>-ydegxva>5$G{HMr_j zE(ngDPx_Wy6ZrrmYnok*O>Neb{P98f$1ekJU!R37`ZYk>7X{KW1d+J!xgZW)#cf|E z0}zrxf*ej`dyi;PyIX|w5B!n!xkeH(S9i75DM6l}1$3(!Po5WbV8zuxP(EYlVG!|YbhJ(6+*Vl0Pz8FppXBl8aXdRKP?BqOU6H&G8w5#;pJ6c-b z1)ZN=BPBPxfU5sD*VwbGLF&1hdgQoZ#;-|GHQ)`We71${fhSO@xPew$U89#z7=VV@ zLRP<%RC%>8CWZx4-GWP`vQz;nc-0WqBm6yk4H)MBMErVYA9)j3lH$5SK0$bBei?U)2fvky^4R1&VvnFv)s-LbB6E|~w($?lbRU0?N*pk%WlhPBUTpWjWIesMeX-#Qm^heTlt z9H6qse<{_yMI^t!yI#)F$G8p2ke+OXmMctYeBgfaZrgCwEEh_eLfk;_MKpv~jE2xa zrrWR_-TTE%V{dWf{2z9~_zg=iWAiPNzS>b~X$}Y3>|o-t&dX2E7nGF08}!n3*JL=v;7qx}Ex7=|kiz50xZXSfJT(Ns&w!>Pv!;PX%WO9pZebE7P{8 zKt-E2`0k#E@4kCsyrUne{(8j~Jle%6CcCj;ah%8RcsT5A2;nt)Aer*RwJk&&O8*{@ zCR1#{CT1$zTXaY3C5xcFsg9Ohi2#M(7vaf7CAhMkXu@$H#%>Kp_3f`hW0%#Kou5L~ z-7T?-#!+{hAjpgkgN$q^Y}s{>)?D2{6d&a%iTjKA&oBe$ihpV4PA}BUwuQzk!_oP# zsc`P?&$0-x-5-wXoR31C2R4k!Q1 z7Yj0Nq1f;XQ6zli)J-0o-qD$$cp$^zy7MH>teU2dw#Lr-RAO275BIZqK9qZmqLP0; z(x$am@Ggbz0G$?rcQb`?Ss+4<5sqB|Kwhe2gvYjskRgZjP=|wrkl|7u( zu5%=QV^8KcfD#YW5Kzy&B*eF5lJ?dQG-l>;5;s2#Ld_FVvg25Z?YO_`?R!HZrCTJh259l{>>@Eb7?5sb8BimNhMcKzV-12JzHBetzJQt+Ws^m$c%AlfkIWb z0d`caqVCG^z{gF*)Go7Wn@JHpyhn+Nts<3|?-V-XkCED|Bfz(MC6uak!S~Wts=Jcu zUQgNHXQw&WGM#ZABe&7Igghe4{Z3T<{-GOp&xT`iCye=J0xb*FQNM`c6}q_ z9j05hAa(u3bVqXq_?}|>-L4zC6u3sxbC$4YQ6CH9yVBq-ZlGM7L<8rN%e&Pr#OMUNcq(&8hqCe{XIUD zMC-nwZv2z@vze;?!$8n%T*{eLu7bRrH9+shg09taDoeZ~m{&0t{I&sF<~L9LYrZ&*u{h&76j#>^5 z09E`dSL2d@>9`*&F(GpWmag@}hRa{b`430wfs5TRzIz%;(Nt37Id{qPRx40l?eD62 zz~oAeoN)&SNK|(}@MruYKdvX*oz{h7v5CE7zH{2QA5wXRK(sRoiCK^nb{y;rssVSX zWZDicZ2nuio$XD^4l!N%=P&NqphWOtYh+S= z#@?gSQGwKdY66A|E}&uDkIZHkJ*i|G(4X%F-<&S+bG;K?QX;W* zb4@p9T-vcvY?oc6o=PbeN_LQ3n~P*iryokgte~yuV`5%03&R2j!*BnwU=bM&^bl)Mr`*ELavOEssOZCvtumjQe3Hj%RYe$Wzel~~$*=k7r`2jU)1qPS=iw5ma$v4_Ek~Bc`g46DRw( z#B$$LLDJTn291k^C*DkR_+|-bS#9r zb67WeD7Kw=LEVq~L;5T$s!|Q$vOiD42A3t6Vr+w9SAS9eb*tFC_KXxgb^(Pmn^`ta z!Edo{P^Jt*m1#HERMY!3-IKlFj_NN9QVnfyM@*o(}*q@rOV zSm!j6(lK2?ziK(y4w+6=>iMp=d#thIs~H3}TEfpXH+*hx4iZz=mDutJ@oJlZ&fTul z=Si#aRh1=N_%I#R_KSs-+vCw@k~f&I{gseQy0xQ1^2Bm>MtvS{k6x|)U zihXNAvg?x|sZOTG8>XQuFuJ7d`3k7L#-oWi8?|o_z?$QFm|MkW8!rc{K3gO-oG+u^ zll@WM%}mI8dXz|~M4-0k-_-A|9TwFh+-WTGb&~XnT55|K!os8^bfcv~3 zoRkg6+V}lfM^Xqac-lgJl_;b$c0wg6Xzq+iA_?DfO|dkB1fP;abB;Nd-`zr0)_aI* znJ?F@Q9!`EZkWp$L!+sTi;hes-rL#z@7HYFyy_m+J@KBVB!8o^&mz$KzrkQH=?3Yy zCsOBq%TTs30kk(pH^k2JyrWy}~tVyp)>el1`?;b&kPX$-^fq5&Eo5@yD=3JYny_9{%DxCUJ53|JF02tKdVfr_nD(fCR>Nl1+c<K@`G z*+jD)o|2PC=RwN*U-@druWm_E?p%kJ~1tia9G3(pUJNsA}6(9Xw}*gj`4ke!!l z*t~G488L+QlkxcE%>YzKX$y_+Qk1^_$oLbck39H;6x|;Hinx(N;owTzd{~Li3%8QY z-EKtdp)ET*{t(RPOvbRn<*4%NOFtVKVxS!l9Vd#3YWsJgA!(@4wj+)znI@d-H-Tuj zU!zT%0>D%rPckQF(afIxhLKlGY3`3 z9L`r&%_YRKIY)2=v5xVS`+K71bA@XN)4QEzR#c69In&82sm&V`&{Xs$IlFj>6_QYH z{6~rHuysT?+JF>{sUydz8%XuOaJ4UnkhJfkfHchqpT-Fw$u@LRo_Is#H*Rxu*kY91fhy?gMqs6XSkmjx- zilBO$++P>%7|Z6|xCWXFm(#=+#*Ca|ew1N787Jll36hym6&uL>W#h4F_B>E0&jg*N zR`}D7`6=$YWAL*|n*PiUZJm43`@Vf4vBel#w)DY_>*F!q`y?@*wF;BMg5dJz6>J}| zjZ=C*6;zBNDm=;VCdRtx`^XAy|K7rM@D&)giN`#L(GWK064kYl({YV#CTxu$&0A;S z=MnMXzhWU~l6unddmSx)mQTY9hl7`WB9waBpknN8EkgTPL2i1s);2u zwT;SuzY*-a<CB(zfYUoV-1QlO@ih$C|uR@97fscrXr3 zZhC?E+6-0Pj*^nDzM!$~0qULkw9YOPKR;Lm9fxfo`1g7sjL!&I5(QEV16updjWP8R zT&;Q?(b`~y(woz1+O`~a-*f@X)V?(H)(OT=mk3Sw8hM`Xff>eGR5$tu zEwqk-^4MY$Z+f2YO5xdjFQ{LF9y>Eb8 zhvG0yrJ~h+O;Fb_jc|gN*(IM;?yn|QIic7Xe41!>^d!Do{|Tu}-JmqHJKXQR08$44k&gN)7-g)* zoM`skzAn6$b9Oe^d#r=>a9hG6ZQ-{y+sk}05^8V!MeHNCQLTR@*j96l!0?as+8>`_{?kjTF!3xUfDh|f*}Atx*`ebgjOTWSCo`p2Vcc!q1OnY@H$`$+KXS*$lO7=9i$gqV!2#6GDDWb4$@m~o3Cp6L;yov{J1x4#L<8gG-3QeCvEwMMnUQZB6jNNCe% zceqCe*clcEsgK7(;)(^tR5mu87Q*lLP>k2VNwZcMpeCUQ*a>66 zNz~9z#~jwnD5I+8y+TEh0U9kb0Po7MezAK~S3EyX2@>}otE{&HM-?92ob?Fk(!$nv;tAI9m4(94s{h-b^5-Mq99`Mp2w0?AwK7nLtyjw}z zwok_fZX9YdK9KazMKoj46in)4h4zLKn6o(^60MYAL%gtkgaQ-!iP*$6!*PoSLSort zs^F|ipkEACUF4~JrlnwcP$ncdFsKXzlu`G;B`|$!iS*+g+VrttR((i;a{=fltEn__DYBHKh*qz*%>4pB;G;wqQmGzuO zEcfi9mA%|p?qd=c=F9w@ZzGBNKToK6xSdF5K6TM(?{uvxxJPAKKe#jxGdyoM13h;& zk>6(+C+0j0d<}x3^0^j{8#NJ*ePjNc1>Ld4o_TlD%t0(>{UxuqQTIFgKyKEOnD%S5 zQsxOMy&e(Ym0yI8$X}#2V#{c@0(4uI0`R@fTggQkDi)4J;Y zRJW7e9f95L>ld=zcncbFWirvdHHc+HM`50RFhoT9)8I#zSe-QxCAGhbBVN}NZLhC{ z9L#3iPa<00VVc*?2#}=x?J6(W>MGT_%hA!xPa3NscT!V3)*gAT$=wJ zD46z52-{@{IfGVWh3jI_ITQl920ds)x5J!#Nf=l7=RqoA+QRdD(=fgGJl9kb0QRjv zi6nAm2}?tAHU3&i)l-^COu!+cJEbes=)WVT9@A+-b^QqKB|`T zZ!s9a+B2q|`WG6;J~YL!!jV1-qf?mWubhKaLMOkR|pXmnm9y zyM#Gku80c?2Z)c(74Xe(wqf_cVD9Dv9bQMWh5t$I#mqTgVBuma4$zO`?JN$9U))ZK zHUs;M-iso^!D~K$)Yx291gWu`d79|=ayY->jV^y=ua3C$Vmv=CA{INBjf72zGZ?0b=#;0%R zKYwHW!G0(Cp-*$+740i(+qZ~ZXh0_4wRemd z!ZNBZsh5apO-ICK$3$EnmV>HKYH{A3y`qzt$A4HffbVj4t9X>(=?qn-sMpb(Ps>`v zpBZB#o@9Jo^|6-9U9pCf=I2T|Cb6A$pr6@>;)Uic@m7 z!qgF_eCzUI;`e*(@B9G#!yT=deQYo9*&`dmI=kY81+6%|G>y}`d4sY+mrs4YSzP)t z1WU*73edZGf- z9&Hnc&fO=b>@E^&(O4`4XYnmv%10#a75R>Oyt+0;^#7tKzTDMKOkuhHX`6S5fvYcx zyEo{GF@Z_qU!Cjl;p;u(_G4;3>9!o2+Ox&ub{V|yPzSNWsuI4v&Jg9R^59#yY5YpN zx%kt&lD>SIAiiqc#qU_&gV$58<0D){cvX1;|HSyRIB1<58aD3ZmmciGvM=-a`}Hn- zkNnHx(Ro+#(E37N;&7BMvaKT(^UsN?Gjzm%@5P8OoV3Nsk6n0^%r*3A&>*q+eLmm! zaGdz-!czXKp#)3b8;OsS=8-4=?-mU1hUs~Xy$bdc1M5@67oP>XwRZz-9g6)6hv z66PFf#J0Ifyw)@i{!GDKalsZP&YeDzSLo~#b*_Jf*|#q8^IuHoPycDa50Wn8M?a4i zGsF%2%3Q&+0R=@T7zqmUp`k$s#u z5A7+w?mdt9-${7QrvtG2ssV57Re@bS4~bJhg^Id#6t925jo;vZ7X?Kg%raxyCIQJT z%X*LKGIBm`3^x%YPfXxj3%$h1Ry`_pswORIg?#_9yNUL|0zTSw2|ql_nZIc4#o{eDb2>V%0<|ew{ADC+)rB&mFap*E1ill|=Kw=?lcH z<|4u2Ko^*iSC0|STls+pxA1L6Gx$ekdtGMTVtuj)$M9FUDKKoxZhj)`tra)#WfGNibo2LiCy~~;jdil3d=%##Dxc1$ylagFV6ZcSnbpjSKNCE8yrsa z!9ogqo;zHZ=ckD3e+Lo`ZKM;lPw-V6chG(Z&xUs_KbPzGWb*NaL=_``=qxo+;Q^bL5E%@N# z>b8L0iAJqVi;6d2Voj4x_dh%`KmpLSw|nCGcM;f)7B zb<_-T^Sn@T_%#ZHJ`EQ~^obSei(GJ2PZkf|yo2^VGR4=wGWlOy_6Wx37W4f^_J-xl za(Htel{m(=tN4<27}n=*6g6E|@?8wsxiZgMyx1|3Px^FNOc~@Tx(zWCBMPRAZTUl> z_Em)V=|f+>(nTiP7jxymRGFAMW2U~gfcT-@ubuV6a-jLtdWt!+`d0gD|UB&l~ zJ&488Cy@KPPb`|Q5U+-B6CD#g_!ldt@jrgd6?YCOD$!d105`t60Q@T-(c|t}@yz`1 zt{Iyai-VqSN0~p%`2TVp9rm#fkm<+yUDb;*bjwbDx50jxbjS?~m3zUVN)N_9%!8hN z8j!n`17|lW_{p`-;>|JtfoXgrs@n3zd(M6Mw}W%VI%^;PSNaywX52Q>Zfi3nytzew z9ZQ9{A4hq|o_1pBb!)o5xd!j+cM}chUA*x#AOHCkB8J`E%vZ0P!%v!*Cq}+L!H*3Y zB_8;_i|_L-fj8Dq;_rI(;1hZ|l5>Z>MRku4P&>MiKldVo-x9u2)RPy`!%bQEr!1GV z7_md_oi{?9G$mH-Su~1IoWl6sY)kQ3@o+xPW;dTQWS3ZYv$xpRP>Ht>rHUq=cQNdN z8QlF^fw*!n{Fpuomi{f!k`^0rSihCxx_`e3vn%gImXoP?wX37J+VeD(#;oI~?VrKl z%+2AA?#>f!8l>W^e8TJAyanF<8{p?T1wZcne0*;_S?s1aUL0R{f{%?!8>ROU%^5d(drpD)=l8C#tM_H`cVL02x!a$A z`SJqaxa^R)PC0w8iDaGx@z! zpW*5shs7g3>L3mcd4t`le9ci0amrdZ@o&Q{vB!B{oZo*ie{%%O{V2*1``$k;R{k>v zB4-7NmdzQQxpF7X^|KYvtJ?8k9Nii`R)+H-f|jt#^21Ry~?M zau-icoXvla-_ENX-QnG8g*d{>lfQd#o4Ah6ikAOm@GA{N#LffJa6&=IxPz@5lEgH->&p0BC^*G6&`?_CzeEtfaIPJ^77%Jztn#_gy z9(!Tm!+J>e%jeI1SBq(Bm-)9puj9JAp}1WuAJ3cqgkE2f7PUSiX@T!iKXo_ncCCmp zoX_d8Nef`^A9HxO$&*m=ycaR^mJRwd(_{`wfw-3G$+>>+dZUfO(n2Q{r$QEbnbO6#FMpZ};I?G&zKI zZnaTS#}{fH>!^iW6d2}(VOo6%1e;8OV(*bOV3QxFU1C0X`7q+F$994tOQ}xpF}UTF z5t!WPRnn8*_BSRtSZW5@bKX5Ia{FX@m9M5`JS3&NBtyI3ZLy$;4plXR8 z?d<%MCOzd*;;`P;XCKqNAC{1Y!IbknYC{zJVrcD|Y7+1Notl5L1KG)ZjvpBelCf1T zC6}gwXX#)}pE(38hq9k@juQO|&Y(Y`C#nxCxOV2dYO9OK0@X{B;Dq4Heki=ZMN-8%yV7sd|1a2)O0V~#mzMADnu&y-A5xa!L+h?KY zOKk}IJ{v=JEkTL;4b{68fzLCg5Xcz3wxD#u(|rxJS??3%{;#NR9%H2LY@w}#6W9)t z!=xSaq4rk~h?p?|4bORE`q}xQa<(G(hb_ed*)~qoZ8665Y$DrVdqGZsGt%-{NNX>n zs(L#iIm!epZumjUwJcK5Vl5Pr>^)J=i(tKaA{Q@zLdiudtYZAy^QS)8KHvpy4hbM-Jyhg?uoBu2d4uh< ziQwONBS}~okFGW>XCreBdG*}}OVleMV^9h8Jhy?W|9wM>@gJ5m@>^JA$TAy0Tp^YZ zB8BqYokXEkz?pezV=CJ{`$u=fTDRGxD%5}$zMaf^4VSn|ZXCQOf9Z>Cq#z+y-e_6% zjFats#8qS_f;8(Ar~7>x(ObhfR=%~Qs%S1$efI-NM!Vqo#+BsWUqw1f*AaE8ls?(V zyiD1{Fz}s}&9pxB!lhYIg-cLxunT%`6sefOd@(`<%~G!e)yw7*UCRuju3b*#!ya*Y zqo!j`MGsImN+G^+B<%Xf9hz?#K!$53wKTXzlOC)@>HATbXR{VltKy07B^3>N9RZzY zOfReaz*Xn21pnoRczNOy^y{}4;`+@%lRuoGdE^AJG=D2pOfZ4A<&TAiVLzzcw9d71 zZ#mQVJGkV7T2MQE3<>T(A6sQ+SY>gNvp-c%4pjI?0cI{r4WSj^pisNjwzt&Va25y#X;@JAoSb49@9LsV_&9&qxq21ybwpf-K-At^6_( zJa5gRww?hr{nIVZLSG7(#TX#2gJJt$N|0aJM3jb;Nb{#i((Km*JB`|jbL0@Jx$&6> z{ymwf#$Tj9{bsS=-_zIBs=na4porua52L!*hog8V3e#4-Am^7HB!f3W?6mNSZ4#Zfq42rh0xzheu>1gH-Soc*>K7h@g_ZwR%$f^{K6RSw=xe3Jd@&lUCClU)JABeAO zGS%E#DL5Z3p)IS|q1^T=%a+L{i5%MzXvcw}vO8K{ZY3GHj#&5f9^J?^70YN>qCS0= z$~LZq%qJyuS5_p;X>sKu78&9J-AHKrbd1s4 zDkE|@!{+fo-*kt9SQpHiwvVx-ORkwb>W-;yHKhDt8)-LtNQxtZXqDszId*(HO7`Qm zpdnMxx#ly=(D7s0m0FBNK27{6)5DtHljg#~G*D|aITke%{Qv%sr2V;>DhIA7ow;SC ze7};MywC>@m_|d`iZ6^2Y!!kxev6uN96iB&U^zVgA4&ceNaAoo zgG%q?q-0G9I#kYww)af;SU3>f&2>Tku`51V$+Ah?>p1J9|B)zPmi5qYD%C&BL3#EL zsxk54(sLM#fAlTMz1W><|964B`sxebWnYQ%!!f}ux;q|bnMCSh<`D{*goh_j1N+(g zY1+KO`16x7COMxb+L=pn9Q(O+)d)=3AB=7D-64L?Oq5w)AkK@+sr1u2kPmyqe5y~m zOs(oy|#(Z!Sq4`;cnZJS&mBPp28hXQ+I&CFU}H#PgRW#N_WG zVK*3G|I3jaNSh4O!+;T&l4x6x-$Gf!a6E8d4k4G_FfmdK8h$M#wmP~j2c{>yOCAOa zC1Yv-n1ZjebTRkk2kL+187+H}PxQX&W1BC_i7Wg`V^jO0?=#Ds2prIx zWURhHRsJiWe9&uB{KrhvywMx91`NfTC)%KHGZdr|BZ_H}X8W4`)l(@6RYHkR{y|Nf& zU6V;jPhWIfGzc3b-xFi)c_@|nKp|rj;+OtGKEEA+)|<8wNeC5E-<_li_bx)Jiuvv? zIuQ3^gVFT%B9`G14vswsfXtkEn0>Nb3!CGCDC<^$Mg zU1ph60~fn-8Fn07!ZN%&iTyuTm>sI7ISzqn;k*XLA?s1ccL0jZ=R&5j1BMJ60T~}} zlC1ArXve>`wDidmYzulq8cG+VZSY7MvHl(T7BmDKjy)3GxdL8L1U&pL7#$pkVa#@xpYyE-oL z#(30x`p7k$+C*fp{b^OpBhF^U8i;Kf0|_&_v+U>Dpc4vcZTp5QFVDq>m=(k_r7N~X ze5PeOjN!^l0#EI4q#(eO`sIW{m7g;;8PJzKb-Gw<=0eU->H($BCSVb}0-ManV&a)k zEHA=V2(O+BC7z6(Ogbx+9XUWHg}KgE#S&V)_YBc}IErS<&(h!@m#8}F5tR)cMb3}8 zMGJeLAST_92Y5D}CWL$sOhL;}PxXaAXwbhd(`i`Z}!PB8Rb_6KNaMIRi zDY4CcKyzR&IG*Z@{>mg&EDIE}Y^US5;87^+Dp03M3NVemMxCAiBMo~!L7ugxWcze) zDCd+!GJLDxeJzuu44#LTFZ9sp?pO#rJQfN)dx2%88);bHLumds2(zzh4t^KihpdV@Pc850aI!~mRBeAgbAmhOTV2v0Kxu-jc@^l83FC7dL zn=H;~fic*e(u3SvUPLvTF*e!OczZqb(P(w&Ov`7Hs>{1buqma=m~i45^nggFoS@aF z-O(tJ=`AexM;`P}ur!LM!93&rb(tr2$UtJmtcra|SXTF8+I*_Y&MbFcNsh8I>NKbjoDa2*m=f_s4DCmXqd2dRJm!78) zDeFkp6$g^?;~r5wJT81LTMZJH#-hJ;AsCHyhHpD3fpW(VfjlUt7VqX@gO!3N4o#!B zBlnOYVC_?k7ce{tV{+~ zxWLu?^PFy39D(;g^g`PtUD_zErJ2cz)Q$NmG~cpl#GmYaRW*hjp0|YY<`JMCAH@ac zdZKykClWlMgPe3o0N*9|8G|fAh0_sk%}aaqJ=KlGEwyA`tQyYWgc8N2WnAssC^X5n zV|iTyEgYJGJ^6d>t)p$#|`?9yjqRace~ z=qHDcn3X77&HOgHT~T7O&NV_di?;R8r?PIVNvlm)uoXGdG;JML-T5L|u3F4V-u@t2 zcH3##mu*!0vo*HqEJ6F322fsX4^3ZJg8e`?XKBvSroLUl`r}Bn^Gn8vIvZ&1JsD#v z&e6;qDQNy##MKs$X5TX)%8&Or&tbl#?ZP%e;eVA%pMK(exd3XneK<%m-?_NnaYXfV z9a`lwlceXb1z))ZDPJFq@n??`-HLJ4H1Zi~XCA1MzZ_9vc!svGW%s&4O0f7o8>FAU z)4-reqFiK4)Q6mePJInk&Dy|e)^!om`{dC!#XnR#HU^Bn4iKHPSkz;lnU+0)Q1IRv z^@=Q^ZTde{`X^x-K>j3RK`-j&ZHUTSyNSB7fJ=2*M^Xo*Q+3UJp>xCwqBfhuRW-h& z1>J8`U9tpAN=z9?!J~20X|l$}1q=B`qIYl=RDQik3oP1&HF6vL);A6>r2C?|W;Uj^ z?kDkCIWp?USr(IK-g$I2PB_1x};Z(CF+H_)cYEx zvVarB^_Lc?KVK2(&m1Bxc|hb9x`O&-JIji8hc=(TiAEE^b>0iWwE1={%V9p&T%QH? zTPL7xSU9)_tV8M7cA8ZE2Whxa$VofrfM%U3$W5#S@4;tD;>sQ%(E0yBnQpe zZL~qJLx{iYhWggqNd(JaDF5b5Q=b*nY?mrpHALrk{Hvs|z+^FTxwSw$u zrf}b34cZvZ$L5bL^X&5!`v1J0+;<)z(X{-LG2Rz_QoG_ebs)WGQ^kcvMEc@3=XrQGYNFSpE}NfoZ}cUO-y%S_p@A{b8A5^7hw=fokinQ| zK4CSOz8;9q5n42r<$pEy>kVqR0`?q@1ya-vLOuF|?9vH(K9%k6if(arai2+SzLfPI z9Oev7dO`XkIW_!m5zAM0Bg*zGT-AiPL~}@&JFs&gWN-3iKAn?X_LyKKTkJvUJDX*} zv+SabErQdUG3fjKCzZ^cPZYHg%n#-OwaG)U_?I>Py?6q&zFPoQjR~}K;X_h2dpX;+ zdt+My%Z>PwMbiZtm8Eat(&wHQ+IG(){^w4RsIBuc*x>|;NMpWd^E>pF6Wd3m+0c}O zE_h7z08>K?T2Qk9>_h*i2cAyHj)p1V^y4=z&b-e!)j~mjhIzB|wh`y&OHrNOlLX&L zfVSX6WN+4V>~#G?PreVru$*4dAwH(n_Wn>d`zO_Vx)jd0OR&K%i!lliPBk$`x% zkJsD8`Fc+!?Mwafvb7iZy#dHfOrqvTmZA0aEE1vlLE8G6(bx&AK>p&HkT`!f8MkC9 zb~b85^~6P(qNpLRYe%ELZ9kO15CseS^>`rP4zd$0(c60~5x*<~-Jj1{2S*vLdRQW; z8tcdbTPZv5ouDy40WG&KhNO}cBxa$8>K8FCI`D#^{Z1D;R?kL#tFD+gcMi&pvS!Z#-ys2<6(6#D)}ExT~o@h*S{pcMqB zbtfqmg!TALXBnowA=}s&KgX>E-_Rd|+Hb4Sv4?3EVb7dZkG%wS@4KA04&#DPzZ9gt z9b6gn-znB`XdB{7bmfDI<}~x>-D@HWwH+o=mRHv+2BPwMVxce=t#40dez0nn2K|*> zYAV6RD~qWnJA)`Zt_!vk?a(X24lmpxknva-O#VmF`3J<5et&$BBq2$XkR(Yml4R~V z*C0tkNRlurNkSzR!bp;YwDC!jB!r~3NxILO9X5mzlF%{;p>1RR2;uvD|MI73=HBNy z=ly=Y9@=nL=6k87X*yP~?+$U#dO^Xx?%=2FgL*$^V)Ek{W<4Pk~zz0lDVL7Q1h`?)CZ zctujje<@cT9S?N|ap+_@8)a9M@k*>g2T>wiX-gOjK zL&2q3mTdf$$uIKWQPFMY_k1OO9KM*^ZnS`L(k@6l{vEW1h#@OnNb{!w=J$C6iDv7l zIAT6|D_#h-pFAn=<`S%lIn3;yhM@Z7CCMUxIqwj)vCOHBpesm%yGLWmHddcZeXc`v zSri7uZUd-n1ZTg_^dUTiBA)M(l-GNMT7RTuQOaed&`XyA1}-ALS~QGu zPiL|Q(HCWR6WFdV6G*kZJ=RYQL-n=Wg3a?&p#C*Ml0|)&@W8kv~R zdCLp)SXxkml=t^!O1Th=Fc^?`%Fu7z2hhgGu@>7fXdIUTbyou^K5qbdd+cShEYGpp zXFCK3%}X%au#x01)A(6;0OsGeMCa6V5boCj;|Fr?hVy19KB5*(fAyl8l?$nC$VZku z{|L)iW{0-kPSRj#%Ai7KJY{L0!sQ;bX64*0wetv-fcB-i8cG zv4ry~vMviAYdTT*`|IG*=Lf4W3qsxGi_D^iGtXCarb_BgdOtgJUc^w=qSQklH&e{4 zN2vT1jpns{uJl19BhPEhd&643=ei~oo^OZRXTzY;=`ko}xuEB9m6>jO47o;gDRH(5 zR`wZ#g}Xj--ZTH%yNy%Wd{AQAr9j9wIm&8+F0lTUgQ%d-dNBH@5X|}c&uW)98D04V z`YSoZX7nZ2vb`M2VmPPyO@`ERpd04_J_6aW&gHc~4baKXmiPYdLjdnWCEQVw)kuZl zI;IFxHi~5M+X|{&q-L_Tl~iVC2ikjkncd`C-ml9O>ZU1~;R)_DNaDTdsW%m@BVy<@ zHGCT(r_7&AXwja(PYal?#!YhjUl`ie^~Q)TS3%xu zI8^`GK%rX=!1k5{>1U`&_jn*?k6nO`=mF;aCPQw9J!^f^o2j>?mq(OmFy~V%v1xn` zbCoUQpOHTJXy!_KG|G_QWk0j3kBsyN|DfaU| zELM!^N2@w_(d-=-sVOarqL@d8n%_pIY3!yNX>uo^X6di3qQ)*nOsN|I@%ybP{M{kV zr`!UTE)3CJgs$Qj?H+t_#7VR-7=+2~(lvigTBZp*b(0j~li5ul2Tk+xGNyU;1UGyc zt4Uj!2U%5zH6Dfg>B)m#n&PB^npklI_OBZ*CgdI!8^1Z?g!?~G-z`V8uX?-K?%@JW z)cGS~`8|7KMU=59>k+8=Vq&1#VY)>;Jn$&0*&Feg23CsH`|Ti2=WD0M z3B6Lq-Jchcy=$RpzONFu7`&syFI_ZCW@OXgww;>(bAH#X-?2&)nX*EhIUq?K|M)oS zT~5$MF8K%BJ{pQ=_lD8sd&}U|7?37@1lf1JHG}%J*L?ePuxOXJNqks3M7$Rnht!a#**Pskv;O8EVrRW2n)RbVbB%Kf zm%JUTxu39GoEz&aN}lb+>N|_MS3OTGUFj;m9yUz$GTW(X5o|S~Tc6+^TYYiGe??+~ z^F(pq=6U5=V%7J&J_Q(Tq|zdlP?B#u7kS#7|oiedo}LnR+>3^ z3enW^l*Z`AZoG4Si@564c5%U4V@>469I-+@SmV&IP;;Gg_qMoq$5G!$h^=ni@4kBk zw?GcYoB0Wvs%96Q+Bj1DP&QCAaPBeDa~z^#={m9Jv?lgnOAk$vTY;wigYjb9<1?Bc z8#Cz}ozx7N+*e$m_6z4#aKo21Z@|d%vE9cB8VAiy=rH#xOpo6zin{KiFKOw6ZQ0Nh!ca8(cevOvDN17`J0%wSsc=%vQPJJy ztfuPPAhG|hWU*Egt66b6S!~)fnHC;srL(=;kz;5w&&;~u=!u`P;kh|$YG;X$3)*Xn zy7-D|!%k{?=XArxeOt<%l5#YqVcRub+Ak2Jx;bhlW^WO@YA0(Z|Jz?PdD9_^2-^a0$e-&_NdRI-%_cZZCY%Se7VN1>n_K5>Rvc$G_r_s~OM65sOp-Js>lHTc! z*ZkLGr|7kDoMhb=pc%d)ROp*-CTECc6ZdPZ z)qTZ(rG1)0xe0mITc?VX1gNaJ?p6 zSf(jGc~mTQIHL(~R&ejRi#Tz`60t3NGiT3DXKs6b14CI)Qok5ouAH4HsZY$s%y%=f z{@~v%=ez~VOPnzO`BRp0Ivy)tGu{>Ej92$dLNcGLhAxZX?~k>RpeM(wS`YM#UVyGg zjUjhyu4FgX3<~Nxpa*x&>4rp+$;2KMb~Bnv%DI0^CKp0KyDqm^e5&f+!NwX%9>Q1bK zkgg&Y2d>5I4_r~@t&u8y<{PsCzg|u=4FpYVTuGW>3W2$mx=i{{V`3Jp#ggLdbk@PfB*^24#o3 zbAH%ZlZzl6{DP@+P|3Gz@@2n=gKgRiZK&F01tG1;jySlpAFazXsJ!?IFz5hl+hq3)!tZncu1iK0C3efU>77w`dbI^mV3e zrwKgM?!@%o2Vl%&OFR}ihh){BuCn7>%AI6xWIDo_LjP8>)*YPHlVl*chD^f3<}NhL zcmQf2J}-yc`&kudRyHE%+xR{KW2Xero1M)3b|f*?&oAI>Z;UnX4KQoMEYgklk+jwg1q}}I<=}Ko*R^_pRx!JI8*$ArcR|}czJRm2# z8>upD1-EDR^x?oz)Pb3hw)2G);XRMDg?PXG?}qYk;~kJ^Wx;RtLe3(93zg=p$uD>| zu6wbLnjB2f!+sX;M5%?E{f-ndcLz)AKM%cZCt}&DO&E859GcD4M{W22ID_c-a@`i-__61LLzZn3eMd!Oy=JtL)!{ zlxrL$uZfi~y`~4r+x*cvs}B`VNoC5nx^m4`{!D7P1;)!ANv3`_)*-7qRp;eH;`)2c zJ7FozIwwOmwHAV7ud(PHo^MFK$wKELM*TJf4RyoOp7%ySk6n(Mt*dyqGM^dkeax=& z9^G?;c*^}lAbG)`5I1=!6ghK;L0C_$9J`B6*f^J}+DbsX#0*sx<&Y6Lgi3yxp|R|# zWM_Dl{p`AcyF2ce$NN-+{x1{C8557i&zpsaJ-I@1l|Gg%nu7shrT(jvEa$zR5IJU8moBhfBvlBRk6oI_oBJ>irvb?{#QNsNxRQp(thS`DG z(7X|~`AI^6$=|GH*dEBtp390Y{|0OCWmJFU60`m=g)+Q$gZA%k^hh^@v$(R#tXgd( z^#L=BgKMx*>S~_{$W^W za+S42a^71<8F^+63&ooK62=UK4iNxPU|zwscO zojq1Dun)F&@qp^a7Ve7fg8@xucs7&g*J@ufqoSQqJ?StD*}9$_sTaw|@H_F&%Yxt2 zJFJ%9?UWIdn0(T1;p3qvP#Noteme==61wso^?BxTgu4ZYJdmE$^v9OyuVB#D3)8Nx z$7X#e+qqwe|Nli_eGJ*xb~ClXe8t8uBM6o*!1RPoXzIv0 zGwD;&s4k2G2RV|*uzg^9zZ>Oq#$;&ck^Ij4mF-H8Bh`s{pxqjP4K3yv-Cc$$@-dXf zyU^{+L#Q@<1GbKgq(?6!$t6pmvJo3;<-HN8Ja|kn8odm?z5fJrhkk5hQU}yaGREAj zr_$Yd{5g|%LuhO|#aeszr&;eJFyDM1L<~3pb%u+02dYkT4%xz@pKN8VvtwC#^#oG# zXSmV^Agj<2>x;I-Ek{!fea81=JQLb(P2bMQz5?kEz2IJ&si($!LTXjU?YTt+ZU7thZ-vOk4 z*Ce>~cR~5T#Z39X(V$LJLS0oDYkknlyhq*>obS{_A&sIIVK3-sPQ&(H`DY=8&rg?e zro}RW) z{SWH)1d`RDFM?ggHE6hWh?Gljn_McF%m6Iz1Gv@;(Xuw=N{u z{S2C{4>Hd)qwuYVJ2hWiM2&_$S=Fgj78hW_j&=5+fSOIzs67mI4SXiqaHw2+dA(43 z))?(tJ;`oAclPD5YWD%)o=)l@UdynPqa{EsuX$Q_d%_;X_1Eao^f8Uwy%fqHBQ z(2CC?`F{CN8flV-2VmA?(vNG5|obRN7CIWo~3P2km}}YN!?41u478UYUVX&zRna*a)zwx z?gAEC>5aO6_kbyR7VY(HA+7u@=XI2^6lG5`KPO|}b+-jO%h9MbU&k4^2_Q?`Ah|zW zL1yh@F?gXTLd-gR>%W1*j5pAw6@KI{F2&@9{P{n2MS0zuwdkD9olRr^WQJ`W>Eos* zR?_DeGi>feQJM3|ZM{IB8?4c*e;HWLybN|t{ES*;g=cS$LG9!x3f6T5s>;?wRmBu) z6gpApT@z>?&>7_p-z9I0d1T!30wPkj32{ZY!9kY8?C*D_c%!~#;W-cOei)<6kzjK2 z;=A5Y3oti3LyGX&3RVS^QEoqj1@vv;UeFL$HDo`N>wYq;_=!w)(3M&e3)n}$uITY! z4g~jj0QK8LN%oJKBIi>Wd7siJ`JCg>H0Uxb-aCaF#`mWHLk&xtct~=6@r9Z1uxE*< z&Vf@;6Yh8Pz_dNXp{|8k%Lx1mvmz8arbP-_c!(amJb-dqMBT|3GnT5Iy0Y zjkH~*(2~G+t6rS*Jc9EJLvAz01uLo?w2rBNToWP;)?)tXwdm=-2K(>kU4V>qW>8iF z8$WX&&RcyFgr=L9Vy+7D?(A_S@1~PGWhwNS!tZer^9BZXJ>(_wHP;eVilG?nS_HFV#^S5h&N%BUV&$uN&Xj0lX=4wvys}=LGj59!v)&02{gzSx zqf2p@-6s`i`~@dN z%&^{lHFuc@u$sgCUAATjX*Z{Umu44}Jzgul-Le7;_HiHA*|FgNVJ)7g+JpD zwBNCmd>#m-5FIG~hZ}vn+mQ;poCb$$iI5Tbn#DQwg!rKI;5cC@scpGWP`eb@ayHqj^2}{pSmxGwkY=CC*o=8tb1Md?$IPOZ z>&C3IxgDzZjhFJPvsw7z#nd?L3j2_+kNFeYAT@0#^VsyA!Rs&FGr9vjN@tV7Ut_6Y z)j}{aF~_j=^GSQ66vB5;QOz3n`Md!zT1z zxE?HI^QpzEKvG>?#4=_?q0J_LN>w?c^{ha&ce5mjtz{XZyP3;|4JcclEmYX>j#)(E z1?`2kpzW0^v4PIy_j4j@R@-B-as{$$Ybaek0*!{X$E3@gA#cmQH(vs!v^OHlJ3E$S zJ9n0stlbQk-VdgxgN1BgttW;?cV+dC?MZRafM?d7DDK!QmTC1j$Q1lOH?o26c7i3< zw}qV5r^2|8I%f506|-41iquJNXm?zJ@$ILe-`*D>{ymKH_nrighm$ea>k@nWY8t%? zTSRT8BDx(t3hc{Z60go=Y+1^QiI0 zU@&;`ixs~34$829A zCB}5Zvc)6F?>%=$y4_**zs6wZn)aAeBPX?VLkPUGkX+~5 zo4I>zO9&Hm^Vc%F+DdS}H3=U@n)8fanV>#wB5BJvNglFptYGa+nHaO5fmIzkb1*cR^4X~nQ2F%>nqOs5sU%%|Hy!O(K{w%ia^8@i*j>u{8Rap543SyXbQf@yaaD`5K_=G=b*TCeIr z;eCnkM)?l6U^si!ryVK}jYm)UI0}fLfeVYK(WMv)Xm_zhelMw5L|B<^$L?p{~ES56s--Rw-! zp*N6~z7MumT?bqHi4+#^N&)vr6Z>+8d2fnjKL5F*`ptf!CGsRFm!+5AjpcKimnDLF z@^UFIw;$7vZ)ME25OizWlgezQklfglm06jR)r?7!QMY)?*f$Nkb)H8a-C|k5_$Vw` zA4l3F@n}9JkVRND2nI8UVRXm_Qs-zDp?9?`araxEO-mNMZ4E{04yj1}ypPbBIvO@|ul&g%d){U4#G;L+VF1V}e`g3r z&nslPiUdq7EQ5uTn%`8bcEX(Bk-fp0}AFBUC&gTHZk=yo-=KF z$Fe7tz_Iz3R92nC0xTZ0G_?X)hdKD+!X#?sd#_@i9`fJV9ko8SkhXIP?K?II-6H#t z%pnJwJ58jT5glpQ?LgAjEayGU28CN^H_R4pK|v>C<}wqqR40S+j~Oi4Wh2izs7SYK z78suDM6UdI#M5S+Ek2c^k5quIR|mX1fM;Adr!%|kJWDTdz|p7KYyA>rrUkNGh)6lw^=X|#==KfI;eQGjU_LtQZddCSPMs>vio9@`0 z!tX(ea2elVHOb)t%0lgZ9(2UCk;xy@YOVUL{78V`6=%lH>e zz5ZBvy>&G_8R&*J|5?zL`EH~?W-@x0-j#Ikc)o5y0en6^5}jj3IyR4YFD_RIRkasc z%YRvd_jhMh1$V{dYeyl^&IwbmZKS%(q44OR!RVJWn(tuhz{P44+LXEQcb7R-DJz)i zi40afXgI}f+sPt2qzDe)%X!{;K1(j-8R7%Wgot8(zdW;pcXzDN&sM|Eo*#y_XM^x; zC*D*2!TEA-K2UF;37JnPbH`6K3lFPjJ}xg=NY@cmnH&jVJRFm+SVP)KwIt5&jjy`& zrM!=PxAOivTRAG4bn_&3Y+?YVh9V|invOQZvbpozh2l-*=p8WupZw{ImT%58E9xe- zJ}_ifh2BzTwF4PMS+e%029SU4Sd0!HkFr5a6}kq3>SvqiZ4hT{-)RRCW6$xq%M?Ku z)SaB2)0uZR_dKLdp!nU^X#P(bc#Bb->9(3Xk&2m>(=evAT#T05FVIkb1b+ItV(|0= zR!rZSi7KA^haGr7WH%&#YKNuoyHZ?qPqZkDMbqaAP`7UicS3%L=9TxP9fua^OsA6LMXd(TRlJSQcK>nK_Bdzf(oQnRHQ zW?!05ZJ*uA&9O5UFX26r<*NlXDHXB{Cxyy#&XT`#P|5-BJfEv1bt(Uxua2c~yw0?G zub{ZLA4G3H53Sv0oEL6@R`;efWcnb3`mqJA z5ukFuFS+dRL*c!6A1-Dmdo(DJ5)(MfLR%rVY)==Q?EP^1iavDRQJ=h*JQmdF7L}_N zQRU%h-a*@ii%@-k7(2OOCE0czP0CTPnQnTxWbE@zXyHEE>I+Au>pc`0{1@*Hv-@Bi z>&ILlAA`?cBglZyIpAwAtdtF7eh+M!{V8v%`+0$x4WB`+H~2g01@|c=j$}q|YcXK+ zD~LNCMvW(?P|9!? z1Rexg$6pYzZX4YC&4WTGSA(f}J^RLI0s1@op?dhea<^->EOYH)rfybIep4u_*N02l ze%82a5AW)I^B0;XdhrbGIdGii!}%+tv1s-h%y^muaUDF-=CA2!rT8kTwaX-DNrEbv zOpVbJlsR5y5%1@*>UZx2-%D|H_NEm@+#ABm5AbJxQ9qV5dKo!PLp(EUux3C-p{8Xe z&$oxKm;#$vsn%CMAF}Zi9*!9H#ICIUAR*X=KK@&{GKl>Dj^No@< zi6e)S!HPn$YkDt@TVf4+n=2EaUm79)7ZRa)`tKPE_D0(N@UZ5;k$>Z?J=--G1HHt= zvNW;c_w|~q3A;7*DZ535-DeESixu4y6L87K9^#7;-Wu!TujoIDyG~CG(DW_tr#X?b zKy(?Mj|M*vYi8!lY3Ar%n!y$c;@*N{T(#ziCf`(E-qGAjtgO^xKDF1htKO#ZugjpRg~-^|;s@teEL?w;+6}lsncGyF zdinpvxF=J@JO3PEaSA)SdU}}Vc=~ej%lS~Ta(fwFIJ8@{<@$ZfYqn^<_Z~upZ%j3Pxr?_#9HaSqD_wIdaugQd&E$Qw-=&rr4+NVhd64kdlN`F} z(~}V1w?D=2%4ZC)f9oiUZdyZLg`8c}VKaD7>Pto``yf1&&!~TY&15HISfN)rlkL$7 zxw8|%mERGnqi%z%Zw739(3{L>901v%A&M&+z5`622L{Bw2Av*)tp9^@qxDm`Z_o`w zx3xj>S?+uYo{sgUhrsH)7NRdV@mzl+6wbQKw5Ov;HqeLaN=8tXxPvXcV~<749QgcN zhO)qJk^&;|+W=>Za7<^e?%r^HfDzC7cZ7!byf?IMuR`zX2E6`>&28}(+Unf!%G-hTc4stdgAx?7n78E#8~c%B z_B6^qG>(kBYyh{o@!aEaSqSdg2lu(!;AfL?42T(mgEcpd9;~An$ZbFqoCjGE_WM5OxrfyT4=l$_O(4)*vW) z$6-Y0T&Dlr8D-Iq3U&AriR*Bf`y@YdbE#tuo%_)%D_=BrNMcRA;}@LBoj@Chv&x;k z&lqDMocz(7Dr*nm4REcemhSvWBUQBZa(KJc~3rl9Vt4lU6H9JLICm z>Xw0&^m+ni2VH@6%>n3V;K0H&7SQ#ho|Mjc+{TeU=zaUX;Fkh$Nj;ICd>KZqU3q89 zj{jZ;rVFK)gV3dgv!VF=vT{==Ft%L5Z1}uWxzCX`aBhD@MGTm$zCqdg5#aUTZBXr) zDJ0KyL~pB(sP8@!tp<6}-IE(A{Jvr)Ed>*+Z5)>?#y(1DbxjxWp=qvyqEl^Wc!=R*A*TgEM1t zH;fgs@qa8NVF|gvwB$RQ!#rpI4|Ds2?|YWz|`9rKUm zdSESm?ihh@9nDd{-3p8=52MV?WGL=^Sa3+!GDX2uvM88}ioaHqZG1;M+h$9u&K(5X zY-k`8b?jcjXs(=6{A2LgSl*ZT!TSyIQ5cc6 zkQA0{DfhK47Sb|uFI+(z)2+#1(P>sQc?-0R;O-ZDJxuG-mAdV=!+;z2SlXneOg8a6 zYgBL+)59QEv;P)^Dtcq}Jq75-Z)Cbj>%jZ^6=|hS5Vb!RM!_!J6IW!3j$Zs3vr)(A z!857!^?J&Eu#CQOpHM{M8U~dQN$OSKVZBFGrw0?~3Qz_C^by zpLBoLh15$QmCO4RWS?40IdSIkB13`b&BR*R2NKnIGO4@29n3VQkc+wC~9Zi6_iQ+C^vJwWDp=gUF*MKuxuZc z=;CPOZ2_y!mqPmQ{M_Gry+ZvTp9$>j3QaTk-uu}ws^0hxWT81@mWSD;&QB!ylxwx^j2W+J{op z+xx66YY7yb8buSPcA>P8lTZ=sNh)JsLB8muly@xzjSstF)4E*DwBH=e8caRTg$nwgvR4R8&dL8``Cq(5;n z7Qcv~xF2!UyuJtNvRvqj%o#)e=S6Oj));qhGUz&SpMYK^$lCb1cUq(({#*lhJ$zuf z)q5n>lG9TAjXvmQ=1ne5yvr^6F$b$eR#nMa=r2c6?wcW!o$4+4%|F0=J|r?9K37QF zYYEyRrhS>ddViZ0{hCQ8sz_Ad2_n<{S)8Ht zlO^csp>v-dP%cyO@6HNJ`+Qx>-#?G6p4$lNi$Sdb)`eFk&ls z$2Efet3mk4ER0%iZD;PKfhaHQ1%BuGo|xxL+JZR8N6lRT+n11QUOMC&cS7Ch+ftl1 z5Q2a1f}b6hP~*ZyFv~3xv#}@N>&-($7emy=_T|&dg&5T_40V2~;B8YP1o%8+9}5MF z?thoL?YzQXo$#ahO})@!p%3;S;Eru4+%eaBnNTVAqJl|EX6(itc((szmJ^?FC%KBk z6YH34)os?2dqtQ~vWY4l4MY8zmKeGtjujXtFxhW?ziBBG z%aGs4dmaPT^uCbhYsI~-0?7W$mW~Y>jZV>vao0hf|6M$mXDoj!uim7Fx`DCm7@rmT zPvY6Nm?xmL_@+<}S|K?(bm5)`zPDT=fo_&Y6@ z=Xnhs$^9Xaal>G0*irxnJ8PNsp;@Rrwv0};EhD@8%fR;7I8wImDUYAeUF>^SL+Ol# zloV$~8UGF--L%DmxOE6t8|F$f_jOXseSfMo;#s>lFIYg;YBc?4Ao=tYDNXN=knbML zGbpFPZTWeK?vcmjVdGe4&|H?uXTS1C_F%+WHrjQo1m{7G4C2>d+|&t7zAS_(tAD_* zAalA>7>-7&5+?6i3EHrOOkUBMReSfO8V8=8p zP$K7-e0XR#_H+=8?lzmK^O&LLn&_8JeYJ*=(-$E@ZJgx?k!-6RO zRu4Y+JzZXH&>qWDW5{*lD*8B74+Bz^)HdM|lw@oL-Ct2;Hih@g>%us@umiI*T1G2p z$ualZEiiD`GV^QiLGEt`l{uCeHzFLA8D)^m&w?>4OsH(mMtb!ll=J6;*vHn<)a{S? zRJ)L93avB!#t%MY_v&d?_rqmb2xX=KP>0pUz9IC1G?}|tnBzmvOdUjLUm?RKyWsA77s?{V-umSy&ZSF zt-y%f{>;GA8yqeDd8Tg-_^i-C-5qC;5kSlxK9iN*RkW1MBf0TR3cXg$=hH)w>9#X% zhqvI6*%^zj4nU*M=b_f27paSfM>yGON~x5S^wm2=;YLc5|49VQJMof zDgsFRgYUQE29v?3NN|n&$-t*C8f>vcV}C!4_`DZ%d;TX~ddm0Td3;w}w5r^)c^}hT z1LQu_m9!gOnTtA%>cxvtJo;~trEV=3i$?Ol&j5Uum}Bs#7d(GE8P$u{P)+Bh-2HNo znYDOO_Kc}y)4_$T8>1;_Gk>PdStNX%y_JQ39!Z7kds4KDXRKEyFvfpgUEysh;QDZ^ zPAi7mTYb=ND$k!xI0rR7uCb~Pd^dCMcb@Nag`a16kK^5}Y#U3$I%?FUVv z&&MM%D@vc<&TB_ydMkN$wk!Dec0u)__CihmR>sdc@O&Ka0`L46(xzMxiv49Q^ll_} zlME@}E0WB|1%o5s(}%bWrkLkLNZE_$cJI9wY}TwItFNAd#c#%#sLy)?9ZICOPY1yG zdmYqRa0lY14E9zVjfuTG^hEVulh>r^em{h-6mzfI0S0hRw=do|6R51AcXrW{r(Qc(nqe?W_k_GbxP*9y@IvY z&t{n|LD1rPgE_aYp`ulrP=00}Iha&3S>vM%s?+C~Jj#?(m$avdt|gK-eFxajwZ$S| zzBk!&jAf@UBg2FJDFUp~XNDb`-L)l|Z6NwYRIrMiGmz;ti847yAbDIhD||B;>)uTy zt3K%xf2C5Cn;k_I)k`tpL-Jq+)jp3y`LZtH!+XK9bBDlg|6pXigOR`CD)a97T~M#u zfW@H~rEl^jXe&6NI@J*}6GE}+)e1`dv74o>Yh`Ig`&rUUFU}OYQ*Lr3fF}6GV4{Ne zFbXeShzMH^4ehxDi#F27^9qVLpMgy^i|KmsYI1k^ld5kbj^A$7_HsItiohc zZrTN@G25KG&X%ILaTlgOH5*)?@p;YyCybKEkU_VO=(|6Za(~}0$&`*4Y8LDV-P!ge z!vzZU1KvH*e+cS6Wr{-cNt9!6$u|<`1hG+pA5SdBR@?F9Gtz>)<`odS$B|s$@a#hr z|Li-=rT!0sQ9JCKq^(anhG9%hoZ{x+cZ3C44OI&vmKKz~rk-VXUWIi#7_+P4`GQf+ zaCWCPzf(^qyKzCFY`qAI`i?lgb_~k5z7|qnDzSRuVoW*dM7l1)f^(uX*%-cH?T-zn z57Q#aYE+nzZ?Xm-jkLv|tMth3A22gORso7{xu^%j!<9M_tcp>}|yu3gGh*RoGNX7H=e|%=OS~xd!$X$*HyTOjhVR ziL(0f-A;#JOj&qFh{$b{o=|s8%?QSC<4hIj`XILuuntcgou|2bHV)2(I&1 zQrj0t)b+3enZ$i#7G0U;dfw*!p2OtKoUqUB4kkcy1Pa z9b+(Q(0Ec>>r-~)dvGG|z5dMx8-n#wT^vTLyA9CRXoHFNzd*g?O1bsiiMY;X16nTq zi`9L|h3Yp+g4%YsB6PSuX6`=<-uG_`@=!~x;B1daizeg8qIP(_>mbV9Igr_9Rx|Yo z7fH8SPIh`FP;grxbvZ2%;_paCMVqklKr$q!`cj7VGq!N5KKIG>8Y}ZEb1j}@$r765 zG0S)%#&udwZUt$e{F^Z=pD}3D={9te`=czkU777WfwXnSifAi7+q6kz!BfU_hM^@H zoc|8;S2D5}+N1i#B*Dr$5Q<+$fq%_ZM$sqIu$KP?j+JO@+J%jMkdHjffc z^vB}*L#(XVYLw^NvW2n$&g7UPDEsXw&o=o7GEV#nI{9O!*fS2z4?lnw&W|oWF^})( znuTLDhl~ehNqOrAp^D$hQzQ82-O&NGrZ%XRWtC^-cPD1L92?bBS(&{fh4%1eEp>k2 z(3ry3h4a4N=wPgT)&U(p{R_qxk*w(I9BlbLnYkSF$NFa#5EqdL%H$}O!&S(z>qk#+@m`Hs?58f8 z`mc2$kBwEVETtQ{t+B?|QR zHpGse`#hgEA-_XwL&)#Mk~X9b-~0Oq^m_HGnfpHXxvuMdSxW(2bJZ36Cs>sYAUDAZ zU1kL^^Gi$7_tP;bozW<$3T{d@9zJAo#U1T_7O<*{=hl9`$ui&U0k7KA0Ppw=Si|QB z_q!~)*bS1qjiI#Oo@nZ}2wa!hP%YE5hHcrx;lK_UJjaEC+Pq|$$38$orzQw_kqxD< z{Xny`34*$QXWE13AaU7Gklgwh0yjDGH?=!-q0G2ZPad3Z&2f6WlheXUZJz>s%O$+Jn7>m|ZI|$DH@B_N`*cBUXXJ zNn#_aVpvVSPGRT`vrZ_KJF4S#hAZ4lGUVF?Ao!%U~<`sTz>Q9*{C?m>Uf;V zTf1;CcQ;9UhIgeEtf1<>OCdLRFjXsevWDTWq}1#dW<1vhbaIM%Vzn#ZF{@bg#pTSU zUj)Qo9zs>ky{LHfQE0f3D`bDU}*5e5>*1V?YoNK(Z?lrzJle>8cCs}r(<@}b5=Up2jd$Dq2|R$ zb;CLjJ{NbQ2D3joD{db1eay4I^~=foVqg07z?3R4_W~2Q|1pc$$rQU{EGchoh0R0M zWGZK|{8^{j4c;Gb`uADpw5}_0VH8ydQA(0SF>l&O7BKrcgy@XS-;3``cMYcIaSNcL zg)_(+GMVPWI3c@!8%*>8GDP$jd^XKs;K6l`=8;0Fv5<3AyP;yj3iXWtI&v1;DvUI| z29@}pSx!2~h%+Y*4<2k&7?GL!0Vm|sm@PQJgK(_+}FfeuynI&;UP(Qwlt>>P! zXTt?eK!IRl?7_Qna=@{M`+1xA`Qj&;?JXzJT=Nv>mW?Ifs2J2Q+b)&Q znM4h@wy^j>p10^4$ozxm@%_YOw%}t-?WkeU)Nd2J5yE%5%?_wJ zK31qSs+rDjzo5GqSmL*@E2RfmQBjZSRCK3?8CHB04D;TxhJ@Z2UGkD?i<7ZX8c9V< zZKxpmE7UIJp4Im6rL3k_7Ik4FwjP^~$w^gkdnUgNj+IFI2ldSTDreVpNQ zj`Q639eyZ73aH>+uDRSBw_top$UO<7|4U=KZ%fn;N-a7#t;QPHgDfxa6ua^J3YvIh zB~|a{yYQwLP-J7r3jcn~D$ZX3W7Be0pF5Rqcso<$-{vH@<9bUe>?x-CTgnX}A%Pd4-UZx>4mn}#}T{$1bnE9kRZgv?8!s5|k4y7u1R zpls>O=lUqhbSY!X+e4`OwZJO94+ycnM`3MyS#avIgBf-xDC40wg$K+h=ZfB_7?f9{ z|Fj=WRM(;M?0hz4fruIR+EW&cMolAPgV!y|^uK4|E_+}Q&ndmUF^}sk@u*Jhh6RIr z@qKb1d|I`TqT>d!+(8@RQHKRs($1VS*EcY?&9SISsRDQ9dZF#B2)foIgaX=3pyX9v zP`@Oe_f_WdnXH#o_gw>}2?Zs7xyz__fD_k3d?mhGW6?h@q34J8u;1QZgHw2rj6G%VADCG^FP5S9Gu+iL;DvJ?~&V5kK!oh>MYXtI4iif zevo`$*RXGqbI8)#n@o9rQTN**L2GuHnNIL#sh$ZK@xYpj28UqXu<@u|yI%@9{}Z#k zV}{CtzodA_x1gTKnFVH6Xz0F0NWSgNwt1(}sq>s^Jop+Ywwx;2ywHUVpCw^p=`5_9 zI1N%flF6Ym0QC_?oI4gQU7XLoEv2o@vB;h*mztn^<4WeW{3gr#ycn&ddF0f}*<~-+ zNr`8)xW|R}upFBQg?FE_+`9f)m{k>e0J`|wYu}2snp@T za3i1l!!+I{%0{kH+a-()So<%t`sX8m(=C*e7AZ-0bgA0D$2ZWfdMbUp*q^|qkm*)> zL8X;DYmKajsykV%@|!mqAKO7~(^8@Q2Rr)KZ!+)In?(A+P7u8$2p!tZ#)&^oqm;QO z6teg@G|ayVhBu+;-<@kVE;EGmL7gzjJxh3cZ8FyEKFTvu-6iAlyR58bEk#BTpgmhg zlGTsV7;0ETrQcqa6dh}1y6n#-@!L1DV*kr5@D~5v{cB71zW=eBU6+{8Al~EqDGR)A z%_TP#@7**DBeSCUw6l2}rr%8_qp}OcdfbPs3H&WyVS_&Yf3Q%V8?D)&2kGDVdnbAg znysFTk1V_?C7?4l)qY`>;j&t*DbV3do-Hw?VS9{xBe?j zetMHRhYdpYGg}geaCW8RV7xJZ9r*?}!Mn&|7;-TP3td{FcHC4rd?o?y^FM=CngI*} zGbovR&C+Xa@y6UXRR3-PZu9e?>QPy&IXs``epX}g4$dEY_Zi?oD7suF(C+<*dG_my zMelmEtYbs4z|536-8#lHyrZ#l${A^!`2z~r2G{_a@mlli_UMVQ-SYLBf@R5*src%~x1f_vZY_k)l}_H_5J zc~k?hAwo@ou;e`Q?`#@G%o8n`llMg-SYc zt_vxA|B$>!&A`e{BfxMYP||kFm_5p-^eJ4NOK)#L}7l&?-2P?9=1XtV>7E;Ot5I?x&fq-vq()N7{NK0Z}&k#d4J4NIgza;_q}cA><7!6Lgaci8Pol^XZLq$?d`er z_E-u=Of#ho(sw!{$O5z=ppUTHstjBI`1jC5AOA|(YtN}UK=)!;^VIiZew~=!=G0qgGMD7 zqQ*%T@o~)f(URT%w;M(rR?+4GmRLJ?9_t`2M#ZZX^~F-Y3mWU7meWiq&+icn+LA~K z>NZrj@jT~Aw}5K6^uS;V?H^cgpU`}dtFDR&4auGHeo0h6$Uq{iI$=}hbC zfq4^6(Iwy`EA@<_ByAieycvytN)z<|g}>w4&n5dDBY)!_lA4+xu@)`Q2Yk7w4%z=R zC_eMN`{GGl-=B!-E7Mro7Xgi}zTmLZn=IDxU6Aq@p<#(R%lYVwrR$od(*DCRw%Z?I zD4z;eNBI6Wp+u^GlY)_5*I`*+N8Ief&uR5y=C&}0%lEjk-r)EiZb&bvzHnE*ej&|)P}4M zy@R^SC1AL_gkGMsA=}~CnRd`x&iS#D%9lN1zNQxZJ8gxZFc^zWRIFt~6q)~01|cP1 zSP;(#D|+RgEB)L{vb@E4-}~3HAtv#d>5>Ii!6!hq^(nKWpIF0&YGG$)B)az-$$ad7 zhqoQMhH<2Vd!rk8xA$nOx-_5fKRUAJC0*FIMQwPl?gHp_SEbyssi@yCOSU`ufjA`) zL-z5vx?w)=$SaWw`ul>S-4}LZ*%}H>1qjs@f$ry#P{iLMP;#1uH+SOM(Ee!N;zB3P z=TmB|HTo8v<8!_>C|c%Xk?ldScW}l>e8zZG+!f1=p_n|j1G&B#hq(invGiftpy|!$ z@4N4r#->?lspnpW{%bH>;QYoiOK^cI(D)W{AN(F@uzLt?M-9Y$eJyx>T!Ci(c32r| z4e?fQgxZg@LHFAnsm#KOG@pk;Nd6J<3!F*8`^`zbn1GJ`lQFmNLUtIYQSCnh7V~}d z-MTf@brXBp5=a)715mH#`=paQORV0FC*!BjJRixmfmJ(8Vva7s z`rg)AWw+ z);FHzju2TzNCIuP=tCylJ!y!X~<^k@Ul?UtbC&>b*_^+j#} zv215+A{I~Wk4D=f$oBupnBRJK%q9@sKRy=D^y!95S;;J(|{kbbOZ}P^xvjfmDcq*kliNfYhT|t%1b9eqfG5=35KtFLW z6oh|ak)BHQ8OwVIe)^vj*f9ddLRW;o2bs?g$3gYb6V12oh4h}>bG#{msvHk6$EV9F zH*_&`n*Ww*7bii^Z=A>bE7!+T-Y}l&hv*f0NaB0~hiGr~dbFCKp*#g_ohp@PMoFz} zEl_piq-47!7b>oCKZA)Y8OHr4DKi2gdkEiUMtujPNiRt0z`5VOUcxEsp;To2RZzX- z8O<4=Sp)AU(9PPRcBoTR=<25++BsqJ;UboPVHG-_4MWy-2mgNP1h>OW(IwfB>7u5p zJ-N3je(yQZx&5II=eb7hE;Y5?wVaALGs*UGIW)&CBV%|6s_Rh=>W(8Z=>m6dZfjz4 znmx^^v(4C%Y+F;>~U0}@V3#R4E z&{na8Oi#N)_+ubjo|`fe_)NW*_pAASWya>k%-F{s45`<6UhM_@G(QRhZro*lytB}d z*quG~_N2g)e$+g}8VfGAfrFa9r2{|qJF*VF2{$JLxc9?a6TKTxZR>%FUvkMy!yilJpZ z_ctw=b7grR)2kQOa?f1dhTcq(`73BkTBMqBTC8k3BjKPs)E~2F$6@X{HA^`) znkq{5PBz#&?~;N zO!qHPO{4Ur#dm0jm#UMQ z3*F$}aNYeC=<{VCIJ((UanBs^Ig?1LYsaBDsxQ^9Ue63AVd!vrIF;UP5cHe+vAMOu zn00~g8K&(4lV5^3(`_Tn{cQr7~oK zpN0FnQT6mkOuPTFaa*36DJY`;gJ77(-9~R%!P+2x-EEQj5 z`dRVpU7Khsd~am-=~p3nqncbM3?{ughTUGld#s#qGW~>)ke-=Dxl@{$;eM!)GFrfE zi?%|N+ic1U7))84{)Na7-O+!aD|qi;jE%`k{I+)@x$h2OhTzVaH+MI)ZO3`;FSx!l zR7ocP?S-dr#$eHgCrs6P8`$wj(3WQ&$1GlmpYk-g=W9Rm+qZ(>F;T2sxtApmvu9;F zO3FX@3*_abVnXyF^0r-xx}6EUht*4P;+^!mzw^K;xhqA!n#ntK5^zWt0bl-LkA?yB zxVQ11#7?`DTZ#wfdOn2vv#l}j*%+!l@=R(STf&qz7nn_W5=BqpSujm^e6BjooQ@@8 z>NTDt%eV}Q^Z#Wk(XAs$YaxLzD2dZ$jN2hBG@Z!SRn18V=&b{hNmQy1rW=u~E zNw+3$3}e}RK1l1{2g^5ZVfyZtp!??mQ}xUcnrqrK+v*FT>Ru-l+MMG4o%L$3OH(oO zE9YuFdc-O$@3R2@9y3@kXUgAS2^!wBlKHhCmZtI@NVzE7?;nB%58IQ%ZW{Nuts-0H zJ1F?Hl?5DaM@1vDS;Md=lH1I#)NrRn%JQh<=h-RACQU@E>sA;#XcbGh_CnEgBIO?K ziZNBA_#C;0axF$s@`x#v_3s**Th$dq_WlFTtsP0_%=JpE`;0w`!m@v7q1|97>JT!W zR4Zo)Gd6}`_#iLN?c?9`9-pD2J=anLLecX?SI!A@Mz0;)!OO7^hD>`5F7LR%c4QZ5 zm=TQeiQ||u?gQJ;KkG5GMq|;^O-xZ}WJP)}rm6iwf|t>hb<2#L4-O|?y;Pzw`GvLi z>VtLG{Pzr1G53ozFztgK_kV{|$fvUqy=@N+F|)_;UCoeubTJDV_9N#EHNwkpaimlH zEw#+z+3`!SS)Iuu7DUTfcns%tbn@n$!Lw4t5>HBh_dBb2zmB4~7b@-SL{xViLCO+j z@bgkMI1E7DafRePmHWUqGAM}iptNcHebjFZmK9o%&a^vAw*HYb?W3q@bROh(KE&)U z@%^gBDk$y2&&#Y~Qgqc92>oX^wce@++W_8wTqUa&_Ibu;^J>Ve4W(@@Yq0!&12{aJ zNQpnM!gx0mjIvuuswR;vd-DIe^iG(YHlMRBw?Vwz86w}=q0(zHD5|fYD_S1QeIk6{ ze9nUf)+@>bpf9KiD(59`2ZY#%BxO9jQZI4B;YrbjuQvFQ62 zVMh5H)DNG}RFAExbakWzncJb^KUs>87z{xNM3!)e|F5ce24%A&=a!ui;yTX2ghVfD zING252F#-TPwPRsYbzK!xJatykENhHi^w*4B*upkG#*=uvAcRu3Fr0X@lK8S@53>K zbMS1_{s!3K!Mzx2$+U&{VM_U+8~sh~{2~^W+efqNx~W+8M^DNh{{>Rhd!x~MHEQ;D z1Cv+BnC-7qsDpkvWpQSz(+}R5&~`ZGpBX`7=}f$6VTQHiqM`KYM$jMPyss~7p@?TN zb^ag0-_V8?{gjBNttyI@KS4>;SoHPsrHVg}Ks&|V9cO6|Bl)RW(% zEv+S`rMD#aFJ+RMlM_0gTfsG}j&N+^1oVCO56f2bw|VT?E zec*0S&KP;TLullxpJv%h{=S}#L04}`)e+o>Va9)_RM(Q)YvV|1KLT|_%Oz9GVf?+u zy_RELS?f#AnYsQSlsg<_9j>_X9a;=a_ZUkG6Dm-4X(h=WUPJ8%6J}*Qjq9XE&=TxTVVxqd z_`?V^ED*3YsF7WZ(1T(`{W+`N4J_??5@*OS#Vhd+nA^P*^U=prbKM+NjQ(3)dCtIc zk1PR)<(=_quU?p~DQ1zwjgT(A=lvXq1kI0`LfJ>&x#4mQ?mkZ8T7Ngz8e7V9w8I6p z%UEm}!F|6ca)o33?VP!FFjU#PlXm(+Na$pSYq{Q&15sGJ&4zqea$UHW0^iAD)ZB0e zd>(tTtTVP0ejo|+8h>Mn+rue&don5de`Uq?w^-tn-WXb&0JY3au>BlLig|%TX=)s1 zzKp{#i=O1a^Cs9n8^d+B<;=KgC3zKRLvDUY*d~Thvfl%iR5SwhU9Yl7oTE}%Q6mKw z2;_w0v7z(`^Sii=?uJaET-y?;v(ey;W9DSPl6NzyW56Z+1#9+jV#z+K#2&n0l?5xQ zQ8^i#tQ@#LJrrOg=NsA&#JYi-Aj=|#{JGDkZST2MRi(lMF9xE{B(7x4Z=9=gcOfXW zCrb7QjYpMtAUp8If@U1oQ0(G;P}%0O@RWZp+w0 z9zsR0{Momt{@hn}R-Jgsof<+4K{a76>F(?n+>eRS^xKY$1TN#<|=RnlX3ui%(Vp%L)`v~eE z)tue@l$DNnT~g|fl)O8P**7-8{!_tJ5&NB)-`NlPs`e0M)&+9wMOIU11k-LUOw0Yz z=|5WY9{abDUCf!&)}5tKdJUd^KN#KYY|-zgKNf6=$HL{7qz#Kfw^NHrlXe<>vL#Te zT(HS-p4IQ;OujS)jhO>f5FO1j7yrssNoG=>^%U|sGn3UG_(N(c&R{yLA&?sS22|CY zmC&n*)g`2|Sf~ZJ6+N-qT|npQYCLsy8Y#yw!b;m3!7|E~-a2=}g6ZSXzncY_c5wsa z_&QKMGo|F@DNxp(`w#mkQgirYwsvzLN|p5B?>7Jf)Yhn5*en#@98Sf1uCuD64NwvM z8XAmDsGeX?t;0UEpvztqbY7Nf&+>l3f0u*y#|+^_VJOCj7Dai%~$_h!De$H>Ng z%yiE(R&et zR_xmj752fTZMn&ei=3IyJ(&gNzm%?}{>`Ej*Mf_aC(mMc!o!{mFlJ{Ex}I1@zHjGI zx_TP=d>+WVYCTxc^BSf)&b!|VmV=FHe=0av02Qjv%-^#Y%RKs#70y$mlS>HZWCc*Q z`wY}CsgzW5Ap2&FL<^Wl6}S4aBGcjQSO*)f8T7-qk!HlI3Yp)(KBPEtdSt>c9qHR$ zXRJHe2eLL_0mFU==5n(u`hU~2qESASx;p@4IrAynq8mJVKA#%iwJ^uu+hJq@=M1(D zq_u&)(4;TlF|FX4`a5$V|8PeN_MVR=11w0h=d9o|6`{f4!(5Lqq}uk&nX2z5>E)yd za+1EXxh;#);rnRHyuO(kUycRy9ao{~w@Ku6@NbCB*vhQ9hT*!1dos6#s!fadn@2l@Obm;$T2~2mA12ea z4wm|Hew+n}E~6NNj!Qhu>cNl5XlXv9c=w&9T3lgb*chth%)hL=Yp6)= z4xtL}xp4W76+K->iqh1PUcU}P`(Zt?%)*ok|H}YR^C=X_H3?%uA-MaU1jCC(fkiFwxsh~2x&%Mx=RKEL}Y5iz=d%_OghVV|h!C|1O(?V#%XO_xY8+Dxou2~G? zI_~-s+rFI5aC^T{&EKWfocEkv+QPgx+Tp29gD`zmJv05b6x@15p-%n)el;twAaXs^ zkLEqVE7YVZKCTX%1r%$S2L7YE@Z4M#D7vncu1OD|I@ugle2rx{`6r z95N4a!7+xB=wggzA)#?-QZJI%rrsFzc|3jF?1=Z9f+=a3IqC5a!A9gPkEr%6l+WFn z?KeZsSX)XNWXrj8))W+xBgMDR=Y5Htm@16-Wp0dMPWR42Rmley^gjm0mJ=v3L?n&b zffB>)c+`FFCw*&OO63_TWHCGt71ORj>z6XtVT>BJvt~&}mor$yM9%#VyIE58inB5j zDxi3JIRsehS^DW^TyHukRYsU&P)rLPF7Jg_Nn7DsmKyVR|H|qL-GSwOW2IMnllvY6 zYus-_jR(}|{`WAZ5C1MTEjS8tXAv7BKC;?Qe@R1n+n^%(g%3Nq0<~Map~I(Xc-ARW z%o8{R!Jrh~J?5h#Zxa^GF&J?3&Q#FYuX<s z*zyqFo=E7`oG3R0>G9>MDB1CTFS#joHJLBIicMOS+T4;iw`i9pEfNLGmdD* z5c4$3Iy+0WFAR`FpYncP=LZ5vo&1Q11WAP%|kMbW=>F$Wx~=1wCc= z-~Z#Bf=X!E_!AyGzCbMA6)4{im?QhOgo(8S`9A+l7HIko6yen;3L45YQD6Aad+r2# z4^eb+_LQH`JB~&3N8s5%f<*=QW2+{N1#i|?be!2muFP02>b-;TTmKPaqp7cKVt$e0 zf9@yiqBl^+&iSJ8-Y*DiI?I(aQbpwj1K+Q7Vfn3xsPxuUPz_s!R&S?^m7%%dQ=`DT z%-&+$?apH9<*uMVnJn5Sxr)X8qr}1v!J_ZjrDFDYCo!7)*uw*UK{+H*J}}i%jydcg zs`ySh&`lJ5dU|86ZUi{I(2D+td_~QS2aL}^VotQX{H@4Sw!G~jDw0h`de#NYe&z$k zxee-`|NYgEBjT#^q7-wVjhUIv%a+%D^GRU*jRIzbV zFS*ivig?9fA~!ji%ZVq3icj_36+Sgph;`v9pSKW%< z&b@_pk5k0wFW1UZ4@^XVx64%PxLVBZvQ|`lug0{G4r1bOg=lpyTK50BlH7xGA@S${ zk$XC2!}!IrA>SF+E?0>D9|loP$}mxPb0>W2-&PE3Yb|mYGrCWAp^DCyqQyW#E{@5< zLO-=w;Wr)wuC5cE(oRuJZmowjZqjq*z zQ4w!*F0?#OEI0I&8x}4m#lY=Snb=)cUUHNb3yPS z9yi_WTKW#j3!a#Gwx*^>Pr zYrV#a_M3h|?dB(R`^ie#ysW)gHgAczHgB5Tye$}$7AzAT`@75Xg`u*cwhb6p24MK; zFXUi1L=K7jNa6dgQbKr)Xf@~yPOMkSX#=~<`sN8@m7Sj)xM8}uEyza_z6Sj%XH#lcae?x3BNrYFDuGz#N?>?l(ut< zT$7$EE8FiQ^R^zM{&qD+^)Qi@L1$Q1Vz?OE&PTS|R)fw5EkyT-1TjOemFv=;P?=AX zoVjj4mL|;QZ{~P(AKpf`is~t^-8@k?@w1al_IHxC1Dm0A4rdybdWlxk*VE=Gmla8Zi@@djt4MXp4{$dBK1@e$#ZN+kvf9P%G zV=7uOQ7)}kuo)jTu) zY!fEWkCxNU4ixo+e}>98f!GkT3X4{+mTi=MXIP_Yf6Id625=KT>7+ zZj8RalY9ntpy;f0G}$y$%ro6h%w8j#x}~GCYgf5Ic?}Jvds$ZB_2g(0A?wPm;FOD> zSon{(sOWpXM7M4vd$g*j2+m<*^|-#WV!n|S=T}iVJAhf;*NT?A+R8&BCWx(yfpW0P z4{~mt8*aNWQdXQrsbEk8?UB06#U3&l{H>X)I-c{Q+(j5XO3pbmP|o}1JSC;hl53tE zCEL3(6uz^Y>^gRkTr@9H4F9L6sLlHi0w%m8pFZ)lXXIM(zV%qSXyIBpdgB!;*{hHh zFIuGhSu5mJO&%8P@{zTzb`VrEgXXdjIdW7k>YRIu@$1qle@b65dBg@RtSzFd8#82! z5A#G`2qwnNA4NsO{t@n9?~b{@^6tZ##ez63fpbP}*;>vlHMUE}^ur%m$aRqnw>~ke zJB5&Rp&J?LIpj4u5xnY2HB~ZuxsiJe+V+-ohXYHJpHAYOw`lZS#QVtT45)tZPkUB7Qp|)UB=(P@hCUjm-Tt$r39DC|CZ(~4i*0D_AARxmp$=%U9S+q? zN3q*q;&=}16nLE#DB$ruC>MCIk1mr{2YONYW8TSNF-gizTng1|^q{)XO>j@WA}M3< zF#jAk&`-I{qTQ4@WFptux`$9?1=nbnOhW%aZ!p>102*;1=;EgH{OvQz>iBF_tX!dX zcU#Y%bD!u=<7loya9>YEpW52M{MB$gmJoxfJy%oQx&i1@kp>AXox`e7PK8DzrJ0P>eX{LInq_uwy$JFFNjOa9( zCiV@aI*Y3iW5V_NV~m++x5b8IMk)PH8wy=k29Z3Am3iKexz-OsvFjo%y^rLRvJQe> zlIhytu9z^7`%O3JgZsfV%rJ7eU}y?Lr&%7nPvlP!|Kq*kRce8?hGJ#LQQ>ZS5E|77&Z9GUbV3}mMN<%IUobl0g6g0{D64O6_1;d)2s9X9*sye@g zJ-_*dMO+w)<*p~8?d-v*)4K}+pN5g;i!pfO-C_)viK%<7BSrV?LXq+dxQ<$eUVHhT zeor>Uto9~lcC`@ninHqHABR}+9?R{$6?Bi*t7|X&37;Bglh@E-vVVUYWcx)_w5mxe zJ^2TV=Cf(d&Dj*B>d&J4_(5D{JnmV*y+WKpqwugTaaq)kDKdhD@)5V8Hro@T6T(Qx zMlt3M&NqrZ{v3?CYpWuBs_f9gGw{=Y2HUSN_PJ+KM7OOdjvATX1 z_g4i{##{$#85d5OCvC}lLnPMP1+(0YkC0dW29ms2V*Pr4K3dA^_|Z3{+L6{Q=E-7o zi@6S2zpo)Ds|~E$><;s~IUK!@E+Y5O#X|VmzNDOA#7cdq!`fBJXfb6B20x0V=4Cuf zZZaQO(0{CcvNdi#-52X+3yl8J4!!RE#;RE)X0P_5%#@+bqN{=me&-qC)Pbb?!vgG6 zms9zi-R#rk@!0A$7)^&ZvdWYJqzX(#!_!QmzHtrb1kR>wA5ODOv(aGqS6~{CEOq>a z`JgHMQ=L3z7H0-*W6ln)n0oLet2<9>IKfC45e0g&KUIN*_}m8 zz&NWb1YH=3`Qx~bIWwCX2Ou@LmP^6>oGQ@#4o^MTqM=hT(~LL`)s8Og(W{wU!*Ig} zmB{jDe}N%&vr+Tsm6D+0Uzq<;Bd8Ack@DNi;Q#txRybh=ImQRzw&}}xw){9N{q1H+ z^xHmUVKR)|t5m!P+m6O;?n@PiAA;lJDU?G!$n0)k)J-aqI&4fqg~C~izmqGpnsQH$ z|H=}d;oT_ZVrM$rcNI1tl3C4Y2Fbji*6N4TEPaCmUYi}pnX_-;n_~!820jxs9$UbC z8h>ZM1_T z>jI7#JH#E+-$hfdX&$JCM+%ys93b2DNu?AgVa;A&*VJK~I z!;7HDWFF-D&Gs6hd0{`$kqap$BMU9!e9?Of^jWfy)v9;#-Q8PZ?XRxr#rKq_ zUZ+yb^W}8d+n==SQ=lQJiZ$O@$R2U8-rDD8)bL<~)U>4u3cE&<`S2vl?iNbTdmY)M z+*PRmFPHiHl!Hm)W++^*XK^Q%VtHO?Y`yUg)Fb?dIvn`&k{V^s0~7@YIlQ$Y@UlwYbMiX zzj%7d`EHFdz!$z*!@0=s*McfJM4g3X#(3N&%Rpd~uV(c=fyzkkgX!T^@svk`I+ z@eYq3J|&48mZ9Q9&B%BB_s$O6%_>&+2H#7)P$?KC+q;GA_IwRiuYbfW9}dF2d(R=L z<0l9&(?NW^4I27XFrU;FtZmaWEVXYEF6x$&W9AadyURU@3pod6oejkQk&3tTtf|&5 zlHPTgfH(FbrSu6VR`{9uw7JNF0?VbGe>!oGzZn@bc;8(;@3veMCp7%*&$ZGBD%y9D z1&zs-bT0+Tq&|dT)hOEkKU=a%bE3$$Mu=+Q91TZvvNa9H5alk2{_`prBHl{M4Odv& z=b4nTK9NfI^+bF9CopY$j+y`OJSXj_;&me1#JJLQ~n_Sz}r7Wfq&#ScW=eH*~` z^E60W#qYoSt05qt&$7S#3Gp9OA@b?3uya})&c*eI6BYt0+CMlK@4O7d_W?c~l}rU) zyfEB&ou$Kay20m`qO+&qdCMeB+%puzhbpMZ@EfH3$TJ#CV#sm%1hkpghN?~PvEu4* zKA%01QY|@yW6efpa=R5`=U(D{A&X%D!Re&&xC^;HKSPHJ-LW|=jO%_Ip-3?R*RJO| z?d@BH8{bA?z|_Tb>eFz}JztDfnUQ3Wm(q-z3n_W59`wZr1dU-JHHR%^@&C;t&7dym zI>wpuZa!j$zkd}nt3NZ}J11G{#~EDPu|P#^=aHF5qR|jn#oSd5f>|-&<8y7={mmCC zb6GGLezvB<2YGDBCH~n67)YD9n4@BpKTEtZ0V{S`0sEEv)5OR-VggqQcK0nlf@8kneIUtEnytJoe?nP5Z-+^Y+V_>Yam!EYt!i5wNQLrm$^=fV8l?Tm^H8}76iLwQFR;QhY_px>dCnQPSX9e*4WZ%FxLKJA_XKYBh3*>a5vo|>9);ak1otYRqsM!E$=c(&zM8O z2WDXPpFguq=RU0Ya|GD(4ZhQJ?iU#M7fXN8jUI`XSbFP=q-cKvw8|Zz!KEy+Dw~dT1UT!ZmRIo?G=Ey!u8{#i106KNAg(Rk7sb^qHy7J&@|=+GFi5p$@+0RU{=-x*uF<>>>rkFy@wvlAm)5_}HgCJt{B2+d0DOC7% zz_P;w_<2*qEXue?*}V^3Yjwe#R0q=h?!^r1D5`YHg_xmsyjR(d9SG`&vG>av>w6A@ z^6jC1wSd(rO4#{s2I(vIvsx)ss+>9>%T@C@&wD@9Cd~t5RT5`0hGNV-ex7qSUHGa zbv&Qe=MK0}m<2BDV_8*OZ!8?7LW}#8DI`A@TakATC+eh%=q=!4;==nvj7+s*6egFu zF-`Zu7`Gz{T}Ota?KV4h`^`YoRGulR+c}o1yU&9g`J6W~!h!DEd5|tJrzFTDO-KzI zL6vkB9PTEfYIavvsoMxSG>r@kT_I@JPG--&+s(Tcaem}i(5#xq>W-wa1Mgk&dCgN6 z*@1!c(RHY}cSJp-F%j)@%`hsgC)Hiy9LwWN>GsSv^!D6F7N)jDn4^cHCF7u}p%(7j zOvGmFOseQrsBwBJc;EM-$`p6dRJ;&0E0j{i>cY%J9o|#VqfQg;ajqx9QTsh*8Y!Y z`aiI`cdNjv5BJP%3?yY$fusrwr$p8hpO^j4^7?*f#dAff|2z`QKZT+0!eVvNpe>-Y z84tmKx1%sVcg0@*iN!4PBh923&;&nL57D>dEZQxC&u9hdmfn;0L~+LPk%@E)xVL-A zGNvv6%6#TMXa1RUDBKIl@n;`;+y5LmU+FK10u+H^RI}Tj zY&B=WYU)smc3(!<-tJ*u-?zfWf9GIwr%p6cXGXahkD>TW1#8^lkF{w!D9D)!bun*1 zYx;|%y%YqMX%7X(sKRrNtP^QpR7fSYQCK-~7YiI{&v~O`(0^P$%Zl9(ZhmdBd5{HE zyf21`r@S+9z8NXnoR<8YMN}P+;(eBVIRD`?=wFTk&0kXm#h&@(-m5=pTmj87pBrLa}rpO&oZMvPgZ=A^Wvwx-BmK{*~;Ja|``x#c6S6`w( z+?lz2<@Zuo1H3yrgu=$oMDta5*h_mIQrK!vA|as)n_tywxMgMOL7hy@GyZ3_{a>!U`+NtO`nqGWunuFI;yB-K7S{f63VhP|^ZD`$cu(fM!O*4r z^MX8w=fv|ct3laq4`fEZ0n05)G>qOOsFws_P~g8(>Xs!0>+*oqa%z+55%^Ns^3&BpFGHWDwGG?qei`kc1=|36&%yNou@o z(l>+<8%9WLlaVCNxsO3h5*tF^Lz1*MwzWz4o!_7S$uvFZIrqzTeXim+lR;=o34yk80 zlahyZB=pT;7PYpDm_#!&>g^Obabp0+CmkhxOHZ&I^@_CqZVYm5ij>*wwBX)8lT;Hc zSkQqusJ`0=?u;-8*BaqfY#aq;o^7FO!#q;4h%@!;6y|tz34DIA9D|mPg|eA_!NM856`Vp#;Lw zulHy6@sI9M<=#%*^WiXDC%A2|DoIusOGq1)L#)S*!K{U|v2sFJ^trv7$YhqV)5sWq zY+HclF6|*ibwF@$gJ9x|F`zqsQ*Ux#ftGW3k(#a|lk37~_rQPO6ZtX~gg)HE0<}U* zXuT9d!c~|nxJ5}%`#|6?g)H^uNVJ)@ljwHUOY-5qtTL$=nhmtXc#SbMp1I3nMZRXd zdLrmzgnn#gIJtU%EhLR^FStB&K)17k4eDWxN9J!tQyhqn!@QVHT?(m~9gQd8CP_~6 zV!GZzr21%}q%{b?#oWbuZ8tmp>GuwxJUE*~ddz{=;fqmj_m5uD(4x1$@`{u+grNPq z3g#zr?KSI!22k2V98_x|)pjwCGL->YdV*EX+eR{+_Uo%}wv}vN%>h}ApWZ948_LdpALCym-Y<0rnP$6-KHcFz z_9Au;YNnX#qg^BjJR!7XS!=L5HH@f>f-rXFf6VO3cH-?)%hY>on0I^?OPkdJ0_;sd z#%-iJCvmQmt(lTs78+_LE5CJ|`0q6$iX-CpwF#GEf3IeB(U*x=C&Fy%MSgSNWOqfF zg_x}GA}wbeu~D;@9deognZ2h$`C5@Rqp%Q}@WaI3aW>QiYsiszy+J#xo1|)8MjY$= z5i6hPY)W6T|Jij#?=wNgw0Y&kPvl)|8%!Y0db5-kGY4YArlLnD1uAElu)IebfR_j@ z=J<&i`28sJcG%7u#%^M4h!M7aJt6ML2;deJQpukr;`diV7h*2id=dPgqU|hRv5hD` z9oEZqEhM&E7I7PTiiFlqz>j;{LgvoCaJ_XoD0R^o8P^_p&Rhs^6ecvu~RRI=V*h_PL73|lyhYGA0seC^@Y^0nIU?U$yiC!S@x*}xawdACV|7y z&yfh-Q-5sww~1KRE`aLt?Sd~n3M*s9yq<0VBikmX8NNj_X*5QaBD=VHhG4CHyGFEM z$ADLD5(&s3f~_An6YWuFNx9_*3A8>gl6e6b6CiTBc9 z^GLtQSa=%Fo?$ho zrbBv9@ow=hCrr~qPD~Ws!5xb*_Uv5Za%;AjB`=h6|CtI$n&(1A)e$L0g!$Q~O5a$YCD0K-XHT&eCoNB?QUz!J+LwkvO%{$hNT~N9CvEH@1lvs5f4flFS zioAke=r${jwFupro8l3xcyp87(=G(fpd68_{x6A&y`c};S z?@Ukd?=)X%2=5cE>oC${I}9^z28gV#Nac{e$?REjzMnp3q^r zg5$5}iE`*Cy)6Ghq0cGn09$u`WC=r!V9)yy zbn%*vFT59E-j1oLi}3>U+70NID#x5ILMQjs$z68$r`7fr9~& z-#!^k1;651$267}d|4_I*+*Aj2Lj9Y#3R$hS#w^5Eo2aUNfGbd$qHnfONrUo8{}TQ zHL$a~D=G{RrHaWaQnp0sLlZx;((9*~&sVM9(6AIA_FV{uY1PFkd;q5TJ!7T0lZ9t( z4Kvu;N%t15h7+4b_T@_x!Dn)0rCS6$OUz&uBfZcNB{)D&hNEkrOT_QGfi?9KIa(z> z(QwdMule{xU(($fW#?9s)8|65dTO+!@@i%AZH0eT*ID>L-swYhFNpTqZIZRz1x)J% z)0Fo?g-IA*HMfBDVd0QhIRtu?IDtdhd}vzd1hP$&ie-OtvUA4@!Aid19%}K1g?3*i z?$&pDS2HV&j_wHmzhBB}ibXRq2T`q5Nk$f7!cP$hcm5W2-}{N;d_E#iF1kRG`gax> zG6V`eCt=m}>BM8ExT~!zSa|tNCm=rYi|dO0ZOlkj>w1W87loF?7Q#JEXVmN) zCY6qLVG#qngO+zA+PIEPbGxEA$8o9f=_X*>?fv>jQDbPbl8Xb{4HvnLL*4Df*=_tG z*pi2pBI{+3yQ`>FPUGG)> znHBXDp57S2V(*>6!Z%C?S-wtRB)rc)6N7Qij-}||X)sX@S;LYO`l8aUN$;Q6hxq@q zgxLp(jEB-@q7G=Mul`f;;J1EYW)qW0sLqm|@E7+?kC{-A*$Ykj2C#-8du$Z*#OB3b z5Y@g#cn8I7FY}b%JiHUwcTo#n#b5eBbD_)cH6Ltyw8Ppf3V7Yk0u?))ptN&aqOIyd z)UCnNa4{2)I`&fkq>~q_3fr@1{pWzksqUcqdPwiT)R*P!4;Jsb{|$$P=?gGGQF1nMD47HWtV_3@MH}(GfI#1&g@1;8o-ZhU2@oSQxwr(mjia z+kx{WZnQ0!EgX!NcaYd=!$F%fP0#8#liW=wShdJheD-n_`m6pDys1Ved*S8o*1QT; z?IZP$OMejguHk6%R}Z4Y37By;gef$m$Q;3#*829<8-6}>udWY~Wc^)>tD2)pS+uCv zzg#CD7f4`o_mY@1+v)53js-8fLqxm73UjThN%eXs$x`+^Q`UIHp?1b%|9@KK^j>Gz zFFAqr7)vw`@&UI>Qw&aC0ENqEih5-seDM|ix%){;vo5SSP6`6Y7roe>yJnDZat0V1 zwL@1AZ&b$qAfr03f$GmHrf?87h1F*=r*RUxuJM5E6EdiHdtC1tu?YByB^d9T!(>&L z-ApoLNZPM$z)1Ae_M2amx?WC9v+W;uxAGe-$j}?hKcq5c-2=U4>riAfe!@Vi+M{kBe(G5RVuxT-V9$^ZrRU5IemlbZ!=m2&u%duHxNu;)a$h>C6pf+d} z%a8zbW{cTYyLI4s#tG|ujxe}D8XR`wPH7L6}n$@(MC~{nCNZY=h!QgbYIMiVYJ{zp7BK+aQBv3H3C$L`KV8`E7c`>n*?{#y=He_x1R%UmFEuwecrkbS0A<%nGD~h{aXdUdRjMZ%1C2{3Tu&P{c=pi?l0__ zwv<^qh|ew`gXNE-z({boN)tU<<1#CV`e8%1dh8;~Z3kJ}&<}c@x=~-aDh!@19S4nF z7c)(A7WtAe4^(~ZG4%Eurs%qW)XiCpy6J^_tFTW*97XZ(*uIbN0=x36Z}_%2`>FSJ&k-ryL%j0FBQ5L&MvBkWOcaP>aQ z{Q3^W)QDRwR<)SPhrHKUJAKoaKAVD7p1+ZBE_^$pKUB2aF2$$4W~HVUdown&|H|+jKi8KVUWDhMzEJx zK)}tlV42q)S|9bpg#Py6kUbSb^}AU7!8B5|Qp1rw8DH;D5$Gknq_nA-DxXPS;F z%x}0o`aJJ|Y5!f*H`fYA*Znr+#b&V=y;Gz=y&_iRXRVO#S&3)=(rB#S)K}zZyFk*v z9+e|CW2sgU-YzwKGiVy9TXjJ1E<<4(-Dl zn0;*;bD3rfntmsl?)_gAp56 zF>T5;^qS*} z-t-}n&-{Y~T-b;<_xG`&+)#*KFT6l~DoD-qH{^)Tc!=tntCy>1Gg-x3;yGxW@Jo!) zTm1w$z1$NjIwUil^9XWi)JlkkFwoUI#Bv0*&~R=s6re{Nz*`>m4a#bual@k-3n{m`F} zAq@u`#SBYNMqQhSHcd&adW|t@SuBE_OujQsg+D22ZY%t+#|4}B1uKhZ1Ua8Ue!(fY6E|bZ`GB41MJtNt57y26V(std)>G8}eq9hENfYx)!2@oSQD@Fb`SRHE^QL<~7G6qW0Zn9Nu38D@0CrmoT8 z6Y)w)s}&hZwOTeu%r~-J)?8#U`Nb1icImXq&{n~(8=kF_gjQOQ&BXs!tJEVdOK4sNEG|f zOM35R%}j3njubk^z~)ER81vH(Ytm0Mh2=I$c6#w8^PNJ!lw-o&d@KYXF9c0O53pS0 zcg%X=Vo08_8Ver3W%kSpqqYwv1{*(U7N6JanvR%?oW=cHOtAI)0CaS>A(a`^arKxu zuur>4ocA6jD%+_{wtX;*)h%KPA4FfbbRnpc+JN1Qb!h2T%Bueq_iCmeNuTk8`Sza& zjzYKB^tp!Fd^^rU0z_Wq%wbaU?E7qgrv;diJ(4xqC$XHpD_G1*H!RDtgMcw3Ff%KS zIA`u7KCU%lr!fGYoLGu3UD|?bWKnVBhf~7y^n_FdtYgl>+llgl6($Fmksqn6(Is{s za97c@{JmTMG1m<|v%avZF`bx|+#T4f5+bk7mmJ+5v7P$WcyEZm;Nn~$)pkk}W!DDo zwReOv-vn5y83;Caf3U{)hXnghfq{%DPP_RH02G<$Z8M__aaD4e<2zB=L0)+oCMxh!o87wA+qy(cROkJfqR?DlU(gja)_)LLT5+*%ZH|=I z;su#=Q;6AZF~d3YLH}{z6bQMUfa>QFl5=8;(BJKlo;WNA!=B&Vb3Ol=h$OBX&ez$yTs~GP`*~oBow})ea$sJtJXGtLVM^3=q486D)MkT(~21 z5<0Zgtok=CVV8w&py(hQwO}!*X3JP>{yUb`wkKA5CSZiuB=r9!gQ$;fl#;v8BR+Fa z>76$$fISapgHL)FW-S*xiZUCB4tOk>CzGY>oM-y0LaVGZJ4@7gKGM~<9yoD+2T=Uk z4;n>gQQACPCigQY@#o8!@q!`Hy6^(2D_MYj* z(%f?>Q??04&%tk*k=t!%Sn|0zwX7GEwp+)lUb#U0xhq8V(@I||7)T~&E+i`FvwqN% zaS-bqfEh6tnD>jJ7`tXYdDt!lRBgkEChWX>Q~y+Ab4&1y_YMHxb)%tf-(@nT{SdIe zE#?q62Y}AHMPFSxhP)m#2yL%RsJq}oEVCBXMtMq#0tuD znPIU4+Di2oce^0GrA{l6`&{P|&4XCj{00XIqWJ~NTk>j3iJf(!g|9ogS|J~kXyf>Cxy z@GEtPq_c7e=~cu0yF6!U;(XgA{lfBEg$C;8EwZOD09C6!^xC!tee2%Wj2i@Zy`hDD ze!d0+OE$5P(LU(k?m1I`8^9_n#AojBi6&FCnez1}z2SI2l2Gdj=`l)pGSM2I?4Jb< zC5MS+^iw7qwW&DD@K~RIDG*JZRm5ZcDls>lBY1}I$()8skf#4t@K*e=qRT(}%zxHH zQtC|f>Apo@oN6ibrR&^FZvSLIT0Jq^^(N7@aF%rj zz7a6*BzB2Ib?ye8&Y{FeN;r_&>!UTi~1;!ZTifZ|42r&s6Q4@ z5HO0Pq{Md(=8ZIlpy>8kaQCZVA6ucZ;E6;o9SSK?iICOkTGK z*QEoXR-84Hm^EFosKWDyQ7lHg-FOqg6P2V~$0?WcjV3DSfC~b0CQOhmjyISzu z{_X-k(q+BL+$b_=MjNqfPcE)k4#tYpEhPHbLnbpl=-%Si6U(peCJOgXk~&<$ydKw+ ztwWtbo4c1?pDT2;X{IFSa2YYBgTcGsA1pd=2(&l~kk7qEc*koJuqKiFd&?la_X|;V z{1%HA?Ckuuq2SRz2BJ=1k#ZcHm|>TSrC4ZA(wWMnGBceL41jFby;ImNyy55bX$SoR9m)WDH%TWEG6GDrVGLYpS z`->^f$Fj^hbFk%)STs0$6R*r$B&B>k-osLua$q}%GWkY|#*2OJ+2icQHV?Qi z-rtQ6RV4Jgf~95sO1=mokz4jP(zwr-82m*&A0dS5N5uI&ev{<-?;yOu12tja*pp9d zaclEm%rI;XG5qhWd%@Kn*mis%8oF0W)kD89|BRu`Zp4y^Y)(T`pRoLYIhLwCzV;==4(dOAB z;_+1lCD|8yqV|vz_LFI1{$W~?6_C;3LyEQ-L;906=D~Yo)B7y)FmXPn zPrS+E2FjtjY(9GT*~8R!lLSv-7xUgM_7s^q!R}bZeBKr4?W67!-Hud}CVcU++X7hT z%5%(rh#Aq9wqb$S<>(xph{4SXP||e|sk(THnJ-xcj>nd=+%CbE2ovbHlizPD%c z|3e47F>AT z!xI*VM8;(_=A6qRGSfsbk>;R$cMn!-sm9QnIHD@*qt}F$OO6p^nWb%u@IeeETMI`* zvDsKK{Az*9J{i)N1W%~@yo;!|td~L#{Jdt&D$hwOz)7?tOT zUp&X-l&aYn)2E&A`n!=T^NvE7uv>qqJOo1;(#h)F5qLU5=qm2nLgm?on6z4G$>T4& z4SFMH-@p7_Y&bZBH3!BL+4B%-%KSjAjh+Sjwf#}%^L6x{%vg}kynd-^@JCiDl@QPR zmt=}ytJsbZeVy|*$$s%0cJ)eEi0R^w)kps$K^?~e&F%@Kf;*wttU)Z|Pz5PYTL8J+ zMg8&Swq)dHi#wwLxM>2`Jv-05*6k(nadI?282~j)L=IF`e+;=TG)a>TQpOMsQ&r7o zb=NKv&--PhG2DSvd{1Y|MSWQFmO$3DP~9k3SmI3(h;Pb1OqnW#mtUnWsr;_mc!A*VZ0eAuTgRtY_Z z@-a$mLJE0Uo(NHcC}fV<$+BdoP@(ll*{i_fk1N}QM*GG+r^887@J|{^?iDKZKsBUt zkDfU*FVUx-DGqw?BRr4W^%E}#!G4HE+HL?eUHr_H=O0TA1-+nXdp~Guf0e}TbHcz4 zHEe0KgVON_@&0z>(ICM%0^ z*OcEOdz|7SK5{N-uU*hPws;cN$r-Ha^fqEwk%*+5kI)r?{(4~$SX_=m3l|@Zw|hlS z{H0-9hqaK`vme|^67z&dlObBiiF|w^N&ntX^j|~q`qNM}7n;V5b*EVQmDeIOq8n;* zH#0Z$If5Z9a<4YEfoj!UX-b_E<%)3O4`(d%j^HJZ8xMDO&%kH>a|NSr5{n(s26fg+ z#M0Rp+fJ2ZpkxY05!Xpo(R@&E6)calri9#SgZX?TRFvOjsfUiRBmYL@puhS;v1eDT z4;cw5dnbccL>MV)GZ2&(nfjpCc+kW@DSmM+0+iAyh+6Vk@4sn3%h}`xb<@X-S!ri| zsJ{gJrqI7-2O1%uMq^t`#`m31ADMDgVf^FLGG->T*gIbZ>QB%B+ z$qf!Hu%NL?zXgBx>aj2bgm?dAaX6A*FP;21?)k99}O&^II`u*1EePunGc9?-)3!8|H z#gU4GhgrOx&}sa(m8k6dksOP2Vs0(CVHej*)!VvA;i`(+e1ZsIZ3EfpzVz*P~}h1wBrUy>mNsQ zXb((C?t-%X&hAyB=6n&>3zY@GGJdix#(JJ0Zb6ZlvtmCneIaIL(?kz=uM@lOv!D~!7|I8>fr`#INK%m(SXGX}XqYFmWLsE^(A8C(6O5QnnUZqpF7~C{3NSz1 z3C+g$g4LPJ#N6f_%lST%T&S9GStY8zd=R9t%aH#?N`ak15x8#^#hTVsU#p zC}vHP41(|K;2n!m>lcyo>2cs$eVEC1xw9#&JyGsCgBSt9ri5!m@o*8TNb+O(qUV#{ zysr=NT!N+43Z@&ng3PoJNrf)DzZSjD!m z^1(C3_ak&s#mTh#BJnZ;cRF~y}?5J?1%>E77kgI+OK z%V^?AESwvN4$+ZN|2+(<&kCnS&{`<{OCnZn1~bLu%cQmYP15l5K9QaEb=O{eCgofY zVevn=qz_0lL{I| zU1wUXx3s)UlGVMSa@i-+9QT|h91__lRf&*eOIYgDl`v5^5-Zx2lgt6_z-aIQlou^v zHn9ol_I;Z0E&B+@;Who8bNx_lF+wlo+}|_?>AyKd=+U#9r5qdA5?bMOjf_6*u!xFYUy@nn6%5?(BvWf z8tcY*KKMimHcY_4yvIy)d@`y$_PYnTcz`0NpUAfq`}ikeP?YQ|z84dz`PN`&`gRQ# zxSS&uVs@GFR{ULVoFdBLKSX9+0+fcgunY_*igbJZi65z?VvUv*xI01qn`uz;PY4!? zowlEaU}3eqBwF8Tta0#RsdZ;QiPfKH6R(AVq2Q{Nc`Zn2!jvTV9~F4H6qED~-o82i3G9rewZS0A6l8)hGX80%hCRpY^QCr8rLO@7?d z#+TPTO@`9;pCEsuE4^CiN~0#vrLARyc#-fZ)M>0hq3}kz%YF!{jG?xCJ$JnxN6Ydk z_x`jOtLpMGc~Ko0*;HcHXo>pD+fZkhU(u&5l6qShbB_)F)Vs^W6WfvOdmCtHEZR9m6TSVqzA~mVEz_er0&^R{~t2VDh!?ZV`k^~RC zxeTgjD_K+3S@cVf<5n*fn0xIxSbS1a!_HrrtdxwAzy1^BtgL8Y&H~!hs}bsM8ga{g z5md+_d2ZJb`ocYeR}OrGTU9Rw7at&Z$8o6WrDG149C_<|f2!$zj#Z==GC#{N==E0_ zD9(1{hVvW1B)>D3)YpOT%xD_7tsgDkGMtzDP2eV%GQeyP7qhSz;NH=X#`W~)xp&NX zZl75^Gk+5#@7AIDdVe0$;l0okZlIPg&w%wLA6n3tqgLE&Y3nASHnSND=J%kU|NaFT zc3w2T3~1SbQ0_K+FXYbez-5v5Naz&d4?Sne+0v;rwCiD1ldjBbqyuQW?MB1KW@0c} z$gZDW$z?^qfkx&FhcLph&b5Tcb*!e$DS&XLYb)%W1d=r=@+{TPQJVowD9u{_3Nk1+VKljQK zvAfE`t@s3G_GjEpa$>kovS7l@(_@ZK_!gc9;%QEJNR%Cak{HirV-Dzbn|-L}?=WI= zpcS?C)3I{5U;y37$I!Dq`S9SjG%#ulMznke!~A9za>ky@+U#&w_bMdjYt6al?=zBS zpWvl0H$ds71LAicOrlKxL*>{Hs56KxySD!m`@XZNd{Ba^4_<)x%L<4LnZ_NiyHca{ zKOsNdnA^Y=uQ$CG+NUPh_ zxN2^uUcEyOIoXqWM9w*|UObhb`12NO{+6+}DdT8Wh%wbp_NF;kK0=Z}XH*0vfd7Fk zq3aCh_nJoWjBim`n!f;wE++D&Bl>dx>vtg}@+4|)dy?FsJ8(o7N^2DnJlV;FI+QE8 z>~;zo%fvfL@Kj}K&-IRNgSmkFuHZ|E0M);m)LD^pvrQ=MRI*XWScLG`{JVXtkIqO z1!hC2r(oUv?#TUDMsi*j&to$@Y1H<4)VcHz@JXG{uWubG7-Btnlk2}A+qI6@-i+p( zudSqMkxIcxc!xUYqby~y114x&CstZsV`vPR?*2N}O=fPET07F+;)9|OmY12$|?s#<-*w)3-rjrx7YQz)h zvBa0!{P+*G(|$))FL=dg0u9n?-5ffeCusv0r^MK23gcY0uUkqeFP zBBP}pl)U`DUgRW9#E|oP%-xj=u0}I>-K$B|Kd4x+#XHiR%s)Z-X*u2cdkxrK2V{L7R!zIg>{XG%!j_2JwqP66b> zNAwTu!*y@hah-B6D+vqXRwJS?z@t0&4E3VEiEa3(9PvzvTDeptsE29TAv3Y|}TekeGG z$2*ol?xis5KRpqLqYribcK{l7>q{$l^y1zrrEqVrAC+|~0*5R+UY9hMHWp8&Ex*|C z(s{06G+-H3I@J?Hwt|)ZG91z$_n}cAhtlaa1Nk7Q+1$Z%95t>VN)2BQNYQb|d{6lT zx;nl$ZP@S)YW}yBtEQa6w46`OpKQnJ!FGJlhcLeL@;V;=uqRi2Si-eKtf}GELH+9c z?mTo?IhO64OqH$mENId^S~4#SHCNg)&-`F6BfH!^{~JVGhaUv1tgV=4cuQVCT1Ktr zh|G(x9jMAjdNT~sW#q#Mwd=N-lZ}8NSt`~kC#H4?5$wjjMRJWJ%vp+ zPtiFt7g~;((dsQvusCEb^??Tvd9MpCxbXp#%U1K2uwdF~)E9I7j-yRx2in|YJ??+G zk*dP(>-irEywWy^EB9%bu08{Oj=n*|+Z&R>$&*-48q76gilnMzWl*|wG>tS<@%n2E z_~xzac)f`;Z*E?PO2OVBzh{ALF@qCXXK~^gHCLauWu;n!;LtIQbQWj*f}9$7lGI>@ zShVFUq|JXpnnrtapUFjnsbYY-zHL!H`gD*`8--KV;QrL7;t#0*Yb`JI8_hHO)IrTR4F=4f%k#Q9@wD?bOtF3?^za?Z zt%{PtBw_X_CM{Wkm#vNlIib2MEa zW!sOseHQs#e-DJMBim31-??1rD$HC({${wu7AB0(mvFE_UqTv>cA}vv!%;a+=pE)4 z>jPe`qi!kdxoT|{4*%AR24}}p^@JX*ZC*F(K4<|?{p%+dWM0FpyBlfd-qlpr^lZ%Q z(omkgI)KX!Pl3hu;dQE7!Le^g3vT^`3adf9uJ8_MKh{BOCCA+8|IjXY8u#0|2W0+l z+~dz5fwqfR(X`HD9=32DwZGYu%R7nO>3VBw@?<>^Uf+hNA4`Lr9&5Ss+I>_%%VW9@ zM_u0-i^gz>)}b=GDpzL zu~TSHRT-vk6Zg+>C1t0BDLecCv?`^@jtmlY**1N}g>xigf}Cq(&ST2C0aRsuS}&8n z6qCMgG{S!qdF?tm&SqIrE)A!|Je*Fvj+3XhKaoT*#%T_DtHBc#nr0@^2)GrG_peQ zT`j!n@SD?VxYn5#?f#DSE`-Yx2QtGyf^Cue7;}UVyynlVKoWa$b>Ti}=})=AP3(D2 z8Q=w3LUk>BvEaHB&22xE>O6Ns)6?;^q|}PiR|@XP<9N8mM(*1pe&@ZAbSyI+WF{d zuM1uJxDT&hhNR*tvR`h8fuGB z1ru1o*p5_t1;G$>S+BM+W2!l?iR{&Klm~X9rlULZOvNQMEFQzsz9+FlvVdC^j-|Sz zF+5}8dS2Sgn%cL!3YHc=Gp zO1*l{gY@D-Jm|3KJ1lY~*OiZ9zkPR_x#H6=yo; zJjj&oh^DcI1!|f=b!sQ7J~{BB!M%A%v79%P3^e!@5v!0>7&@peSG^AAu4O%i3d5DB zR4(Monfa{tO#pqC{U0_=3*frX?VxqdRjBM{#&sh@;X1daWPB`_1vyKaWz(f9n>t9| zlp*r={L%TO7D7i20OjB;LVi4kkhKRP=H>*N^vgWXPBjQU;4e7nW&}^$eVNstoJxzF zEjf9$84N=IzjK9x$`=gb+AT#``6)$kGq<_RYQ3o2mfq-d=mC_Jya&hnpIDR_PZQb< z;iaPhN>hU{G^{sPXSw5{aYL!%r#-c5?+$hTUD0fT84dh?1(WxdW1!P7uzIO2j|}g{ z8%AEilx-orbeAG+0ZuC$_Qx3i&i^DL?@ zvccMb)il-o8aVIo#@(8KqAq(AR2&JV5z9TORc=3_``!&+hs7ND>j;qx_K;;yoJ2o* zy3mxnE>tNSK${mMRtyw-ev>e|^>rKSzugvsj&$WkVt;GCdN7Ti{QVdVFw^4; zwwC^j+L6&zw-eEDQDVO36Rm0koXIo+s}&pl0Ao`l9O~>bCqmXdZ8o{E~;zhz5j$ z{GqhE(2}&uy70_3ud)188YXG1dDZDUw2b^s@Z|bZXYVGAn;J!%OnkU5%Nd$Yqp8io z2G}2AOn;17N9End@SMG!(Bxp!%O{(*wwRQ`6rrB9^`8|d|bF!rcGk2i+@hFTs z(H*osW@Ck7o~WhVnByI>1Mm{gR(=<&&F7#p?JBynm3PGfy1lRG=8Q#Zz7SjVq*pIyg8qz&(I2X zz*T*yiO`@VtmRsWLe~5q)TsAOm#$sMRtt*v%`Ysvn1@a?@>Urj0 z##FV@lB%Dc=BnkQ=DOWT?>XJzj^9Ue)$iR<`SSrU@4t{%kDZRPm=j>~GKD6OFc*wm z1I_um4U&`ZW9W+`T$g=;dgrgFsnc6|+AtTauKx|aqNY;$(_=L0_F>xk>sH>ntSv7O zEa$H6){FZjmTD%cnQZYx{k`_7TrtWOn%9k^wWBJi^PnNTx-1*j#|{fWZzMI(d(Aag z_lVrIADEROt$sX%9~p#HeQ^t)b8`m|?fITNzFWxU+ciMkV>!$Ap>>ak(G;`2R8z4T zeSZAL-Ts?P{j7WOs3UJ_(WE3QTPk|DBokgZp@kY8%$Y7J2IZT>c(y*B%l@fn{tGXI zZ$SZ74;@aE&x$&D?hUTvQ@9w-ivMdTtxn3M6>E2kj2_Vgj9w~cAkOHbjNqC87s=4^ z1IqlQ;*U(sW*WP2ZF?Ky`@IdVzi%vsxz}Q$Q?VV-9~vf5?a{?N z*Jm#jZw#g7BLgT2G@(i}V|dobjyDH7(2CJrdDQedaHykTlYjaXyo#-%Ftf!iU-0w`Zb1IeY8SFf35KS-zkbMeZ?x{TFJx!e>}7>7%J~fQ} zUssTPPbYK?4#bGwJt28n8S|Tdi|x0XiBT&}gomJ>*!Mri;v*BV^8a~bZ9SReif^p0 zA0Y;-cf{hp86;0TL6XOd443@lOyTZ;Ua^THGev>!79zW8R+3)1G=RCyh+~>@w@6)a zBzm441^WjrgsOr9Qa)MuP+USGV{;5q?|njw_KpHYjVE|_s%5;L0)Bi_po#efa`kVK z*|T#ZgziZoCc*DmQM?~C&b~;zqUI5kpjoUs^NB=EFOmpt3aPr_0-lG|7*d<~e-xcz zK#W-%hDRz%GLj_8NRlKYq2@i$7)g?3R5Fr`Bq13|l4RI6Ev-$G*fz5yNfO$&de1YG z*p}Eq2+1I}B_tt)?|gs#GS&OeInQ(7*99IAYN=PuNGM}o5OtGU;yoi2I(*u=uI81X zSv#HOZ&pI`A5!pPJVBXVJ*S@2A}O0v#tSTstFPZ*h*VU@JeP*-{AtHoH5-8MvMQ?f ze9VUq>W$6XW2C}-7?@@b!zlYv&=G%|FE}2BtsfSmVA`LOoYr*^K|&z`lQ8}-)_qxU ziKtf&L=<;Slqp#JbH6@6>wn%{6=GMsPRDuppm-Lxbe!WVYdUS+d zS^%Pn6Zq$!=b&)I30@F?iEm1qPff~zhKD#J$(jl@WCHe#&7;Z*=FC~LodmC~r-^na z2}|hFyt1E($J@sw{?j|!6zUFX&CGq==1d)&T8LAn4fdEb{^ulrsIjc)?6422&z`Do z+JAz!>@jVw(iN+*883Y0Kl8pTM@a^0nmiu#p?)Ev*VUW&S-*tu3ibU*s zc8ZcE@gUwjh&WyL!4j5R^HyD90hEK`?1d|4dcazEMH{7V2EE z3hQ5m(CgkCK)%uqyB`dvb(_Y+lu=2Tc1MP0>|-iaZ>cWw4Cm+R1>u+Hg4x~apk{0x zy}vD=Y1YVDTv-GyCIjJfTr9|j^u>y2!_m8L49YG{OWBTg z%~_)V9>GVr+pv7-EYfZ|4IQRlC9S^~GRB`F6mOpf%}@2DzE%p_9V@Uw>xAm221y+^ z1{01spw|EbiV>}(@&r%SGc&30&~a4Q!#YikEJM6-D$0cITylX8czDhMRsKY@Y`;gZ z`>q7p-Q`$+GMtKoi#W~q0Oq5dP;J?Km-fUSp>makD9)^cMAP-O>)1ZxF5OLC^nD;| z#y}JtT8SUV4TjanY#>w^iM~1sHfZFao_?QLPGlaAhnY0G@fJ}#jg@HT`k-p=O5z&1 z1yt7yxy;dwFJASWv^-mc-J@2}7csrzwXY8dhV?|NADTi%p8>M3(=<NYD3wvu8;&Lz_Eb_B^8FGS6E=9oS`73>Y%P^0E} zuU<<)-1`sS@zNH|{6&uCiF;WW<|BFGD1<1T5L_O7Br(T3iPL+=iHLnlnnn|1csGy6 zP2<4*mrRlrlL#^UoIp0=TeU~xahhT@0O+1sARwMqqLZ_E%P$pVwa6L`U;#u}I%3|< zNnq){pEw!pqM|l$zVqv3W~!0Hofq@b!+#Iy_NgT|SB*yxze!NjcQv2$cr$3Cj5%di zH@!dG7(|tT1=$YZH0~f3K3YnhZaksZ$|PFrG#!kcSVk?J@mibIL^*yjxx<^Gc zU9cH+cPUMDb|VGnUeMGsJ2ctBh)u~Lbz1te|%(r6sj83ASyqR-RvR>b%5MF(E67g$J z!Kh)Z$MIz?1g>C=l@(jSDc6Fis`l~OAuQ7#=u66`TGP6eWVGBYLBkhG5L5VsrtavC zF`f^oqAHK|!PvV~wGJ(hkHYd2Ax(Uk;JOg&o$^Rc-@Sativ~XH=_<&#@kf*SrZgQ{Z*_Jo zSUM^&7a4?AmOW3;L53)$SwWlG9ZG=R$6^!77ly{Z314`(TJbitV?1 zNWtr1Fb!IS;$K&z+d!7>?>>mAu zF?qz8+2KID^na3kd-g8SrSj#gnLF@EBVW{gol0(6W6K=I^Bi%6D(@#k(XZRE;kyhd zV7Gki+_K>WOSE*i63v~uubQZiqKlJTQ3%h`hEOa3OW(??@RmoJHra7WpO z%e2+IjQG13k;Wr^u(PxuB<{LOj*J?Jg@RG29Gydioz8h-Bu+ZvYRXI2}Swu9$0&ID?Tq<1eU4Iv~!(1$Q};j z<%$UCo@hfP4ep?+?}vh)LqWay14k#^qV+LJAe!IAMQv1o)ih7==ym~J$3Up~I35Kp zIW*Ch zvfLJ4{4SRy+irodh9M}J^D`|z;0X4^H>0vq*g%0%&=*6Y)n zf>m`C+#fX$TYk010QpQz|FoFqu;)6kQvmeBODeB&CLNR3;`8btG&#PHhA9Qc~ ztdpT_)o$YN`<|*h9II{a5cK}`fe6CKd8RAEN$$OY5M#6m%XID3=lM8v={`e+V;=FE zW?Oz&RVb95W*oxKDKz4*t*jre1iQ1dvD-sM&$=&1**{CsX{iwiM%u&b%FS3G`aAuQ zG78Mz+OqGI1?2t_OALb^B0n~fG@WJLTBDoQP6HC4>x=+;=Jt@Hgtv_I-~|!u=R^Fu z@3hWI0)e9^Lz&ZBa2*;1jgvE}s5&0P_4e$XeTnJkN zP;FU<1qYA>j_`v!-)2G?$>(mq=0LrRz;i3s`PKbLR=cf2rS&OJGkPMI5xtj+1`XlW z0l)HDip5}mh~>X-+hAJXg&>5JoEc+rG}5I|8D&p|Gv^U`I6HBhu^e+_LX}OzV#qAt zLS)mfK^`Sm;>=-E67%lGrtm>NP@5X*|y z#Zy7Qot{}KA=vRU6r!KLq^ePgw2X0q8v}i@AewOv7xx3FXdSuJV+S^>4Pd%;2I}u_ zv94LwA<7c8c6i=sfV zSyr8K><&@CsVAxd7q|vbKh*RcMtw!h&3Ute9yz+6ooD5(X&EY?7AERzO4r5!Rs;N z#YQk?+3|w)60q^vhW5eioN>wmgfYWt@b%NQ;gu<-I#0#iJSG>dgY zx4)%k(;rf`tqXk4mZ0GL96tB13u(XEMDpDj6ZP5eEa&Mh5l$V8MR7@J=+O^!4{~Wv zm2MqE7FmsQ+&R^-J7@idT$z z{lV!`u!1ATziZG4xmm71vCe$6*XMrYhn3l1Q*l>O-x(HbJWNI|+7ST$|CO zz+AKv)QjIsyavQWdygAlo+O6rUsr&@W<1|=mvtCA*?c6*45}{(uyN2Ba9SJ$c|(VQ zc)(^7`Z^8^cJ{;EU!7xyTd9ReD~OcMY2Ds^?`pzD`n+V*KP5m-EwM8AJcjV;+7 z@}!(-R8lT%_i%{(BtX#-e@+mTS|vSg1G0BgA{X@reTI}1ehi1o9aBJ-(9YG3TMp$5 zEFrCgIbWo=iPhEl`0ZFU%D61Pow)^cFJJMm&E?Q#b(YBgDy1QZ7eh+X5=fjsjx@g- zf<674XvdqAy!`kzBG|IWv&br*Cc67z-c`nw_{y@+gO_5*-N(Fm!G5ZrevrD{zemEV zbBS*ITdvKYvH!v-FO#G*tbUtBR*v~Abv#EgSGefwfYIcX65JV~mR}mHaUT!L}0wE0{IUj~%a%~lD zOB)NCJY)1%pQhbEej@do*Fw`-)*;#%579@aL0i3w_J*Vcncm+VE`ze8`%G8QXbkQrk_7i{?Vp@i6H4&4gIb z*iQ6wTX=l{$8{aB!1~itOg=pc8sc`a9RDSvxOs!RMQ#B1_8AcN6{+?|6L2_d2AX5} zylh=5O&@oZbo4DHIX?ukoJ1qFdl(3I>%`FhVFaW{%%Z};3Dp1gRVwInffz=Hf$@HG zu%78f)j>n3$t`~{k<37iIDnHc)$;a2##$a}j86MEP*MLhzLB|9x--JT?o$E;+m6Cu z!*bI0^)c0cmZATvG-9%oWi1zNAl6={So+gw<^b@34zdkP4vk`S`oD;%A)U0{Jxc4D z15N3Ej(OayK_35(q^k)@I~I?H5hL+_@dzv*TT5cy$D;gCYt*cKN6Waiv@Y)*H4a<| zEiYGq?CoRD%p?pVw>+ScdUNcQ&c%e;BS4{?L*r{c(1JiIWQ!N0xUGpczb+;s<~+)L zbbkSlU6&6LQAq;%@GJ_FTPLE4@DWib z2Ujb`Zl&_wFF3mq93;Kk49R!x(D7OlN~AL`WP3f^yjX66tw_ee5BN?QQNd&R^ z5l(c^oHSqG41V=}LDJ6rJaJ`=_0dJed^6wmYcw3HO=h0ORnX?vN&F5jMD>YK$UDL~ zn9Y-DM{G6~)Okr9SeGRFXb*APpFrHL<>>V(9aMdhPotduM&ZNuxH}&C|dDah=B_xqq-)NS3l}U zG*bh3{p9snc5EEVXZ%aWJ64cGwv$oY$+#1}KX4_kw%{J&gnpA*7uoYint6c(6RRK^ zHs~;^iJnXa&OSoCK1ttVVr862evNDC!IF^q5HQ=s`fZ zfIo@bq+0hA`e?-82#rV~ioh1zewv26al` zYU|vg)FW#oZU4&%y!OuqQS=F}U~xS?GRq5uwOO=j!JqWXtqIUjltrE!gkxS5+qDIm zb2Z~+e8pyK!!8Q( z?C=?2-aZFo%H+qV<)Bh-|*Px;fbq8~j}GL(^o`NB+o*Z*~x;rn6M=d{R}#XaXkNhk@VMVDP(U zg0fpx)$tb#$>+bkSuboFG`x9CV_fbNW5)T*z9&P)Ss!#+CxF<7Ak^Ejp9h^y+Q)n( z!sWe*)2Z=T(q|0IxxOQre}CbWC6D>cfEq3?&uD)%SG1(VTu-z;cwRKXF(%!+%$=9eRP5zGJnBjVCqoO<%b>fm{8 z9coy?p*haH#cQ(ll zip87Ez1j3dLfc-5XrBFFB}OP9W{KU812HU$%~;oY zQT-b$8r{%H6g!(~yUk$6uFd1ix6DT62Y=?P^O3a5dSR@_2fA1KP}R~Ke8n^ql=sWy zg^q)1(Mvt;IaNVd-|hpMu{&vz!x0kjmpO_B$N3h+F%YMl3%S`&V0yz7WP^f9mc#*L z!uw<9=A}foXgv*P;F&V}Iug9*3*+leM6=LvymOj)F6}0O@@pa}k28*KF1uq+zDjF4 zG<^5zVA{GklZNN1Q0yLs1yA=8wX*?r{%tVTDFOi^n=K*GF_9wdl)ivQ*e4vhz0-EvYkJ(FCS zJrG>ut$_So#db^g`5xVSn)~o~=6VZO0RRdXWQtE#DRwkIaNwPbvcaWpa|9y4YG zM9TM*4l#%3wt(UwV=4+CN{_5{1@U>tN*q|t{6HTiU1iUS@0C1qKi&ni$80Cz;lWTh zJ)2ah4NxD;<}E{esBl<3aa%qE)x|5o(#--&y!)cQ=>U-iO$5ONJs-Ga8YVQ@gUxpr z?2yTXslN5qkKl*dFoMa##p(GZ}x5;m7HYzhlN9t zjylfzoENE}bqCq?$e6MG|Esp%U`cZ$?1@q#A(so8L+h_lyuN=H=$#MJnz<8r;ZSq% zxZOZY{g~5Z+zM#zHy2v(F%G~5AJB(A;Tw$)5|$yQCZ{c+wx0lUC)Comp)#r&r6VqD z-%-KRc8RyqD-!vdot-_Jh{>1fD6>CVoyb@jn$}@>{qr#Nv!00M#-?B+-U83pO@&xR zJhly83!iUqWo$uXP=0jeYi6F5I1GADg>f##r&)#9_iVwo)L4>cn1HTHR+!fv0EORY zvh(Y2)I01f+wTORc>NAiY_tIM@ATB_j6Hl#8Gs6x^(atPa%G#>Ve(kklOOh+ahDg6 z+~_&wL=uh7OP4`L%}<2!Cb$4L!xfe{(Y%_AbXSA|TP$W^hvRMD``j6#7kyT@fL`{F;%e@#= zCd?2@9R(=ZbD=8loeJ)cV}GlMYiajOOSb1X)s@ z{40#-Eo~t^N!`RF_ZEF|4bXS>IxPBa4Tg5i!CgB9SaQb#ohh5?F?LRB*&R|eaSgfN z5)E{{5!UZpP6b=HpqC%(sSn#ib30$t>(BaPisKYaFWQO?;&9aK?YXR38$l^zIrj%< zL?%)~+EjLD`q7EBjf^A$uS!ot+hQtu{f>`D<_2wR;7l?Tp|v<1c5GM(mB05z=ij2( ztadrI91)9|r8~K1GX=!Q0PdQc$>fdO#J>vH-53|||O*bym#1%$#&+RyPVYmu4kACN@|DH?R;s#Us zGdITcZzUC7tFie?9!>vmA(U^h$7Zqx@0U9=?vg2ZP-pO6;DyTKA$+hd0XzG%tjNex zT99}4{~0zSs!k!s#1R$p8?@tg2B&}g8`-nT5Nr(HP-jughmRSIKATrS$*g z@l_`k{Aa-W)`rmh{XK2Fz&v2kO1&p0qcG8!nBQ6in#O?~DOiHCd5)xg!yTf(%2OTV zlDm8zgC2tvG}YxAyALKn`s-jSkgV|Zbt-dQ!R*r^_T9V*G;=>u(-3oP zO-q7!{TUhb1L9y5nvLg{MIT~tuhi9sM+84AwB zSjWRMl`r1aA2lC--fnnw?i^apy<&i20gk+zI~`U~~_qc^GVbR{(f zY-T)B!nGuBL}lDr($)Qj3gX%O;n5l*ui8&)#`UGm<Wfsp!3+F2eUj5q&n7IZ&; zPcpdCsCYjXHP27+VLSdKt|J_=^?ec;?C`;cp36j^w2w}Sm=E$lFL63mDC?;IN85Dw zNc_U*)ZxtpxM^n%*2~6|^88F%moyLr7v5F{HZMor>wEn5hyCESeH@s3_xgWc9a|^= zNA*g7Y@0uZ=r%D<%N#a$qmMZg$2Oudd(78VP9@EEj}x_=wr_2&;~S>F2zzJLCM7+jEU7#!^4apwf9dV z2tHgXkRRf$Z<>PQc|6U`dPi2i#?vlX4Hhoj}CY#Kl1E>Vv0qgVWAqsz5G6!jX2CPBk6_bJQt{=xdrt;0D% z-|y8a{|twkrgMC-&n@bm$aY)r2Z80E&Y;~h9Xd?bfHXw{e#=Hdc@&#T`nu66idgKC zu(;=D8#r{<4h89MsGx)@QXEn6_7Bg_!Jdql^@{gTFaYKF*_=u04C0IbQJ*+dOxX|w z<)bYz_eLnm?l@0-=E|V@^K5t_9FIAd)`ISoipUaPNjiS_q=Isns?6$ksyuy=thOBi z>Qhqa{J{`fCvL*F+b;C_y#Z*vWG(n#`9|6Y#bM@ey&=QuCz@V3n#Nt800PwwNtdRI zL`5^!(CTrRX_~@z75P**(G9DU;<0ApDoowdL5nXA1C82(d{72qTW`vk5n;4t?__XU z8w)PeHe%K&V^H;!lW4Is78|P|XILM&={6anA4KB)-fNg!u7FpiydoYCX40~CM@fxs z3ZIeL4-D(qgINe;{hs~Gmu#(~feVt+x2%k)PnlQO{d|u!-T6iw#SY;9WC5xxwN#Mz zlf=Iy6iu$pCZXG0@%5T0khF+E-)seiCuL|Hn1p-IO#^pb8W9NpkvK4}m@w)BUleLh zJhuHx1ZtZ<&HOb~lEys5ul8`?`m&tBj~taP5fgPlR<+HH1(4TJMg!UGU-mSdw1+*U ziu8Td)pd_A8=A8MhWeb%7jaxKYtd6{%f=^@Tx z%dkG*1Pm5Sug-j#{^~34W@RtPOqWyBgwbgI!wF(r840?sR=)k$AV}T) zJJsVLE;0NJ2_pj;SKN$CuH1;(-Z3az5DPo*h(NI5wZ!`no5e3RA>DV&sBQyu2(6q& zy}#@wf}X3MfwzRv`g}9klv+VspOK&+{G5pTTEOR@3_$+kH`+Lw!@^xG*KoO+Ka{ux z1sm0#UfZKU^w(R`koJ*%zn|vR@*$i$XpSWBV+6ZjNuV;Jj(GJCMa!&Th-wnchFvYL z&R@L+qS^UBV_q$dpV33Iclcq9fbl$+B(bw_W0me@F3qS}4=q_M*dEIc{ojs-ki#72 z4Y^5maVqNHr;=uEFay!^wb0(BBY~|$*!^tah&k^mnM@s|qvh(ny%Lkpz1vv3}$j@Nj%i zf=8{x`hJDfc&-<^opOc3QsxW%p@3M;@&HY%5!D&r;|jxq&~Q`-5hOcQ3$AV8)9(&t z^MTn|F*^xOC$0s zw3}tR+@IZ}QTpki-;+hZtsKYtKVN98doX4&mrSB7%X&C8@xot?N%`&zglhjJ4Sk)# zLwbXJ-aQCgA1ue#+1tst!P7B&h&4Dm4uS~B=^zN!R0>`7%%`z}SD3sdU6mSQ!tVGz z!PboPtEXju?dF6F4^hFPr=DGMN4(!7#fZ^kz)EI-SC-Gmu1hyaLwy5nsvAm_cXn~i zgT#y7M5J+d5(tAUc-IdsLve8t$d4Z6G7W;r^HA1b)yG4=WD6?YY`Nk~LD*5pb`V$k z(8OUgNW9=NH9sK2XtD{Cho7a&By(e zh#6N#VAJ0%#N=Tz^$rcfiUm#>ePj)ZsblXe!M`+fiY3b2d-Dl>mV$1`Q24NA1GL)p zkm_;kVb8JYj2HNhe0b9b6sJ#8Rh0@w3O`H*e|PMV^l}hq4VF{1LVv z7EbD+VFzDO-;l2~bomrG^h*G`U-*f>{%nLj!)!4$!vZ=+N06jl15nP~*RsqaYVH{h z!Hn@}9M9gHJE}?4_!KlVW^Rigp7BYQwjlHSz_}BavFognEe!4Si#X5sIN`U1@@fpXSi&>a|eWlgakOwjj9F zm#dRbf{f4CY1xWoZ1bPWc)HHCy4)2qD*e$?Y>WBqWa54NZzBIMhjz9tN0DbBc-XsO zJY!qCEuO^k7By5gYXvb@4F=|2p&gxXIHiY>_&7%~cE%v^wUDAo|JO8l|2_Jl+y%3K zn~rt0e(u2UI0ce=aIKBTafwIhb zv~J%xP^kKlOj$9nX?ev7{*I=BWg9T>`Et-4*+Da2r&9UR2CmNdH<}B~dpu_mt-HMn zo4nS8zsX0c@!PqAExCW4UU^`IMaP1vn)R_V`#V$J z+Eoy}YBQ->+?S^3hJaEvnclSDjES4`Xi5S*>oK>c#;6Z@;=BOc%spuy+jEGbdvi_| zv)N}D!3nN~NVGEY zUE2|w$TGiqL)K$a{R*-=$r!s>CS8AbG%tJhxZ3`O6j%Q}5E~~Kk@m7sNKRb?nlk~s z+aoj7T)e?p-*ZUiWfKzDe;!u%vV_npy&!3(80G;skS>808P8skdVM_STm(J)Qv2piHffzQ4m1MP-8S+%3Ong z`p_cF%|uu8lI$7d&vML8RKDK<+U9+rMHhNO!FMF3^OMkH`UtW|XoR9OA2~sshBtRg z!n$LO<99iNwv4ld^o`~u{wn+Iql>8bkG~S9om0vGGfky493U=cC6p}~PQsS#B&MtAzIlRUNp_eyr&Bw z`qDT^e>svg`9#zFdie&>#sQ0duByQ14EL+W(!M%@=OzUt= z(JaE!)@^Woy*K)q8$!2t5+=tUCvAD``D<+AGXK`o%2aFSKorpQnOXEYbK7<7G6ym| zo67Ixa7hD}uztgTMA!Q%pDlku%0dQ`_(_{UJ@_IoSoBWPFq3hXBgdl4KT}Y=A(e~x zCpv8I=-kdjt?V_0vhIwa{`6jd|Fqyw5fcau=+n9g~i6 z;*1bdsfea!1$MNio%yd1P9v$|6Tr$|fNf0!NI7Hh_&uKn!aPUJWwQ#eF$*wvN^eqg z=OlMM*9i0eTMuC@`w~rtqF~IQ)q-C_$n^&zFqs%kT!xNAy{LqmmDphOqJN1X@Fmer zFhq~)3pC0>$?_==sf$QUDvc&U5o4nlt_WZ~`e0%he}b4d*+Pvdix<@ta4t_>AfdN8 zxa?1YdT#~sx;YsyO9!H5kr*rCU+U|UL^6K_C_J}_@t#*=>En%{ACyTQhWLSC{dz12 z+fGY%)X|!%1g@m>cPd)+p6J>qkeh~L_FRky%by=pl~qIE2Rl(18Z$5Q+Lx==O~im%Ca3Fn1fT0 zMq}Q4W293F%}$<+>N2bvRRYa@+D`L}lhNae87ZEX4E7@opg7h8 zJ7(2Tc>?QQf3KN5MsDQdF*PvUx7gwDdLApgwf*(VpH(=Urj?Ui95{_7(zNJv3Z zrxVeemGbI??`VW=5DHEVp!y}pd4Y8Xl|7fyBJMUx7go_GGZhL3rdJ6Pk5^ZhSTmOO zOl;b=f+!|xXc6Xr5jkJi17aXrC@I@2V3*YBx{i! z=vEtnVrdvnoVA8piAO+NAt81BB~bHa1Yc0;4uXd<65Z%ZPLs2kF)m%vEHIhn-j#e$ za};C7tOVcuiKO!A8X}Z9K!~LrIyG?+S<+4u^-h>t>;(+zNIg6$E&Awzs%Nt>kvTZ5 zmtG-)d-g;#W(*{%4KeZEG*US+1a$8=L*v^aP=9I@h&UBZ{{bNH@1+nnmqYu;B`Cb| zjfmcmF@`yWW>rZtTWrAfB|D>T-G;tX9@6MV24M2zX!zi01eL2Z2$7c(f#-RW zvF;aY=F9F0{T}e0*L|Sy?sQZf_(she`Z6Be22dW-P(gY|b){1&k==SnWik1Z;Mg&c z`*JEN?wJb0!$q8UtQ6v{DoI0K4wX%_<_#rZ33c3pH7Dounx7b3C1w{@1WQSC!cLlz z_n7#7a>dL~H)!@h%vWC52mI_-fNlCY;&#{!q7ILT+Gp#qqvRr&mh6Vr;KG z?1_#O+I-~#O_(ngvPi|lPza9`f%*m#nb8y~yq$#COYE@h-zs7~^F6J#m;$k)q2PN~ zKk`=${n8hMDzo_>qSn%2^Qw$=FIs0jwl?28FMjYR!W zC-jIbC7C*N#+q2m3wAv4^tc#F{OhNp(%@HG@iZCyQA0Id>~3<&6<_=u4P`-Jc=7nN zEUU8sdaflw=sp`r*&T%$pO2Fly*Nl`^E@SEh>953MzvQ-5|6k-&5v%xM#-4g&%Aj5 z;{7C)m%)_s{-9f4OPf|kkOJ-r?RfGBXZ#bJr4I^6r~Lm&&4W9{Lb)Dm77XHpuOFhm zc0DwI*Caf`_HEXOF4H{EmC!i=Fexnz$}gnR&zU|bvlQ}{9%Ilqun(yjewA0;UQQz? zO+mw1tjl+N1eQvpAk!m=7>;3`I2#2v{<)86W_Zz}Yev+#j_vyUnlay>05rW-HSPBF7KpJ$0PGs24on&2}M`_MF-6V>IHp3Oizk({F|eAU<)E&oupwSpQ){P23(6 zg~XLe0+{pL?+b6^JPNJHtbvBjld!1aHq9H<2P=<{q#YJ)hVWMh$(*-}D)K{V&7TuE z#ZO&SvNZtC4o-$(H$P}qeW7X()-7xCVy=#H#N@&ns=luRhst3XcJ4Hd*2_So%;TC` zjEG4uH&VCD38nj%gZ%6P-kse6GtE|ETTu(uTg`!X=dWzu6a$)xx2UvhIh(=$K|219 zfwI(}_>2oy*xY{!D*7BCyPQ@){3AIiHf7MWlUHGfK*!tpGB?Mx&!nmF1<|~TM_uR& zoU%3?^s!m|w{lYmTT)Jgg=}`!{E!rmoQUd^Yxo*?R&6t(4>a-@sPO${Y9~s7%ft)( zrIsMPU{AWg52L!1!)S(j0v05VKy{8KSpHi;Z$2fE?Rk(^zUWJX=P6MC4ohr=V-3f8^rBnsoKJb6dA9B!oR%HchE-k ze;v!7+W zjx2?mB13Y-G?;k;9l@k?8ckZy91N?E@m>eSs6EZNwyH;bQQIwAdD@f;_O_As!VM^1 zriA2jCA9UepzdteYIgbpZB9Q-$~wKMAR?WeV=OUx>^9Jqz9I>WqOdF{iquSv0EYd+ zlsSP|7@vq18E)X77JyFYmr>ipd#L;V+oa@v9#wbF=3JW$z^2CuV@z_1AZH8d7@kiG z6mvm0_Z**C5>519*xj@76i;r>L8sZHF@54}vU@*r0$G#=*Q+S%^ zJpcr;DU36`oI9K8g}E`pP z5cVBbGI%vkabb6lq+Qfqa+ioJFOke2MuTA27+$?Im(wSP^TGEgg1+uJV{e})-E-MH z@>?qNGe~&z_x70nw;c)aUCdmb{fK|fJ(`p?6TCYflMKal)_u7_6>phy=FO+-uG}9< zrRh8<{1yb6s|KNi@++0IOd|b_d5|*vXuD?~>p}lT)H8qPGw<{zmCe3TwmAe_N4+A& z;RB)ez*gogH-_{93P|kjjH2BseDs>NC>!X@cX$pUmWIW|#j~7H5zEr1OyZ7Q8iAif z)?;46X_Ax2GTdhdqIGi^blUa>-z`(62B9OQc@IxRS=(A^+Q4LKu(k!#)z2_7-VvLx zT5H9c0Bu`A4rY!TtZj2A+GrOK%=%-XG<7oLowaSz$_DkOd9K5?F1I|T)PAJ4BX0xd zJ~)aE$EIpuB(0M+*X#!4o~_b`^eSj>@Yb5lVT`(8A3*vak0EcwMs0Vb4A0IUFKrH+ zB<(QSi_wpL!FM3Lf8AZKZS~!%O?WMkcKxBy9t@GW+%^t!27 zYI61&+K&5#@txMvilu$Ew%e9TlTI5->(`{R-XG(ZJp3KAwCklG^np^R=iXA~`%T(- zewg-(gR3?>G*%k&wzpLCWu(;oM-yqY#cZwbjkRd?595Aq&&HJaXl?30@miZ%iP93k zQV9Nh083_U)ke$p*wd~D_xu@JvFB>3Eaf^xpPS8G(^nzmyc2~vUBMe9GuM(TRNO)8va4uSoXwMWjmNMF#w zj7vNp0t!}Zb5|H@O}58M3mkp5ZDk9va-plX=5aEyV4OJdx%b#|Zn!jX_F!#t+`o)3 zJ4pNDw-~K{TB!EP#H~{K+9}|kbRK+-BO%&lvs9fh9?chrOU;5RvFy4tW_=ElHe|en z&SS%*f)dJC3WqWmSU%$qE!E1;6=M9N{#sGIEzzsaGT+Z;X^qVVkn!zcRTV3>xHU;D zaOuT&&2B}3@4PDO01K)3&KUN2+8IMDhx)!=iLDm5pvl}B@+z-D<-udn%Dsd#e9Q8n zVy*aWA|%hbik)Pr*7mBiH0OY+wr%ke#x`c<-c~Pdr+tXDCh|RJ{&BI?_0Sq=(gwy3 z=G>$@OZJSlF+b&FUu_<=qH2Vl*28BB)|5X4nQH*RPtzu0vh=K#OIDSCWrA#~NmI3oJh0OG#tE?Ik1%bwc`I|q*-H~l&9oMMjI=uO zBkFf$tJWY?rL7tI8HzraqhLY?pL{C~;|Ex4%X6FAZ0Z^f-Q>!k5MrtFlqGGgI|}01 z8+ctgTxz_2mUhY|BdN=To#6A|IH}naTdl>L`O*>-J(A8wwljPWNmmlIm0|O-{r+fa z;9G$-`rpSOx>$y0C5#0bHB+i=>ZSG0zYFC}cR>A70Qz75B{jj`*!Zd*#4jwR)^C!u zHD8@*a@Rm@_qn~e$IwFR^n$q!AJ&2EvXxRnwuUz>_yZlv-$KXSjatRWY54qgFKP4B zG?0C%=Q9RS5RRGx^5cvXHf)tvF*6BDo*jhZ#9>m+4^yPRDiL@a))r!sSiK9Z5AFiT&Ap|*yKi9h{%B17`yY1CxyjWx zn@ZC@%cM^G2jdHChDnc3&^Cr1!xAUPSgyGQ>H0>-epxSlZn}*z%5AjOJ7cxB=cQ83 zEHfI&P1L$OJ4l;GeFMY2Go)o%TbZ~0IOYXiU>SQI+KwrMJDz@0S?@Bw?$btX%UVmR znPG2j!pi|#*{V>8UzCp#VuSy2bpBy6rr#eQNe0PCWh9IwNl21uo^!@XNJ2=GFcOla zk|be}gqA_;*CxZZ#D=sX^W0}PBrQn@p(QppHfaeV_IrQ6fA4i|d+p3T&wZcse!pJo z*Ncan+*-$*0uLWS_dY?CKOmnSwH~6jym^zf{>i4^lU&uhhgV?-nW?v}?`R5??qfsz zwd%}m7HUb+8_L|Un@k6t)agu*&)b=)Q+u>km!wsZ-H|xc{x!Aa`F9sn-n^-%j2BPw z(a*y;L)Fcsf82sie7+~qKP&Q&Y{at zYIDzdoCD^74wEv;z`GK)6S)hBa|xR`Q>^kqD!8;Jt9AP|JF7O9q$zjeb zZ>^({q;b@g^$+94U8%(97S-e24m~DxY z(@&V+_6vAEe1K`1?JQICh$1W=P%p$s_c8^IcuUoD$C;{E7ab6Dbd~AWFyTX1^*sZ6>LRU&eW?vVyR9#eGiLIkEO_yF>2p?JJ5Xq z_lkG$FfF0W7k~_3jdBJr6KBC z>Cbe2pp!}6)=ceU7h}o~UanR|Pon@&FOx#$qt2@bwZz_C9T0kq8veFdZ>)4PO|Bkc zsvlU1VON)No|M13OqYTB;Ba*?%r<2Nqsi)8u)1ir4(;FfFdc2|q1La6ReQAdFomW> zs9Qp!Xq!o6TE~AY`Q;-_rSfQX*^U0DV(AB#Bo)!~ey}=z<^oeaXDwO0N*}0(^k5w%e5us)n%aCFXb4;9^pJ#JDDl5!oyUZ&-u|zVv@Aq zXte2N7vR^Yo5{>3X8>oYvtK%!oIcDoRhF(%=SEml>he%^$%<#xdzzOz z{_!aF!GY`4#r=T}PO&o;*Yo#)#g9Vk?O!Q&+FdB@nrzA+dI&l8+*I9~OPVg{Sr*P$ zx61BQZ14{}JLO|aT{6qGlUtV`M!n>%cC%M=pCkbHxp%X zi>UgSR+2Py6+-$D>T6oLv$u`9c7+dUv(H1IX)%Vy>d9q(J-NK(Ea`Sn$nx=hRPK?J z@v*t7*1-S%`HdK$e}Rflb3q%?9g@>0sE?g@P*>P>R0kGJR@a^MHCgUku1;D|k1DVE zCV7VrSYO{w9o8^TtvlY!RFvMvRLGxMk}4mg`-cM1HqGaJmpa~&s}ahtX-vAr>&UuT zsI?7z?sMlW3maLC$v-)nngd>8vewR2kQ8Ln+dGlVp-42WHe>AiHB!BrZPF?y^WND` zEWTxB3R%K)7i~YIWWZAv-Q=g1t)8UzYByBvFs_rj=r#2eBJow1=^e3Q z*Kg?3e<#%r5YT<6j=6w_BJduKlS$NR!Yor^A?KaW*uerfR#Tv=lgTAZ%HQqd)a>AG zZ0tWron-ftYPTGOgvK+NULTF71b21eF-Mc~c^`Gk10$7KxU2O&Q`CVAuCU6+VQRzG z81x8CQtN-4hzX0`)CILR>I};mb#?YPOxY_p`E2tyS-zNNdRN?2o$#fGB$q#^k~jy# z>7bul&-x>*E2GvY8|di55hleWYf2fWRcHTaWr}zarS3hbzo~G@P<3@JciFy_n38;L z$>sG_46*J&ZZ11WclQ@D^65B~*;a$*f-f_8OL*sFpxWu9LfyM#yh;17z#N8;G1Xr1 zW~Cj%Owq2>P3vYYG~EyEV0zb0W%A53Q#U4XHuDD~y$&2|%HI{ox!;WW%Ks)~+G11C z%jKqUrw*pt_Ty>FDI67hbB_2XuP;EbvySNXiE)S0(&8wbR z`SwrH^v)Biup3&f^uqA!U~I_rBF&n5V^gOJF?U`ZIGFj9k3|d_HrET19}cMWsTE9X zmcc5IS);yaDq1)uQRKo*R;(K)%D3!huebBe(%e{*-%VqV+j&0y;8yQk+i*zIbO7Dw zy{vS#fXqCHZ9L>hssHnh1&qglV{>@FA`I#?V#uL?4LBbTqo&b?kUe({Dl#+Reg}8d z{CAYqHIJmc(-TRzbO>^uBU=B+Id$Oym}}?*n!^E9+$>`)JtA3dehaI1>qwDfv{<;R zGd_Pi9ZQ~eM(N?5Ebu*Jo>hCG@Tc{pTi6VFPnF!W{DMV0&%h|JXxe?>j{CSYi;WYW%DZ7I)ej+@ZTa|@Ui-;L6p#SGQ~XCV6LBx>Xy1w*c|TIIrWvvIU`85Wj5vf z{x_4H*)N87cEY4^C5D=YQRE0Ib3eLMNNmv1{yIm#V_A;cpVh22itkZot`eKBnxJr$ z8Ok3Zla+23HRW%i_Qh>M_kAfVFZu(N9)B{sE)EoSgZE1-`r-Ab&Q#lOm6)$=%RS|e zc$)Jk@|)LtS-1qV>*ZdZ=k!Np44egpg%@$*m)UmKBaH#C<6i zXMa~keB*Pu6GNE$>Cq5=cp4UE_M=Rg#QEj9kaTS+Yo6JeUTQ{R!fpQVpM6?PdhbZ< zOxC2>>My7xc^`-3Su@)OxgJSSf3f$Inf$a@)XKh zGm=Wo+M{C9D6wLC0GWIzqvZu+?&;Q~6K=4&1=rY<&K{&|iQ+u_AB^jGS2~@$CYn;8 zfbkE0=XHJ$9jj-N^?RT!-3*F&V2$a6{$!1^?wI>j2}YhF)8tPUD|ffDN}rJ^Y59#i zojZ$8n=bO%$SLuoc_=1zo`AmT9k4hp5ChlWg&eo8SoTW{DIASp*pMqqx;u!HS0+(9 zZ9F+WV;x&Mh>TA zKLKi-HmQmib`oDEwd3!bYN-Fzhrdt4S!VodGKDXt*VRiYf9q3L*m9IPYk0n)!&TAQ zyEp3hIFsRyJ5x$eGS3*E=P$DXhjJUL{ku$bJKqK+L!NsZgm##*b0s=Ik)ZQ|38X%` z9*xV}@ee$i6k|rvll(bU{%JR}oIil(UR+ByX*ta2bvKGyJe?eTQz5H;3-ne6@%<6M zv!rH2(l{&d_4^7f2ZJc((N>Ujex*{5X#{4yilP@Rz=K0OlB`g}?w2M~6=y(6YGp$7 zh-h@^@&JmToD}oM?qV<3_>;!m215Vg{fl!AV83=X@tXxQ)lR0hzbm-kYzG>pyO`s2 zodtj8^Q?K%*x_ZY2aOhqd@h{Ga~41IovE0q zsvk9xYTZkPdRZP!UKmfhm5!)v*AdD(%mi)Eqhi^myC508-gvCQgY?=pXqm+w(_S5! zU-fm!Kf@U=iwBbGmJH{5^5@B7Kl+}!oFw0F!s`{I@pZsJDjWVQgtbg1^S{19X~uYx zed$Drg9ejaR|I7*%D5*o4EK&*Na@E%gO554U$?~2$yN5`C*Fp=j}Y0s)9}&W1tpJ< zi0N4}s+>KZ&23GgR=dIIu;m)qlq?{Lv-d^4Fp&7Wl59HNXR=H$o@W_^n(e=`uq~gU z+EPJ^-2URWmNrzQ*aDKRt*m~GCrh~1#EK*M%(&(zRQ~f4+#z46e5e6W)k?Cz$8)=X zFBPlHHbLy5b7081$%21WQsU@wG-NK{<@@YnQN6-2c<5SuH?<#m-d;rsZ~LMfXMN=_ zNrl*FacJKCEX)41g8Ltb3);uigz!J+QGK90RE?XDfdRglAF&Jcp+PL=!A7RlpMtF4 zW{{@CVIg?Bg31!RK;U@(*=+F^^ogID?A9py@@5qlEE-Re=i^A)Y{RtSQ$c4pj7e@E z6<^*Sj+X!AvzNPjVN3V1taw8T8eTmY?;q|;`a8Cy9d%C3*twtOfB6NRQll`XqL9VL znbCJXXV-r}4$AN!!2Hb55IMXntN1XPGP?(2b(`7bH}OAKyYw8-bOvFTMuyop$J4m3Q~9itmmn9-L9;VH1YZua z@T(3u*JcrEG;MM6prKSOd=r9SIAP_4b?9a1LnY(ZQHgym=XhGsj?3N{cs3S$XAh;k z1)NETEi8R*cb1x$%^cQS&~EOT+<&euOPvi^cARI0#vKZ#z!4yl*&qhqRfmje2gvy7Jl(sY(LzeRM z(BQ+yz&q_RsY-#@m$k)mr?qIvKLr^jADARM!Q0Y_f43HLo{e=R$0B%=Ue(e!4(n>;Q8w6C};?MEmP<-ye zvmxuWl(qg1bLr9+bqCB%o;Sn9aWc#>F_wO90anid(j0XaB^lR6 z<*Y7Xnv#NL7v8e`IZ>>#?>@*MIvE4geix&U?S{HjyTCjykOEJ4hvNEZRx>{l{pR{G zmyX_)x4sIre|N*W@dw$Lk)1K>-WcAi=M1M_TC}p_KEM-`ardPVs{7W!EEWb+e)C)A z^k6m~E@{T-*x!jeMCy%hwtCOfG zeLOT}q(OQkXSE$D5?U@5F{SGwRLR1sv;4_nM6a^TC* zSs0epl|HszP3it6Q0rdI%CcsI$J;fi+tXq!_=Pi=#5yrOP{I-lGgz>jC;3(wq5k?q zmelne)7nlG{4M9BWSthDOcJmu^B_y#8wAtF&qUqFF`x_`g6c_}1(a?}*B5gC+)@Xq z@KU10{FdtIv+3lvsSp4AuR(LDL3O@@XCNX&!G4S@HcjjUu{+(6^|%7*oVVh0dxh_xPi$kdIq!w~TM=k% zT1wPLH5aTYWd1U6w<;Hl)jJx3+?>Z4>*F#LwT+cLe9+e&}dSqIDgT@w~-aVt>>yX{{H=TPVnPST5Tr zI$-PfaEwhcLW<8))GT@r3e_|*Vca4NTkr>~^4|kqDtqpI>T9gd+0Byc29dm%D`~!` zi3ev$uzWLTDmcFpY{tEVy_eQuR);0nBAtlE3+><}v!}^zXP{H|Dt_)?Lbc~RGpA== zcn-@6;C_E<(#C`JZO$t&jUvYe0o?|~U^;gLxJ>;K^zWWC-RS@bINQo%GriD0SxGV9 zCt{6$0JhA&3q|kTS<$x+ob{?CS)(&^S=p7PG;M>({BrOyb|Hs5JUeZ_itbeLymsak zbj}@xg|BlU=f>aQKCB1xO6rV}1IwXl<7Ut`_fh4ud}ut-7JUOAg5>rqW98%aRINyY zr2p)|=L*pGUwQxV4_|6o%6+7@J?Y)q0MZTZgwONbN%tg~`@V#R~;ib#Ma<8kP@xcg>)hZ~j!P zvlXSz9hl3n?I8LOeitk`CzN|#fuwuS;V5@-fO!!_9_!0I_QX)(ybGM05(=?X?Vz>~ zcVLHGQ^5KoOghgQ+0L(Yu0o~8tL|I%HI)8pS z6_zi=qxIb}Gx9j+ksM@Zg$mlZDU8BKO+kgLn$8z=prk3e;GR4h3hM&Vd4L-}v6zhh zwgbq1uQx^+c)r^Gj2Qj+Gw2fMW6{!V*q@mPneG-?>C%}!Ss#I++HROUF_H5JUx21G z7IY_13;UJhSio$~Rdt<38=ZU!wmG71*eD^F`7!Br-bb_a!t+PnIaj?0`L1t;^ij8< z_M)Q@(*7D}XWQi<^^FCaOKqvvZVu^W zL!n{&JLr96AlBI*28C@aRJ)9V$j2S1vSKAEe>UUYUI$gmk!7R|yu=Jg7mLaYE$6Md z!kvwsNM{*I5uFn;?Wz;$>I;k!*LgxbX*)7J+EDDFYB7Q5ikiIs6W_h*Pal6b z!$t=yEE{(lw8z{8i=}RqbLKlUOPhwFm#y(hhhbFJZZ_sN^Zb!4UkuSE!1?YGq&xN; z!cOmJp}i*XOwCd-zU)OoPaC1B*pgJ76OnR=LFA}`kn5iY+Tu{!xN9O+8)Er+d@S4h z!U0<@bs_f=wrqa_K)|)#Y;UZbQacz}$@zOMboCsBDg2D;HJu`ZLhIYcTaa}s%YihjA$n3UVGiy@q*-F8I#D1#LVyoWQb$pNB0Eudm?4>&xOpn+iD6d z`z)w#_C-VVSFvfOD-Jo#^Aqp4v-;p}n6x<$Dm(x^t$j)QW;FWLEWqYT-jqLb6MOU} z7G?JZ%yYTNGxu^9kwE41nJnuoex66al%p}^oWh~jGoM!9Z^uzEQS}bmAPue;= zO8wOWT`uss(KQAQ?~Xv?69=9t{1+q|!MkX07xMVQ2^E1dk|bX@7RUEsPJ3@NL+nmw z(Kw9r5xm5J=PM{<=@#DM{7KYIOcpa5AG1Z#Jb$`G%k-D$1H0A;lEc{-3$9pVBdx@M z*jr3q;EcVCqNw6$XVPr96oM<1Sd8QW zjP!MXh}q9=Xro6gM(=E5nmrk+joJJ@Z{T^3UR}8#x*jBx|6+kD7r>?{mv{MGp)76z z?B^Yq`iPN~KSqI?$O#m%_yC;TItW|4y=Mvg_k$<@UY4vY0kg?{sJOp1lx&$#_WW7W zwCs*h>KlT!hc^oP-B-nz>F!uNr472i?@Q`gb4j9T9J&9DKjfNLP{=tg>Q+V?!(Q)# zz{GD*|L_h=uTG>8nvVfxxvbztACjI&RK(pDiY+VIoul2T+^>uoXg8D#mss;6ktvIK zZq(3@az|`}RGTc8v(g=#J_PW6cv~i$m4p(d>&1jW=aJX5iPYMCJ}S1~1(%hJS>!Nl z8aE`4j#jlJpM5^`C}$O(&z+9jtW+rZ*zICn;(1o~-ijJ4Ib&)9&&gf+#6CKf)ij?2)*EkTnzm_!Go!~ToGpm^dR9K*)7WeBc)u}t3-vVfVc^MkJPa;X=?@ZEq z2xK0sG3(@Jmi3Ht?k`>uEgTXk+oL~4{vHka9sXs-+lp0X@+8j1`ip5dZWJE%oI>^6 z&+vKP8hGN>hVLI^;LDUjG`Fe^x)(m-vlE_a<6LMKw4LcI{}gn&n?+sB&(OSjJnD1T zG3~8qtRv5?KbkI~77GXRTmG0i$$2j3aR^%TouKc3j+p733q!J&Q)9&hs@pS!LRRu@ zW(a`Ns|Tw*kje%TG6A1p`^S&uF@p;d94BX>Fxt-cUsc{I_*8e5?4;_zA$9T8# zy8@G*b!A09!qc7Glyb1znu3X4W@(4u-P3H6I5ctY6FYB^cWOk67zMRj5)#F zQSfvW3!E|^9nJ<~#nGvl!@K_aoyUc|@4i&MwI8#2vJ`{vyoIKZ_XJ7UE(pz-hZ*n3 zl1r%@&;GrG*luRnU|df{@jO%Ub_``K`v#+{E1Y@*`5tM zj={%<6!IzTN41krKmd2FG~d6(l!JLzbwf|`>cPDYzI#ODt+A*`a3r~3H5)RJa~j0{ zC=^q9ZiME<|;M7iQ>rE^NS7?iFbO$zbk_Na4?7 zhY>v6`xWxDXW)LDzHGlziC=!_y*|%(kmdFb?mr90mZ{sISoKg9)xiy;75{=g&$K@G z;%qY8r?BI|e0sE^9Vrbn!CKjlcMjjNk|-O>8odt9Yc@etw~1sB*9g0(24lV3cqmWo zh(0}oD826vmi8^4;x7f$W$S7D&KOFI{!`+UX3h~jTfZ>>pm8ExkJ8d&&&dc1Enk>b1&P$v&c0|B@|;e8zIt+d!!a&8t>jt zPV(Vg>{(d*d>2Ud`$5h#wt+Wy3NB@%F*96>MK7b!?dKMz->$~!;e)97_H$KeJAvwK z_rS)*!0)jmG41#WOn*0)<-9b5mP%W|kFu z5HC`ym|JOUPE)10wyI8%;JdpOUV7i07Nxx<$vW+QNTgQ3BvU$|e-k&NPdZB;i z49pvK7HZ9|LWN~-DsH`JjQuB#W$owL=;Q&cXvk*HobjRfpHpyh*Kt(RyoovXnMJLk zGsxdz2-SKuGs}{npw77?XJhLjvTQXN8tn=G_NI3S!>QvD8%$`&=izHdP<7b=7H-!C zrG}NjYrtrx=l-0v?Nu!+$78jK*s@_g%h7!2J;*#&+07s->9B_#Ea*Y;%?X&4GZrn5 zcf!Kt+u$_Hj1-C~%*A9!=IT=Q&zy8b8OKw2On?=mofx8hW_9UAdA6QswIg}MF z!sgDWVfVbYlpfm&LXP!fr(X;qP1di>zoakb|7k}O&x_u^25<5_aRSWmcwooe;iR?y z0ZapDlH4K+!n>r9=JYZ_vwRUaCH=+BEaTAML5;3kld)zPKfCxilHL?Z_D2yLe$9l| zLA)E?<}#Sreki*sWPqtlKDN_mLCa_zg5^y(w(OFm%7$ z9sEuxNoVzw=y~V_B*-{7Zu?^N{p$~qo?3@tFL-aQQo;1S&a;l?wnt6tPMdVwS_z~~0ML1sRdaxt8=-ye$Md}rG-Gan50 z`#>}AHb`9CvL|<9sbsl=Rpm&p};OT(1RM%q`nVq$x zDlhKNYBL9=-&L4#Z7~1ro-)NfgV-vb1j|IuGO@ZV$ZDn36#olzZ*xUx`t7bbu88w7 zI{nMt(wvPG?8|(&48rT#rA2QhVDs^=D4QNiIgZ>b@rZj}W(&NJ zxDXrLbfzNNQqY-(870R$d0&qo#knpM@y;ZBn#Q{dlKtD6HcciPT>S(?A5pX`bflUJ z9}H^YtfIRMAg|k4Oga<76jq%mCABTq^BqWF^+Um8mpP`&T~IN)pCI$JptkinQ^aV&-OzzeE1FM77xhMqIRlVQ`W3SHJI-z~?=Y?S zmre8WLv3DxP&{dsnD%P`21JaboNmwH$2I-bnWv=dqDm3|m`=o+*xZeV`bInziw zM)a@sqL#1Q;Bs{|W<4f2%Acbl+5Em6(u3*St%5v1JCc2W#`4Z}MANNdsN6Jw$-ZqD z!g_9j@^kzh7e5D;Rh>cc6YuW-+ak!F<5>oGxa_|dLgwYkWVdZHC8@Q{X^1ZdesE^7 zX_v&16@4kG$dR@92~68t#ks~QOyk-PqGAVOlm9!2)T>xidX1Rnb*M1~h zwhLrdZ$zF!hQz5uFfwWYWtd%OvEFUSaqc9ZDVsv!Jr?2h6ibqqR{0Ij_Q7?m>ON9@IC!Z8NzP3%n*Z&I4lnv* z=G@U3;mNtmCQ|*wyRdF(8TW+-)?RYVV%`X~k_=^uz|G-H(A!#cZk# z9W17&JCc>{Ld?L47^KdJ__^T}@@gq$M~@}Hd23N)mj}A65@Yhq2+Gk#lJ;tWn0}9Q zNHy+|GpG&KPAh@3bGyNHhBv`Z71kUJ;yd@v+;7hNS1os;Hp5LQp4tT#g)c$<))k;_ zvxU_zjAvoT7}F-qU~2PGr2KOuu#;XGdzU-VesRF6P#v55-i_pYJke&34ircHF?JZ= zqyAjNG|J1M+cAx4E^z1FB|D5c#^(e-a>tM5RA%3E6cwLYD;i!XA&YmZw13}V`D;33 zdKKUI%!;5d>*u4JCXc0d8bz|phnU?N&eJn*Q5B!M$dt*W@Q#liroyaGZcdPm?r7X)eSf@G#nmIC|5Is%nF`(pYj zDHINU-eI_(*`yx2`P9UY7l+Os?tLlZ}@Tk2r$#eOAe#=Vkn(+gJ zs|^&bp2_mL^K-xXI&zU$g6B=1QFYlS#(v2VEiNs>jOt-n_P&t~*`9*=ca~s^qM1Lp z29Tm<0%xT0{MU>#aA#^4YOD@HL+clobWe)f`tCyQ)HW<=NpGy%S6IgS> zduXWs4w?ME*8Hamm1oydv%v&)+Py4rOb}?C`hnu>79sLm5Q}Nwk@E`@p?1I<@k@9c zI^V~Pbc6&21W=zB8Zk6D3cOK@(m4mh;v6LH?V0I`1^UnVap}y_t^)@RqmMq|1yICY1 z`W@WYzGL!Ff5D>eld0rDICsNLLCMR#-q9awL3{0%z?ndT+otzWwqg?7|LcFuCG$2* zvWTO=rXr9{=uF8A`F@{qE^K*s*p|Eu8z;F_^RE{;H~cSe#XpOo$=IJHaOGn5x^N6jue>zZ3$sUk}?PR)^m&V{`&e&e_ z8?)eSu%w-9v8rx5Y5L|0iqHHVXv=#n7yn|5Dmhy)Uc<0y@YI=G@_-HYZca5h~@rTPsLs}1>?diiBGRh$AaSuwmaE=*M?dkb} zSyXt>3tJYh#}Z{S1t$(8rS*46DtgOWyPsj&^ie|eEbc_CGsnOOPQ0`8BPfFAGu@Vz zMtAWL_yt!ojdzn!A19@(@>EC=mXbSE{=wL&s_x3~`(XyK9j=c~Lm*{UBu5j3tA|T+&))i%l0SC_P8Pit0{+JlY%NauaKwpI$BznS}Cy70^f_Yjg^$Vtrrh3*6^YWE&0e_RSN^}I)av=8MBJ_YGo zb0|J{P~~t|P5U#vsP^M!(eJvE<-W5e_W{!(e8pfaJfJ0Z!=F;CY9K$nKbG$eLY@3N zD+yhX!9fZN$^F12?YffNZ#?VU_MxCC{lrpN`=M@fqDuO)9op+SOU?WSJE-CNuYHkF zH2E=;9?4_7Cwo)vZ@FUa+*g7^*B%-wa#$qKALw7-gkxS)(2~FF?Vofb1@p(<9yBCnqY!{2=&f3PQ(O1Xn9 zH32pIcB(9;CwTYzfvD*x;e4tXD1Uj1?M|0dAn!qjbT|PmF40su^@Avj7{X%8lyqlp z8!}J+%2L`6#i)~A$p4-XxfJrdlV@LaInQ^8ud_wZK36#7Bc5rCmI(R3@tH^8nMU`o z>$oT6Fi7GSilvhsu<5iFl`8{zF8Ty${C^jvk{?;I?FZ(bABKLM)!sBPP0(4qGp1(k zhTW6mQQu~tVD7=^JZ0TPL(^vAWB>7}a~;R;G=7x6e+p-qWT@8p@%`KTp5!*Qf`yMu zNt(Mh|BeRu4Rf#Gv0_xCi2Y-mHJQvxV(3->{u4raA&qR~CqnX#`ar0AM0p3EIY zCHH+vViSaA=62}PPDPr(hp`5|94ieYSn8Enbm4iW%6)D$uG9%@pN&K9*HX|J--A=z zF;rFZGi?0C-O$}t4JS+M28+&5rkA;I> zscOYV@V8wOxI?Q z&~)zyQR93Bf{IeX`3>L6%BN6SuQ^N~e_tr+*bVD<^~TE8SHy9B#-b#G8TVg#0VRLc zGS9L%tmT)nQ2f5Z==X;^i&-%pkHvdY5})y2-^btGW#3pveHUu@G=;QATZmj>V4)d& z=IXl=RKAlaCw?Z?uj733A9;r2@(6NuFvsE(wwSOikqU0EA-gwL=y%l{q@4@EP*=?V zTfR;{Gub1P8yWW6fzcm#6`D+BjX zZ;FcTgZiO=gEVaj-(^YRV5&Ds%Im#l!;FHS-zyt$_v4)FHPrAjhcnYhqtwz39lFKP zxKCY3_tGSOEb4%Bn>r(WY=hR)7%cs*D{7VdM3;pZq4>?OVx++xoHW%;7V}!v^*pGm z`8b|;FgP>lSMGS5GZiaio0#O&2(d|f0P@DgQ$pk^<~(H-6%Fmnl*fK!(&hrTEHt=$iofA5JlpT9te&s1#kolFh2JTG4K z`$da&EAjQue76^s2Zm3##eCN;JmY-MJF8(ID_E$YWbR=sUiMj)JMb+8PIIN!UpY%- znh!;!kD-@cgQ+Bh=T2hIfoE7X)1;Le8-`bas};ZBCy%H2i8D#jfoEI=M=>awZbZe1}va@GgIL_vin=jz{qQuJWRM+%Ty9!5Q@HMu7RPJa9hYMeY$J=C|s42gHe^nOk7 zq{lqGuJ*!h%Xl_!CXma16Qt~~2gg4Z)Z{e+V)=aD@k%dJ*ky^OE!;6ML&Nk(er7)Z zJL`9<1|@bQ-C#GtFzW}{c4Riie)}X!+kb-lUVd2IyhN1O*21`mWKunAhjkgAST@v& zl?4T25qIV8-M|@se_9IGmz0<{azkfGO9s!&<8sP=Y1{j2+sSvHg+hYaG` z@IL62c9|XhVImc+9RX$q-uQ9pLbSS)j5d;9sIlG#8?)PCUi;s;FY^#&N-U|a;tuHN z@n_~A8AhArvoLuOXA_nkW(W1-afiJbDJNZK9-Wd<24$6@j4kG#h4ycl;>03Qz2tk$Pwk+({Zl6KIID83SkL<{TUj*sNW7jSM|a(xJXmxq(Ge+QEF8Ay_kOTBe_y~Om0#c;>66DpqWgPc#7_*p!Ts-|+c>Wu#c z!(S$*o5LB#b9~YBdJ>wiDq)eocVwq+rjcF~O}aI0RC3$4O!BOoG4*{MseOB5-Li8m zvvnC3yE#E-;2|czw+RaWYh;e&$04Xcfa59713S%_U4RpP*%^Xq1MM;NKN(KD5=8Yr z$06a!I4awG6x3sF(P_k6w(b5*lErxPF8vx7b%y6B70;Mq?Eq#zZV<-XOre%56PYIe zrK&8+fy&Yq%wm`wd6rM5BK=wT-Y$q5mQUk+^tohid6Buisb-aNqZrd~WbT(%a2Icr zsAF*|lSx8Ox;-FY(t!$dcd_8iwbb;YA4HBx2hY-Jv|lI($K%7v5a=e>e9MI7X||}S zpARK}-iFxECJ1ilie3>s%l5M)z5bQ6=-)ZueZN^4;nRV8uvS6Mp@)!aHUm9u_#VR{ zfGA88JdD^nAhQD2tlHyw+SyW(@>7&{1ON%(uxD&)Ut>T}f8&kB&5Z%+hGq`q(G51cCYwJss z^`ThwwI52}tQHh6wWN&Uvzg2=-k0r3tnD+XNgjYY=W@_J+rabl!6aX34zE`a;5~$4 zP&n`@9OHMZ`=5c_|Ka_G*hG>Pj}>bE9!-h0(RinnbD9(7EEQ&8b83%7!7{CLoR>%|%xW>APzFblkQ z9HzY*j?oeBG;Y**jBK4r{BwqAziP;B+QHcUli)rBc-P1kQrfM>);|t2_uGK2_1p>E z?f__8XAAFc%P4*|&w?1%g8hYftRDRv=q$g8`Q7*oBsvZ1_jA^#MT)_ zl_x}w-T@6)+tB_EJt2SkTgdJD8V*iUQ|v8&Huq2dj<@^booD+OB(3ExEzXL|Yk!bQ z9}kA!E8Eg*&O9pWF{7Z4Tds4!0$r&fGiW@D{3qZbGx|6fB!D2DK0V7JCQ# zljrZDv_E(}_Y@w1*g)QaO7G(BKXw#ab+kpv)}<8qfp_hebfKId3b+@bz2Fznhdm1H zOp;~Y1am7BtGUQ~eeIDful)f5NA9t?eAnYtG>sH%&4hxDUCBZ>ADcI}qv*ODa5^y( z%bXs9I$#(j9!w_REEjZ(ZiUJR5g2PV3QR%PsGU2GJAi^%mf0Ri{k@dsW?lx_tz5Xi zgrDP&HVX+p#jxYe43huUhx}qzvXC+#(ib#Aky%$PI{KBx=8on&vH7UfcyK=D7s&d? zv&4>R#@MPef}ibKxI9syfWdrz9`1%=6Jt?V&AU>+@l5H+!6^NA59Y zka6wP$27vFvqwLX0Te|!o;1H`BD4D*1o$oHApspKP>VZZmelSkdSPuke{{*zO zdI5oZM=;&>=c;5Jk57Pqzrqi}kfb&EI#^1t?`rvd`;ZVR*~9WiEJhen%8C?8O!{yy z1SRa{ocYt>CVR>ZUJYUq&x%V&s=;oVGc`>;0pS)iN!uY%lv+f9U*mVM85K@8&<8JX z?}`RzbJ1n<1ZD^wDb$=ygS@Mapw$ee*z13=@}w&)qU{DKw0sFsH~79hp#_|DGsvlY z2G((Am8^3b7}oLgs*gg@_gb+e#N!{n{Y+9Svm$ad6!yts#CM~hCpXd9p z`264(*VWb4{pER|^Z1G{4l(Qjp0AlRpPiOkvD`76QJemIW!{xxT08zQVawOYA7?7Bu-Y72@=t#G;CwLi^$z;rc7i9Z6^M zp2i5V?ujek*KHLoo1JmbN+h}C>PEF=t{&GxPBi<lX3ivFvcO`4S&-18% z_$s8=`0yRj+fy@aU1eFndPRMHX_e*?&l_K+5xe7-q3pW17~7_Yjvf~5`Vt!^dw4-G zKUXTW`S%hVuU-{OheTjONrvdaJ&($m6{2piT6~){27SX*F(r1e*iaCL$uVPDM9pOG z|JR_?yI;8%!IXDu9mmY9F)U%X6oRc3P_uF&l-93=$aw~&m92pThfGO*CrG&1zYi&o zoD^~^w}6_T)jer%n5^L_hUBJEm})5Zei%|~Tmi=fy+Iar7A^PRVF7~+_|B2dE~Xh! zTz(sq%SM66q9EQk_FmY0djnh)(jae#4_SC3k43H(NT<0g7!Q$B{rMC!T6}@`(HoJ0hdCu} z&?m{doXUiO>!{^_NpxbnA$04TLEg*>&^ops#mMYf#py4s_}F_U42%Jp^9?}~n(FE0 z7E7I}TcG^r5YYO~r5A?jWVgNqwF@Q*dA}NwcfVTHj$8;%gYNU4XPb~?@F$aK+{CW% zEfnW$NiHJK{+j)osoE-8ScM-+9)alc!&1`SxqvbG{8_FV4>^10faQ{PlrsMsdNg|gvn$WpazG|UZm0cDBe7>m0e7P#ey?pjS{O<6*7 zZ$olO4#CFC9#-CYi>Z_xw>i_v9R4vRzeAhJGtLuaJ93%P^?VeD&x7`^lUO@>KKQ-c z&6;{=W3Ej;%Q^H?^glBdTIyPH&nri=Yd(wJY0*?Xz>1Qm%OPyuMP_8Mh8>+W7mf~` zO$j6RF{gn`NxS|T3;w*E{PVX!P7TMVeFl-yp=Hd|(+;9mNl5r`f@!C0qWGugqzTh1 zN-AD(T)G_%H_e5*!9gU?f6ioP?kw3`N;Mz-p|iw-bnYsM890_+KeK`QYyS%B*}kOh zEf;IczF_hEFv@t)i4kdPDt%-M@+$|pU$UsG#&8xT-F5)eiC$EGq8W4STv*A7jg)jW zmST+ufXDL*OxrdFgvU$4sosbRobR!SCzcc&r3ah8T0_y#@q$=5kgVs{9irOG4_MmXPnc0Q{@{ z!8~v!#XPj92;V5`_-73?PoGY0`30yN8_v1PDPXW8g{0nFnP|40!mgHbuYD-AsDcm5{%A5S17jVa|oCXgGa32%QFW-`1JDxBQOTw;nT9;bFn-d;W}$ z_#`OxIcJ`jN+LaG#fzV#<>F*4v)usJESStn*Tc;l*_Z?|ByGNkHOCGyf29klX#hwj z_@d-QKJP^5nWU$CgJXUKxV6ivzBCrEJ1zr#Q!A3|OF^}FEHnRO5Bq#@1eNvY&uVBe zXy0!nnaMhkJ()=*`gbww&Jt>0zL;u@mt$G>LELAV3VYT^QCsgQrX2mHs>yOY$0PKi z{?kGhM=H|T*(uH~90h)T9LewTV5Yk{6U%%Spv*Z#)Nmimj)`HQ42@upiw1#o&2yn_ z!4$Hni>A#cL#f@f2p#mQSDRyAGWo%G>QZ(;+=kGI4QLw9g>T}5F|rs(=mAe|YC#{AI4!pUh-)`(tB58|&J%ijKAo2kUQZS@&l*>KJzr^L}B_a)IYg zp7AC5I)BzQvW7*i>tRNZITopl0jmii6mi9Y)Vr4Pe3mZEiQa*-d*6z(-gQDsz7yNA zA&4~d*C_&aE`}HSOF^=hd&UYZFrhE+7ByQ-rEhAuW@r;DyUgeEj)P43IF%A+OF`K` z9_-4b;5Z-_nsUCdM<-373BzM>|77xkP0+^@t8@UJQvM()nP(`87twlGnJC&~*oU8!ITR2ms^dBdIcpkvycE$@N|fu6`C|KIt4b#_q>QGsZ%N_z-20GcoLi z8--YVLsD7*NoGD^9d(nLugw$+Jf#nXzZg=RbqwdmysqZCCeyaZGbyiXH|7kV1FFJl zAep*e%;9rtPV-(N2A`q1jT2}-DHJ;2f6!yyWbWB{z`|=+laszB#BHBQl2M1N-2Rvg z-5yy?YrwfNzg0Y!t}iQ5yK|4uV+^y}$+eYzD0BTDR<`On@3CEg9=EPDodNGQtMV1p zgID0`Ni$%dV<4os@SNs*ub9Sr6nY^%U0a^EcF*ofthUxvlBuc$XLu6#c=koF) zVc?Z11EW=Isb$YS*64njReS-a?inoR4C4LRcI%*MpFLX~Gne?~qV5aUDCxagR9fl8 z(0fJ@c6A2it#)GBV;h)BUwt-z>3fQIn-lrw%In5>9^#omj^V(o0uTE~FfFB?Gz zRp^jDgtTk#^X!CNw3GXh*mxXW=0!q)OE>FKe2WqP z2rfczaPbVIb3ZSJ_WWW@pZknSl{ba3o2e9W`6j#R^BKQBkyFbJ6DabZCt9lQ8PzU> zl7MPV|5(Jzww-1!-tLregL^mslCb2;*>qVph?LWxi1lMHF^l98ka{wSG2m`WxOe#1tejdZ|!9XNd4KvoAs!F%N_2zTB>20PVYkkbb`&rTxQp?2)J zVM1-AJV^OlikSXFN=4ghSOEruX)k|r%NYVU+c%ST$~LAN)|ZtnPGu<;TvJ%MO%z_} zf#vp9WbV?fN90{;FX$X?9@pV=D$s8%2_910l!V3+kQk@qGE8v298-OWov7Vd+!Q@0SLq z+s)6&x;P=w(GM;TnN3C0x|w=bUy=-rQgoa#=iL*3iRSiS(fqMMt(7U{X%$4~6Z?SE zJ{OWn&!E~i1Uh`bV*v)yu*WV5C|nOdn_I!+W|OJ{o|^C8N{n8LcpF!}Uyv1PzQh&i%@HGeaW+yV!|xlz-gxYw(Z|QNdCWU;!$$3{$m;%O^?Ldus<+5!fAakQw~p4oQorQ%g?- zG|n6Z;`suuX}MIH_|g}|$4-#=!GelB0$E7(1d`8|L)(A7L7G!9NH)y^Nz8O6{p&n- zfA7OP7yPO8d<#?mHeG1>FPkZ?Oh`M(o=Mo@N}Kp)ASoUIxe<#XO1_b@qmH5SjJxQ4 zG#=Dr#X{{$q6e z!xiY^T994t%M5xtxJKm%!A^A-tq#wJlxr0{hxczGq2jU-@@pF9nHOS%i$3TgI3Kur z42z7?C#N|J^)0 z<+TkfY>0-Iqr@8e`BC==B`WP|covHmW4;>?>XUo0^~i4Sp*#z8H8Xbdeq4t~qIIcMzj2m4*88Zm7E0>bQZYDhav=VeD&9Sx@$9hK{!24JF zk-SsQidXQnHqwwKU3r44@xO|GdVQIy%m9k`p5MVe3OSy<<8r2g+Byd^%Y$JMcLku; z-ij&`I9GDY8*(@I2Kn7Yu-xN8*N@6b!_pO}wev}OpjA*F(L?#ZD?-H+W2SvQjxq;D zk;$IJsHs(h^mGA>H8dvk$673l*e70G5l6LUW~AE_jO`)6F#};6*4M3Ird!uR`R-w0 zbnRQ-L(t5U#tejl$#2oC$dR;axlq2L1uwR(1o_Sff<@FC@~gVf`}odcQ@a6rWt&my zHv>tx^MIhNzb|MK_g59)&0#iwg;RH$J4-0c#?*@;kh}6Q_ikLmj>yg2i(y=u;8)34 zqXbfBq;d^MtKhNIo9pN&lg?Smq{oIq$g!Cq{lI;9rYRs%+!6O}9uC>Pl^{8IwDR1s z(NyyB6z1%ii3VF%P~4?|SDoIn8!u>tz zpucN22**C+XP2R1XL<@VPB>85Wsx=99!bg@*A-1CZO~&(GE;rKj)nec4}K?RLA&__ zwkL4_bzD@VLr`BPOh3*FpHHUH-p=5?a}`LY?Wnx|qYoHMox$nu1JtHwaZO|{Gn-@& zEtrjU&lXa^`FiHqvX1WygW>HMN9g2v;F?R3XgJjfRMoYD>e&un*i*%n>-wVHAP{YT zTMsFtBw#vzGR36#XBsP0CR;e0@6cbdpePo+u1p}Ov_%m5BoZpN)Uo#Q&zQ!fQIJfU zOtlr!V3)X%a^By;`sOc!!Kf`H3*IJlUT_9;>kL+EU5%QqokIQ?1A05X9~4ddNz5Gk z2z?9P$l&*Sp7)nkbsYmpIro-O`XB}Br4NMLAI4HlA18j!EQZ?1)p&c6isUYDnCjo} z1oQA{(uQ3UGi$wAg3V^8I<%7+jSHkbn^P!^=XGiWeig%l3Nbu=6m|ZxncD6gXX?st z#d4i7b%#}B&x!pgbvws8*)m8<7zai!7Fh4l!K{C-;9V^Xn87+k?Ugj#ci4v#c2|k% zck`IaKb>{W8An$BN^mu~%nV$|kol{#X!u^AwC_UkgM0{B|GSZ-SFVYgS-&Veq;IkA zP&lM4iDzLq=Rx$eJ`iTF!Ll!=XhsiHeya zW!Bt9+uoDGJSPprh7c+-wyq*=EK80UbEF?=Kf7ZEt61ZOS zBF*b&=9;vFHRW8y98YWD8L6ao-phR3H-T=*I+QF=Wfqe{q4sJr-_H)Bt3wJkOr1+k z`xBvar;6-W9Y?F_vqAI7RMafbWs!euq>j73{LGM1X+i-CQ$r|jMJv;EA7(vECqd|l zJ`^4?l1eUjvZI?8gY}f>xH)tz(ui=oN8%45f7kQRM3KZ>} z1oc+IJezqT*9lCa!ZR~KGyEgVJpYDczsn)heK9-DdolK{v<4xfhROdR?sJqweU&Z5 z9Qqe!hPGmgw-?(dIs(izr{*VVbf7T~WKMHgc4G^+Y<-Tw55|$^Q%9BL!&9c*Kaiqa z^O;NAOgeGZo4P`VgKTUlwL0;=-FG*dPh1Z6J2q3!taGgUI`0M8x&br;`=iMp+fiLK zmdW=_!km_7!S1XN6q!#$%U*V%$-67cVt1gle-zyI^dY0UyniEmHbGXm%c?HG~8$qrnu^=RMvWxQukT$zsk@9FPXnVG>qO%>M zN&7!cxp6W^jdrKpOXHw<*$A-sJB&JK$B_49XHau4-+b`^(u7tiOkg&-JRCvQRx&d8 z><3c3AQml+C5vJ1kk!{3{0hQJb8HCrmjsZB-);1J@CSw_#ggY6PmuVH6>g7-2Je3a zN-^L)PaU2R6I#ub?6BxR!W`Orze7QO1C`uctsR!Z;v6=R`u-K6$nbliaLP)mFJ~;G z_bcYB-9X~xTPVfN==(;GR8j`7Z?7lEUy>;=+6{6iW}!AXo7EKZ9*MRKRmwT{6{SWy zS;paDs4TIjxWTWn>-lyT)~sQ-hqWb}c|jpF3jBv1Jt55CyhR9I4~G zKg7l$eR5s@mV4p1i;b}lS>po}(0TtO+Vqm!GPdgLgGIsA_M>Xw1Wy53N`BnulKma@j<-qi6T zj#?95@OO)2fJJ7|qMkq@(siIo*&#?u;yi;hhEwO+`J@>%PEkK^BxydrRml785>x)! zice#CcShAKwDXE0o1rU7>hz0f7qNv*wBNC$Yj;t{aZBMo$K`iUfebH0p0f^O>8yho ztaF2`i4owdzmzIM!qBME0*h8hGG%ThR(8rs`pYE9tG|aOW$ReX?)lX5^AM8IM@2=d zCDU#6U}0AcpzM5=kkT<9!Y?{fg7ha*)ced-@j)ym<~z(kI*ZJ1$tk#L2pM$#fZKL* z4CM1mj2&-6nOAyIbjkpVHL{_O6y7D!H4v|#TTfr3+@Se)pti+7qBv$clN$Z_1p~*_lVDJ z{~TaOH@2W=TMPxbcXMsCkN9+(K!zrIurDqNycUfHb@l}GN?HX^B}%%vqM3Qj^kMQY zLy)9ed1`)JTNSxGlFAm=ike-#H^F!tdaaA2ucLgx@6#3Zd$JV8&lfP8XX0l&HQ=6p ztI7FsA_dA4L3^hc>E=f<#}}iZ?0zGfUbQ3Xzjs;D)?Px+-UDLZ@;J(vxe-)BXJ+NG7sU6aAIBmOR6}XZM0WdKBBWf+M1vjQp|Io&wrf^WZ15IH zcYTgURvcqoP>q=t3{}d>*!Zdt)iyU#eT(a6$MSPyPy>E`FoT*0r$gYZWu%V&1!Eph zf@=MlP&j`Agiag>{>H08)$6^e^mY{!-t!E~Sy2>tbuk?H-&|;V{*g)A(-mfCd?9YS zDH$wWOec2mx&4RTp0WkUh19pp!Sc;ivMF8*(miLzkmapdlf9g5`8*+ibPsRt8bqnj zgFt_@B}n@&V~sEK*yg6SR6BbKb_E-?_{eH;(z~P`?bP4N1WuUy> zEnKeAgX-N2dFRnskRIaQ>N{kVx9Skqp4y9=kyC`rp+o5UKE7v-x{U$5$vb{xB3iIF6!@E!LpE(=Ft$+en%`&aE~sqvB@~q*S{ghw)j= znLDVHBnvWkg-|@@2J=cy1KmRx@*FmdG8WoUh1~?wnkSI8)k~~(wxP%ooKv||D@IH{ z%5wf102TEjYg^lZQN_nmA}M1#Du$9cDTO?)1w-&iDV*CXC)L$cV)>>gEXVn_=wX=5 zqW&C0F8ha5j`=aPHQo#vrkkN=>vfiKAs=05cd^v^m1Ozt461!xiQUl6w1JJ{iT$47 z%==d)R^NMmxM2+SJj0@B=pD`>KfKx<7=&$6>+yqR9T~w7tYaY0H~R4)=1gqDqnjku z@$WmXU+co7p9fRSkC(CO`){#3vjt1m4rDU#r$Wp5{XC;%28$v zQ>ZC#5zE#8i)poc(Y#R&n$TmYj+!WHU(Tn5Y1g#MTi3R57O<%jUSC^5!jt&S$p}a_unBHW8SPmQg}r3zN2< zXL+{wn7o*K9o;<@Wq(}}I=(K)o`LJ2Z188HCU_$_RQ`=dpQ}-S7oRzn{KYN~w1=GG zK`iI@%VO>+J@Wr+5amp~z>1oWiQ1>#!nO;RRQByXCU7nw!Cw!Oz6_<3hhN#bd8?@1 z{yg`PCRVl_tY%p|H&Oa`izy}L5xT5i4*9G6$sC7z;_-*SO7pHF;LY zLS7i(1D0ls{;?`bIUm73Z1jPopXb7d1qv#w+(==yq0qh27M%?HLBgc5Z1Ki)3cg_g z_6w{?=eJd~T0I=BHm`xQpf7^#qz~Andy_mk8S+y{@cSRi97>n7gfFkq^Rb+AmK;RC zGn1&~?h95oBo^BAW`H-10;yjd^ZnHYs?}S_^k;w4=}i+Q?Q<|s?~v3fR#18l=CQl5UVMXgr!(PHulH=h&H` zyT)vLnL3FZZZ?~-yCO5MIo&oBf8d7$(yb0gy9Ct z9Kz?><8gFu9@hf6hC`0eMHczrde9s(68w&PgH&TeP156>E189_d#|EeYbD%%X-_Hc z&8XYAT994i&j1*+hX0KxkJs}^Q}9Z#T;9YSVpoy$Ryx(c3!|pANi5=BBv=pl86!Ba z?J*++lP-LRs%RaPH21-Py1Puf)I_kjB9M*5jvo03@h+R$U^QwLrQD0c_Ws$-a`SfP z>eGekZojc|r#?{o$&O4aDws<1iKR#X$h1eNLC4c9u{`?%8p?b?uJ?!Vd2$jJ^Y@51 zm4GtO7b^;Fn9lkx_s(w+;{S5u{CNd7ZCixu1+%Ga{a%dWI%$LuC|0Cgd134DNeQqbM-bh9Ky>!z3c2~%_aGqJ8?_|8R5o*pNbG$K&YJZ*zCE!Roz3Q>Z zgI||BpM>ZW<`gr3Bc|`ZjP?7>pu1rpMR%w{?YB>;f7&U!Zhp=#ZZxKNw;^QYBct4B zE?^tDfo|&!q2iw}aYla)Nz#8~u||A$P+k#=k4Yf>-WsTA7Erb$n%u$$L(WE5lB7gW zX$rf;oOiAx-&_8W_eU@AexOH6_YGoW+%tCh^LjFS6-QCgpRm%>m>egEfwlZ+X4E$j zn#%R@ktz=It~D`BHWVCg`taOjDSjC52G@gZLH6Tq%ouqc<@48)|8S0(S`Nj}v3#~V zw~sk=@xAjY#{+||3-Yax&~Lf~i+c_*$;s=UJBEcpduI^mQVq#FX&Pwy^%Em@`#_4K z6FutmnDpFDQT6q2F?om?wO^D#+1Zn*d-Gn%Rhv?R`w5gi7lks*WY*!-o4U`h!P1rA zvzvU$lE(?GVB{b=aeg&ezqp8v|8$_O*DB~b{ua&Ft|D2{1MZ0m#mqW$fXZ-E&X6+Q zz@38b{XijFI*2?9O0exB*Hm!hs>RuYrc*r!$?Q;e%a-Bo?!HFP$ z^NnEJv5M+{iD5OrHDaS`6Yo^4MNdAn-5m2X3+Q)|xlObHh>xe_pTZ%pO^3>hX+l8D zbdYX8!h2HJk~(mVSe&#N{I*TRl#Cfj5BYg>#~bstD$u<-A+#U=5v>PBaIQ%SrHf7I z{yjZ1c+$n(w0cmJ7)w#_hm(BuRc0rN0!hiZDGt#_Y{!|6klV)vV*ZJMWZtnBH?IPv zmepdqzaQ_mTu>EJ>kbDNOUQ4@R%}|mi`A74rI14lXrI;vWSJ*fjIbNqcg=$8%E1)a z$@hPAchsC}5clPn@UG5*qGb6&mc8{A%Zl?O%b!i?a^ZYPx@ASmrPW4C}PgkoU2t++shbmNHwK; zgFTorfbW{a<0!aI0rn5n;M?jzO0&GGt^z}-PY*%4^>n7(5`YD(7eNx&SZEJ9U~1D` z3YZxRDJC7fQ&3IY)(-%AZ!^?vQL_3GuI%d#j_z z26Rrl95gc>#iwJY@UzDX{Q6sh@&IFP>xc2qvWuvmGmJhQ96$w`c2G0mJTob@0?Ge* z@fV617gfMg_NPJd%i)x?H50pAZ1Lv)U{F4^PhS~;Zby>n{of~)OP9c9j;ATtlqj;rUom0OF0^x- z2^Obg_GI8+637mj1XPBXZD#2QYI^o7sSypv1F6Y`$6VP0@7Xa@gUB_29~`v%T} zn*H_Ys252cmyck4Z!=0+(!)~z^#|`XcM9*ZB)7_e;2GmZx-C4X%Uy;!@A|{()t*q) z+lXA}yUk@d_5OxU(xELqC=h_%U(yG@_!`&6)$x|3)W_7h7n7>_1ia)`Qg6)V2pz~KK) zpdZAKoWYa_4k-WM@hyx4vM9G=#dVlG%aFBPgZm4qDFK$vnEaKaFEdZnJ<& zuCPOfwaX2qIELI2GaIn%zt`h_EjHTpVYFO3R~z}@(9bz z%9z%DAi5fS!Issp*wMYqc;B6~qGQ?)!EWUwkVrIB|9}1~KH13p?CeSQYnvd;w1PJ0 z-k=^ogK8RPQpEi{)a_S`WgAArwuEFbdpVH810?iz%`&R__7dy%8pm>aeP*%oMj-Fk zDP(*|Cy#YHRFjdgr=|~eKRk;`Po_ZP(>Uq$bQm?84u*~o{!B zV`YDy7cL$jLM#3xCp)JTgZibZJ1mQi#p*U24l!`hfq=J~@c@^j(W z#eJ&a{WK9u*FR?Fs&BC8w-#)j`;L{{AK`hPt|Tj%EXWE>#SWugqC;>ilh2ZXi{$_a z96KNOSWTckqm^*~&wiA5;Vnx%5eey+O!(YX%VHlcC0_18^+$N;#viYksXC2}HkzSC zb6#vRnTbsUU$K%MfuPY}DJnY$K~u*$mcaGYt~sSBJ2w>MHbtU(%LEk8rhtB~ANfwu z2kkvCRzqyvLlMq?%mh`Yk5C!p?}J5zpH09IeaslI{hAYOl}%D=UY z#EcFmiJT}j6?$N*gE2JCJi&W&^H{})DX5j*RFr=IhWEt=;=UIgujq5tQ|@?@g?q?I zTV_FdC4peRbPGt%4WRn^-^I>;8$t6T1f2f-7VUgXn9N8YO1e5(jMpsIbMzjTQ8KID z#5JEjJwjP>Fy>F3NG{(j0^{b{;J~vY%6V6Mk0Kvm*K8#JWE0?-ag=jrIG8Vb##H9b zLWt*9Y`*RZ5ypue1C2s?H?h+}L!hP73Oa5VqpsIrQT-u_S$^VLwt1Wv%Cn+|GipkR z=|%PPzlfpVjDYST!!SMe9&UEpLS<*-AXUkz{LBV2fGuP`;ZKYiyNpaGKESf1_3ZO! zt_MG0PoF*c{yJy?J6FuRCL3Z{@J+7wnm+*@l$|Ugi}xEO#8dN)VUYDGj3iI26nRA{ zw0P(S($si@vZ0+NsQB5s^;M;PNH`TYKEStp9;n@Zg{kKoiy@MqSpFJ&sROp=+Z9%@eipqQA4rnhQm7tg2Yx@VVG%3cp=n|QYs=k0k9zdUc%o_$3Rh^#gPAC zHb~cx!`wzasIxI4$&aI})C=c{`iWdq+c=goH794m}m%^{VI@NXcHynr>?4mGVTZE`c%i?SZm%N?4n-`WVITD#o&p2 zo;F3vw)LXH`1`DNe~GmAD_P1k)AKso&`?`$d;WgF7aW{5W#Gd?p~P9)ueM$zxj zRIJFm#lLT*n71tfwKq3|q`pijv*|~5s|Qe#TLJ$b%_py=GfCqLRocmCFm=!*&gJQt z>o5HOd{0&-`!A%($y}rPV-$tY91Wo`07@Rmg0o&KoHN-%&Y5%R+@vuOHr@zT?+2l} zpdV$;ok%a-$B=|$b5CxuNY6r@UB$VBErM3YrENQ+B zLNY%y-QFJrn{cf_1X=dO%t{-{WnL|$LaaQ)tebIIR=fj#efOObzLigR( zRNQrsm2KL^ya$`oo*e>7bRU^|Vzn5mbfwh%WJ;)>4IcJUsC>STUyI$S%zP%g$~ZRM zaaNQpx$YVBHV1PydV=b^l@t}Ql9aU)vF(3eWG0c5Jj)kkCr8ukbT=~oJ&1&w-sCa6 zftB6<$|OepJS(ru$cXEhRZ~YZZBIIV?Hmjqb4=0jWg2~M^QO0(`tqse5;|{~1YysW zSmElzq<o zGLdt{OZze7XDM)Vn1tGl12HFTJF`rvVP@ZLgs$(S$b;vz)pWT)RMwl& zx1#D{jQ}$fD8w+03}fP{EnqRp6J5wW-yZUU=U_swpHPCYMHe?u>M%+rgWxb~;d;Hc zC#y&@#Fo7MOdymdkVBt^%xb3*NZvfCbf}SH(>J|Xs>emMoS`4Zh&g|=5Xa-Vqahs1e_II6lh<zb*n&LfL=2W871r(An9chw3nMT6>N?-LwYeie{l~-y^nf@lxto z_mfb|?~8hf3fqEenaR%H&=bP%->5p&t>-ya;nj*R_ZY}^{>XAxHi#aD9hfy-NgCM< zMf}Nd@YgK^n}m^cU2h?o-%SIVdV`>JJ*seox%#wf!V0alOl$5?`W`WHHxlec^f8eWBpX4py`4 z66?4!ggmwgnDCnG*|W@CqW8 zYTz2}oW0p%aj#kQG?8P3S9o^ewm_!g8fA6C4UU8St5E$A4%ychP(u7dR=;69m=QtLw9^N=)gD;i z?;4h*-{QXcda>t+a+Hk@6==Bw)y)2tsVDpJUUxa~O%5hmf*$p_TxM|896m&FF7m%; zEPa$7IrVSC)*sD4E}4q$rI%5oH%VdTEeG-HJ?8w=bSU4PjYT!*SdYm+=uq#;Iuo~m zb;neY^jLzVjOTlMM^J&#AAF~;0Li-^MT;}nMoI5@*0lwb+HARC<)e1bJ3cqFl=)5er_ zD-F6PXL221a8>6TAIj@5;EQ{VOtQ?#`u0e2n>Pd6;sZeK^;T^6?c#6o5^C!=h;^D9 zgXGdS{7{ty68m>mrQ^0^db%fQ6>=eeGsnsP-pw4W|7Ew|&miThkC?ONzVP~e800N$ zM9bNavFzng>e)B}ik6NND;yK4+hYzD-LB-{xq9?HHW_47oWDU{K2hQ-&dr;yQh)XMqNtiQ*Rn{zOg zv_HjwVP)8`U^F>*Z>Er#5u{Vqv7*-_sG?#G%k^wP-ETI`V-Dwb;~pwL?CnjG=(eiN z2^JLbo_}Z324m9JLo6Aiq5P#SUCtj(x~D@)J0Y*i;Oi;o@ylAc?zR!k=ki%uWeF+% z-YoxZ5V`F#q4H>Zu=+O{$_=lxn#x!To9D>X|4RUqqxsBW(t9SI`JYgxcM@YRjiR1^ zO`$BG<1f!jFnZNOkU8$+oo+1(qXpcj?87;J^DC^zY7qpshEnR>*`x|Qhep{=EPePx zw0>954Bz(Sxdw{uLRn`c?2gr6-g_6Sy5n@kg(c}>FSS& z*;%i!)@KOBE!)oGpDrRxl`+Pa_62Qt0DAJ-viv^x_O6*g9#)GfdYCc!7SEuVbEB!K z!%g71+LZA<$F#C!9Mk&$ivu-TNm%CmFRXT_GT3`9-^*gCThOk5@PSBL)^n1>}Y}+WJqktZmltujabDBk532P zaDgeODTOBb1GBzFKqcqB8oaGQmcTP&=TC(Ce{HDA`bSK@DWim2=kTceI*RD;1k#ZO zqRjA(=xr8B^5F`$ST#&Cd;>;rd!~I5HWIwoinTX`#%- zV;DJi@$XXF07|*VG1|$?L}%l{5OroKnJ?@E(xTmBiHk3}e%ef3eTGBRt$(n?dppl} z39SqsXARDUu@EpqN;0LKc5q&OTc$v@Qh}l#yHIg>BBfrngJ$CekeacXS{qxjz(B(! z@BBrH&o=abv54=lD-_C);UIIni2GbPuVv{$<$GNr$6_68I_8ZX15Gje@8c{Xc&iZc z*$MnkKEa!7bSyWD-++U;s$(c`{lKG<8!*?9((fQc0gU@$^qM7TkzHoiT zV2FF~Krcc^l3isri}_TGnP&MYjb6eGo#mi?C4;Y-8%e!q92p&bhLZ2^@tmDdvHRe1 zba}9b93O{~#@0@d8AXZ(2lk=0lJK=^5ygFbhZ*na4~8LJ=Kv!pE@uo#7R9pLCV}+! zYXCSMG9p#ZVKHnp_j$|>r;9aC9N+E_F0Et8&SEab{%F9t=42{*zn-PM9}0fiQ>flX z51ej4W)i;)tgSoGyv|LB5|3}NG36!m>r=yVD{JaFTqv}?nT#pHM$Bl>bjYvS1WEF4 zmR!%Zd$R`NZTWC$ySo-X^yla04;)t<+=J@xdqHK+Fe(pEr@ZikSXOEX>H^-mTG)pI z&3l1#%tnyS7KNkH{UJfv#OsYtW#E(QzQzoN-Ht|2ci<~=m=Q16q>v+7Eiusr}|`Nze(6(La*&T)ZD!H{dMCocU}N;7*$3 z0Sd|R=V%?s_ww&fil;MEsByFn*bUeK;?@eR_?*stSo}PF{D?(VaGmG*-Bsy@DMW7y{PgOu-?uH^w@}lXAFj}Z$aRct4 z=EBOVj&s3S`hz?9PXB+ToeNuy-}m*m6p4~bl4MJgq>@tF_nd9#Bne4Kl1d0kl7u7_ zNeCfHI!Op2b+1|Inu&FRtvSaSpRtk}`wwAdn4x!$>8cIuz@$H;ya2gi}Z`bMLazihS&y5kvH2QH4yDs}{z7Qu1hA0>*QSKut zISp{-S&k!u?Tii<^&%CdMMz#c!Ghcgm~w>!^P0^&Jcr59`5f2x&aPvwt^>$&^#ZIg z`pB9rdXljZzZcq@!mi%B=n~Wkxm*5 zL1mZ^>h0OfYWA(*ezvugH*`HUW*CEip*c19Zz9QpTs!yQB1_=;Nu34bcrL+-<)nE~ za_Aa5x5*aMW{k!#4J7gE37+HsR;W97EjkAb#4ty$4~)LU?8L5EQ+I}$z7NHSI7Q)H z4c~LFdk7^Xtx-4aGnZ40!*cS47pbPp%CY1mEI0S4Tk6wHI#dcM9RwJrXP{De%nj6F7 zhD*`WJ(zNzZ>BoOr{JNm2Rb{ANvC)qG&XcY$2wii&kMu)mKWTkrz8}7SWU%;pFozR zg2@MtVa20nqm^wUv|dB3%rmCYem0ahuAEht-V~xUyHWh(?-0J=K66_+gM#=RkiSY~ zie3CL#c(?apo!WqdtjC;?-`x(1!kSxjJ12 z(IU^W3ix_~CgwQXpl#B6Rv~SNIy%BCW1a~r6}nXEkqsR`eJOgsHZ>Z4g1VdLbSR&D zIo~;>L8Kb_d;NyQx?ij@T!)qJaG-!Q&Lr*a&xG+t*jCXO8!s&a51*^7-uR&yx-bUR zKF>nuftKVF{6?s_76UHd+(3EfdFEjE8PtcoX61i-Qb58HmN5P<(+SpwmZ5X0WZ@mC zQQ$f8I}I|m$MaCKh4TU%Y-PIKpS2^6>)QwWi=DFd=+H3@n<^YA_w)$PE#tpscix}T z4@mBqCX388ATcijQck*(#>bu{@4a2f=Dd+_n;Jmf;)}3~>$TBq)IlSyH|aPZgCA2> zF|1Xc&UNThz3p_tCEb}q|Lg_nm^QKTE6-5hNM`xhcz%ESQYdy34jPl~y6e*E_Io+TJ8w zeh*nyzU0p{+uzpAqWa&DpgH>!MAfxIisutSK1UT)Lbb^_MU6E*>_+uJgYnR2KG(>8 z+x_jb0^8lVhiB7!R<|skMcnC-r40y0Ly0pPzuioFQyeJeu?>cf)FtUVYoXhQzF1T8 zito}kqgTj&LHc+aKig-qf-@3S;NCg$mL>TWX<_29Wl%h=kOl9KW`SovvfT6gp}y!2 z3tW1UwQ?U@L{}1ZzW0TiDkbLmFbpRuxzhX}+_!e(uh1AGWfA?|h2SrqxPP4?`h6dP z*+DbVY)&{ztWWTL(P~n8s6g{o7g5bmUkqKL$@Gf0f!Dksz_tA_a8Ossa`vE$MwTc{ zIS$o{eNkn!A9=mpE-ZH%ip78L$sR6TfziV}vBcDnq&S-@1J5(tW%rq<$~Y4LEJT-o zM_8(@6iXulN!rYqmlyvV-qVE?ZBK~&wwOYdzjI%>CN1ascwR;e(;dK{XLD{rW#3uM zxq1=LdN0S~z^km)!v{-p){xie4uSLQ*bZ%;fhq|VQ(OiKNi(;QQ?)?3XZu0KyHU*P zSqv+ldz<$tT!#Go2G*3zwE)FTCh_U8lh}F60&gy79tqt4r}qQ$PhAF&9B1-5wHV75 zcCyq5F0{GcmFi~y2fg&oF_%6FDg52+(H;%GOq5WXHWL5ZTBD(PC(F9D7g}8#SjMj) z%zF5dc|AQKj_Geg6U(}zq)Y+5F3v%hX>;JUwlhW+4JPdvE1pvrO}o0UruaHvraUW> zNhW>=J=MvS!+n@J?O3 z$KbwcY`M4+izgg~j)8X6oUoH~qATqt57VH?-G-FkO9wn}3?nh91N8RJK(9MJ3CFvV zW6(Uz`tq9TiCfV2j0X$1I?H%W2kS16p!lJmng791GQIf`BnG&RFrxgQ$vlr`#M;;P25B!Dnyep38N+$5^7~|=*A1Z9 zecXHK^F(N2z0k7Cg97GTV7|m3awnI7>%@U1|FVir+!Vq+n>O_J+cxl2GT>TitPp6W zORYxxh3c`PC~ux8I={2Rg0q`YzW2E(EnOq-a^-p1gQiqby907J{}t+a|84&KDfGIl z2Bstprb^8}kh^p$=o~!2I&G?${(f`R_~LQGJwzI+HZ0WK=Ku_F@W&eY>KS+FEkIu>d6L9YRFeF zQ#PirCb?)2z`(3WqTPBmEM796DZKK*_67Vi*zx|`^0QzZ6+*kL0COi~3aL@tx82Q) zBqPl^v&$B;wCq^ronI`mW(cd?_gIkLjugiKai$3KZK79~AfC4#i0&U0IDd5nsn2j@ z34M1l*E4Z|!jwAkT387B~|NeXd^si?1OhFEzyC+AD(lgyq=sCyWx3 zY|v}kCiHMfJNE7!^#7OVutTaTIA_f6EQ zWC>}0+;ca?5lYSYv&vR42;6%GJZF!k_MTc4xs>xS25@#@W&@0AoRq(2VVv%Fl2>Vyh7jSd=@S^_nf1!iv4IJHL0qeFp z+rDO_uYGsV9`{|j~<@UfJxot-mkriwnGQGy2slm|NVlyN151{&6nC|YJVv* zv_C|DSGC$ZJu)m_dHxt#y|9zd-Wg2l+kjMNyW5W&*t^KLM9sc4V?oh}(m4B?w`17B zSQly@rB$@a@&(LpDx--(LH5dHZvCIVN)74WXt}_U=AQGw7b}-gXW$B2Xre${?|Gqj z-=nxdwv&oKox@TU6C7{R%Jg^khoN6jP{V^0)ZJ|W9@u;wyR8nO?=SrMUh@t7_)mo% z%<`k8Ss7>-ybXT&jlw7)qd@VZu{gQgRt%8zMvnnI@Rr9BW_EaraQfkcf(PjrAmZsZ z$h)(LE~MI1xm5^OOy5IOjFRy1y=V$sw+%lxT_B^^i2K&Wu%_A7E-|!xFnuBlPG$^y))Or{?V199t<8X-0JA};jbhFJv} zcyIfDp=ae@;mh+Fn$)%hPkv4o2EVry{;I~oSeFHKcTTGi!};C&o{h$#gO1S!my5D# z+67D$v+1}6dZDJ;tT)4xkn{%;#a28Xk+e@n| z+t|}P+i~ISY?}OM4&5j_KnXuHscGO0DjzhKBJ^jFuZsl}CG+vNxeN9S4MVYX4273Q zlF7oesM~WkZ8$%w@NuZEXclBa9lMXBe8nrVGjSKy$_v=de-5nQ5Ux4@+(qhgdt4hE ziZ8G?%ytI)U(k6y-XHTWsZIFb||pPYOsWy9TsG$YoIx8}_o4K~bnI6N zq@=Qn^6(Jt{HccH0{hUqIZ9%e#>e1l`Vc1l{>)T|Jlr_DC*DXpNhaUsv;svZm&lBxp(2*;C|HaT@oJeat{vrBx2!LeZq~y z*oM?7;VoIedq9gtBlDw3>0ToJU-$Jfu2a}{QHNHnUPr&G94X+c7mOVjjpkpEv32{cX;xnp zV+>;`c2gK`TNViZub+r+@89r_$&onlRsp)p0CJl~!COnuj3vUlN zL61M2CS#bwhOSNoCFe<$G5N5#uJs^RrYyvF+qTle7kfpIZ%t5Av<;&!C9&o^S0QHT zZgkcOqA|i*^6nJ_yIW48MyX|?bL}t~6+9m=jPSr^MhB^XRW3oC3wjAZxo7Pdv6F4= zPI?k%`R3p-#nV)KI1*-En@=xqCgJ-TduVE>FA9l2d4FdPPCl`b>^BRbd)kzG&rPK* z0X@mnJ&e9P#h~jEFB)|27|xuL%3|xbVR)<=?Myn(3fvV103=4 zlQc3va0}YM$CFjrd#E_ro9L!L+V|T{XEkl8Y_knqcQv5(JGtNBuRiU#u19ZP=izS+ z1MD=pL^pd5!oj1p=v9sd4b`25%L3Y&-lRy>=D4BZfi0x1<$_ONC6mRVT)O7F zy)efm9o_8{>3PrqFuJITFO1LP=M!8rR-FPZ(@xUwkqH?5F&lq8n#2y8XNu!L>=u;$ zmXmptDtJ>oxx=&c&N)0p} z8BLRRAwBBD_Z5y&G|_SonFn%qXY(N}`J)LX%a!T8=Q*Ao+<>EdA0xx}pCLWm3&$Uj z5F8l;YWl|t1q?pa?KCuc6Ny_p?CExC0QgjTfd}WP?vDLjcsb63>(Z+* zPIvPfhr)iFD8e&))DE>H20Se35_eopu{=;M9^~fO@8@@$$ z3bAI;^>yF8E~P zxOOveQ78iIr4zto-zm28KO^?3T|yq}zoFXXFxslmMVGxbBq2l*L_I1g2eDWy=S4b}69SeYEzUwI8xhuxD zX;ZO_wru>~5PGsjPV1C+QIzdKnU)4)2D?XK!4v&S{F-YQpa%Wzw1>&+TlxZ@`620PilbY-CRkG9KkXY0WRGi zjgLoaf~jvdUGHIz*D`(3d&(YMu;3J|s69_wx70E7;|W}{bSD0HWgVS-GgCZ0vxBLR zMu_W^gv++&;M~u9vFD9*==Wtg>vHFjaIXJD&>Z?jQ2H^Bp7~EF#WQ{|&|xt;n;gTe zjoinmc>=o(vcMU>J1M3jmo(Q*p{?t;qvGzql)W_-ha5ji>0dUZhNl7qt<0mC6iw_~ zu%D*uo)U(69L2k3yWq#*M4T0_jCPk>!0urjtu-{pzPD0wR9Ye>hg@T-`5y(F8Huoc zdlL+*-3>!-1z_fqHA4K@iDap;1y@*Kg{xV&#k7pGl(2m}HhufRWa?qiJ18DE{@F>! zeU9Kk(UtmCQvG;cq{;z!J=6>R<}aW= z!I>yeK8*YF^624Vo|hQC0B>9vPPHG$ky_}gzORHz#=+)~m`d}-hUyQ{K zs|SOx#$gJ2`BKoE)RT4w{ed3=@wm|b5_W&v#Cm#e<#(I+5YRJ{#xB@ucWAi-%EtL( zruIc@QH(5D5u=5!?rUhfiWw?3z7*^|wqitgCE9+WkR8AI9Ezk$sMh0F;SG$y;+LKj z+%$>ZzPk@=dR)Sp8NMu$8AEl<3|M&jBF<1f#O7^3QmBxAk^br}q@M#)Nf!PPmfTLF z+0k3^%*6f!Xt*(+?i>Fp9RKSgsb`<2>tnlN&b^Uz=Y1~a zK3ho6r%pl5jl)>-GysovJ5TLCJ25yq8oO#gWY@w*!e(VPuzGM3Zz-=w6~_y>_HsO? zSzZ(meV>a5^pAs?b{Z9ySF)hW&#-f-EqnhoPdKZvgX#5u&FX9aV{1LUdB%Mk7%lCM zBi7~7QWpu{HyTVoGt+6kcNWZ^tcu@1sf96z6pY2)YAOnYn%p)BbXX=}xO1?vXhmtowEiWIb{byz0k z{_0K7lkc^YJ3m@AVbo~miQ`X5W3p6o2(TsY8M#IH7Y9RjkEm`<)zXE~n z`FuMCGcG9|CT+e{ctFJc&3c4 z;!jpPMiI?kTxTb4s8N68LLs=gAL(3?VAkHxELdF|f7BV!r+XV=Us5wm%eXDv>g5UZ z_KhU%&=e>*W)7{tb;ToJ+t_#`Kk=JtE9)FQl3XUdW*+-uS$kxE?r*y*i?7}d4JitA zXF@VWL`bl4%V>z+GnZMpePnwCZ)Q65BGV4NB>OsN1a4i;*aEu;Y(md9tnpJ%%<+Bz zcf*Y7X)h%-nj#^6buGMlqZm@x?}6hPYB+dm40!b>rmz|FyL*`B;iX25xiNlUhZgWo6N z-9}w}7gGw8wayFEehr4pu3gZ4tv>$P*@Mq_=`hKyhKXjq1xLQ4{_nP%@L`Ps)hTqv zJ#O9Es|FE#-=Bbh2~u<`?SP`DC!iUyi3xwtu;6!zLXUJ^ykcuXJ>*hS>AeH4^ERI+ z9m=vJrdx%Wl5ns;*a44k9)d+bjNtbq3tAJ_E?cNEi@9m_r(>hevIPb6*ua>>;^~S7 z%(37ZTd-&diK-3^A{tr0(mvp1tc9ag?lYU0BPhYO1Mar`hM(T9>}jMG7RjR^cxMY6 zs5yu__M8yTt}J23qcWJPp*{^flMhEqM`8C>yBJNkL>u8D>`-0@%9D8i)QA!2&GU{~ zlm4)q0p+Y^!)#WW(8`*l?lK$q)#8_1|6pO&B{8yS0^GWzDStI?xIAQWYLPro`{y@!SM%eUe zlN$kF`C;pAHB4Ag#$0#lkjgDba_9H{=B@t&F)?FZ*?oKo^$BxCn>Ov+5dx(pUYT`1N z<+!-~816Zcfb$McU|Tb{)4loJBSWd68kC0SDO`)|XG#4BXtNDF3|L>zm`i`$6_?Hm zryKoGVc(Z?ZjQUi^%=WrqO~)@7VSkd&f3#yA)+;I9+yj%Xj1WetzX_$Y3*i_v@KZx!4$Q+`PxWZGFZTXL2?|^50Z_W z6%P7(2^|`56#UokEH&mmVs+f-Id|?p>d}2J1q`#nAgiPFw)_rw{I?1ucCoPM;XJ(8 zdmwo}IfFZQXVJ`W4gjWmXw=~S=qWiyJ#{+@iwm~HnCugHvh$8$cW5r{$cv(%DV&SE zw;1A__fWFUJ@z!5O&F3N~Ogk=*RN|xM9F+rg{G$xgJ?gxA83cO>(D_h!aBT zu9MjJ%|1{$9S?nvZ^y?IZ1HBs2wGNahOgU>L$ksF>M%Wx?Z0z46KOX+$~F=!t%7lW zj|_^oo<(6jHsZb+UC4g;cI>AcMf?7@Q{%I#^hG+o4$q-{yFGMgcN`4-7Ekg)2L*{n zw*v9aM@Zy(B+myP=&(hRcdC>zp9XbInIQ;v*Gzby%M-T$=M;4E?Lon++00_RDJFFt zhZflq9J6UAHgi2KFfJMD*L#Be@g7-aMmAeM&W+?D$wJxgTnHY~6BXKrp|aKhbl$Cs zvzGOz2uqv7P@R?FH91A-Rg}*nGJUalt|5h~^+G+f(b#&rfR$Z~gn<2Pkvh4KUv7kN zH3YIP%?0&M)9IU4Cm0Xm{uVVwYL)SBtjn`7tH&x_p6Wo#dG(MohwHX)xG!pe1%%~W zW32Ww7P5FcDo5ooAKmV>e-r2O|NRBp4Mya+(uN%8o1-LSAPes5f(bi#rqymBt1llQ z1{z!zu7m}VOYgU=J}Xy99JCOko@cPcCND_MG$ohEbu4?VBdX`V1$_ev8TE7~_e+;S zPr|cz6L`M!jVg7zFUDP}6Hsz`vn>2$K1cSY6R^)~Q&g;-gwu1Qsf*(x)LtKr z+t!UD^EJ!x^!Y;8<$(<;Z#JNfJtD#8+$rjeAA%#69Kpv2mf>w-Bptqx4u@OvS;&0u z|5Y-i^1P#{p9PCMJ?br5%G`JratRNOsx2w6RC*eX%Fd!qXDn`yT>+n_I)8>(9L!1 z+OQR5IYxq?Qu^VWiO=DxY&1>ZIG$z9bfOr3H)$=Dp@YM0;#xSRWtK6ahG)}1-e%+1 za9+yhN#q{-5>ooLi1m-u#M;O$XfR1bM$-kcV8F?2G@SB?{M4{+{?7H6gJ2nLtxGH9(?f{kPJp?Ti|R9|(B z^T>9J$~iZfL%$PH<6FWm-4Nz5YZ<*mDzJ z)LW42DlW?m=X2GfRxyt0V5(0ap7Bi;P1qXL{xg*vrmbXhduMQ4>4jCTXQ6c@GD)xA zWy;y(nFUWeU4N_r?^m zP22`G|Lq6eU!zcO%_HXUYc%MZY#_zi`7G z$XT~RJ-LQ@1(NiyKwiZiC{CXtT16~G<9op@T<<80ow}d36>xsl#2}Q96$|HIwnU3d zAI02%X^`8e8%yNcOKGGd7H?~nDVpzOtwSD)33tyy$9%F}Fqv>3ciRXFdM9-A;s=A@^mDM20vMEe+R-}p1&meGwKkIDNz~-UH zLF#xC@|!pd#@&eO7ny?l4o8r%Uf>ie@*eVhJC#{INj)ovSzcI5VQ)=wqRkO-8^W2W zu3DJePfDFZ8!)$Phv>Gs2exsqUzOrNxY9a-GvlI9_`3nwpv2Ovo*4s_a!W%Vr!w-kba`uCd7N? z!;!Xyq0=<(T{u4DwG&Zk4*d6qMw+Iyhc(k>WX{2g@vbHdR4pHTTiftlKtLj29Y zPK z&Er9GOa&|^_aVDZJCbsGNWFwJRGhEDWL*`CbF`-LM~Zas)L4pF%NHZQWfn%yF~c0) zJg8W`0~&Vo?{L;%78fy@bPp}V1cwfGbmwX+G2&UPtSlj3wpD1V9*oPodjF3vSl~Hc zi4+r$LFLA`tk;1&h*)$?R(v_G&{%Un&t^Xm4MGjMXJ8=tV= zn@MFi53<}ppTy+q)nxlH2kI(EkYw`=yLu0IO5LP~()rvE^VW$qdQ8Q%sYy^adlrT8 zeYDF;&i_5zk5bBv1iNT`isL?47x!Wo@oXHHjFOPf0~6*^{uervt3gs?YG-iWiaG|a zLA~J~=;4+NjYG9ybHRMl)i9>|2|;2)%Sl$Y?F39T&x88C&&1kuYq365p90TLqMECt zDMD*dVe6$HxZhfriW5#2dL1kh6&i-(izB|=gS{R@Pi$is*AJkqSCgP=qCW<L^WSlHyV#0lq_ViV6uc)Vk6cLK0oEezwMl}Wnlk9f}8lPXWY5?l(l z^DY7hioMEv&IgB~zj&8x6q7G9#2q%ynD}A~ zYY(!fD7_0zXTeKQ_IE=%?ne>Zo`Ke#_TaMtYr(uwfkJJa!`HRTJdtlJAI zm-~zAoBs$YbW4;s9%m8W))=&~AGXz7QDlHV&XUa}Y4S+!L!5^>+P_(ijR{TW&(HA6 zCphP}4t3R+Qbgf;`pdaI9u2|tqU&gqm)najlh#AT?}WnQj}xG=MUkJI33k<-iEm-P zN7Sh}&Puv!kh)@5bic#BaC54Jz^GZ&dA$sxNAoW0kQ-u0{aQ@9bVF!8I*@0tvP8+z zUqV*3D~qU`EBc$g=iEPSs<#^m)gg{lVQENNb2pO5&>R-a_X*OU+~c<^j)khO!}cgm zQtT;3!>{v5{fHf?=;~1RN+*oJ?*^u=3(0Cqe+=mUm^1u(fXh2G7CZeWw0s&$$|WL8 zIXRSOEf_}94t?<9S(S=cpP+SFB^XSaKoXsM1sz&5(IH|CRL!?Wal~J+959J`EwaXj zxti#HeHb1(H-dCzZ&_9wXS2!8A<}j;8K2)kSA3^oc)~r%y4|1mL};S4tx@nBvj)?& z=Ab-atT_HeCZ+KomYav=}-wz>eYTPHkr_gv9cVU(0 zu^Uc3(OG{zN?eZEb~`+rYJUG_Mq}1faN9b{+smKTt;HrDhY4I9pjejQWDjJ40{|`Wg97^) zvChtC&`B{&dXaO^jRPr5XaYmMBM_^=IV^|xF6yBURs9~0Su+NpRmCH4iL|3$>!ql; zi*skr4KA!qn#Q8LDzdsc1f?rwRA6R`rAZ^P{m)`9ji`~wa_$#Y??yF#X)Hq9MJP6U zAWrT+1|`Yg?b3!^V8$k0&>P30tLab*vE}#qvqQ1O&Iw&-?PVr^R#N+_|9F4LCJevK z-$}!Ig8WM%MEEKR^;WrpyYn!r$&jOkwi$R{b4L%gpAh=58`k`Y0~J+I+Fvmgy;gRm zUZ2Bg;$kDpJG};-!`!JlD4dL|H?!Q$FPXB%W{SvMFJ{G>LdfYc6gt}-+;{b1rsMhN z-&HR-RT}Y(;$bkI;7^;!%_fJBPUvAXjehVPS;~iQV(js$n8P?@=pFANa|*|Z&6

  • 5o_Lk0p!>GIG1rI7F$1&x!B|hdhd|hJzT-!$|u1%xE#_-@A1C- zC4#P|9;)xvV8ZcJtd_HTY-{gA&DA=PJ7>zGR!pMSyX(NeX(qPD%gEi@7UauDf@RH2 zOc`G$DERSi&cd7Q+azs@w<3tx24bedUOXmEH6mM1;Jrkof z`(xImXiz?AKvuh#k;mwb*tjhiO+OC6w=s8_ydhnD=(L0)c}Bcy{S790=xVp?Y7ogs z-jqdm<=niu^(2v+0Xq(~NF&J_N%n7O|G#q@5U z0cm$@N~&CqJA9=Ww?G?fIKN!u_C~zdvJTywIUDDGsn~w85hmVTg7GK+L6(XRQ%|W9 zq>)nSWpxpxNqx}$(?Yt~&5X~2|1qobV=Tab2a8Jfp@spYI1erX3|;|-A05NHLF@#{ zZCjS*Q_IE#_s9Cj18Izud--3_11FU=l<%w0boM=G9mmXY*JN+9+LHp6VIwKO(t;$Z z=ftYP+>3jRvl)$fH>>MHvRySFbZ%&an~fqxezZrgw{kI{J`s{EYG8-L7>WK)=Du;JL|U=a@b51EEIu9)ij_23qjWwh=50oo;3us3g8^%;s$e*E zJt^$>py*;R2r(N$HA+EmZDq3_*t@srHlg zKW#$(n~Gr8+*$ZniSzxImoY^{Ynsp5&3;~k$UAEp?driZ>QnpTH$_!URq;VZ0my3g zSGL1mMwMzcP?;G6rg}{fKirYEJ`SY3?8i){c{R22UW5=|WfH85nb$0Rk`~B?T*X8& z|N9NrSh9dsIrENTPkCYMw|=bLR|)01N7?)v78G&xKQVOZIhOHq1kSp`*{HpFV)#i0 zNbD;LEe!XISu2XzTlaES^JE<>vC+rCaf?{+#zJPAQVQ}T-NcS9b1B+WVBhrlSvlxD z=ieQHN~dEG(HYKke=VW#Z)ccfpc|7vS7j=zb*a{v_i7IDC%f@8sg?6OBIY%SiW3%L zdlk=}e(ld}vjXVeb#L;|=FGx1GpYFt=lNed1(j_Rc`jfVRL;5!>dU`@gON9Dz1$&K zjD8MvR|N7e;q0>H@8I;{B8%9Q#wzg*^IAJdRGy>8yTQ(LX^(r7U*7;7?_YeLF+lY% zoY&xTo8>OnW=5lY$*1BAh|gxD&%h=q1I`$pKa(mUfNXzsBijYjq2*~1_HsMH^0MO~ zu9Np4<~xb`D|>;}J%Kb{_QUqoYRqKzc+NIEUnnuH6eP=&3M!|zvidoPMLHjb>euU; z(N#6_i(E%Otw_;gFFX|PhIU(mG26=-uN+yARa*A{%N2^Q-=<*7jbNeKrzciEt%vfS zOR4Q&0J&Cvgh-`AxY)a$DbBmYEN^ZH-JdZG-RfBAgZWUfErsn^e;n%W3Cy7AAahOC zhQJ}UVtlm=D}H>b(0SAarZ#0a6!hS8_uo+W=)*tG&M0T{%?d)|`t2-u-*?Du(-un3?*+e8 zam*rOHb{?m7gIieDr_G6fawJ2Lj30K;=@;;nafaBQ5rm#J)~IH+TIfiZ2p1a3=!Ht zHG#vJ0W8o~157WKLc{XYEW)j}P;oMWx65OY?z>s&;?Tq_zAY9cttMk5Z2HO!%xMFaRhe$7ID@IBF*wvvv0F0O&C6tEeGfx(w{QrWcL*M4onZbtQ^5cGA2C`d0#^CnWl}2z(QI2C z=LLRbW;;4r*y11HG1Ux0Hi*n2@`Ru}^$4`)1c(v&JQH&3G*{T#n9q)ryemA6DSi(E z^}c>W=fM9!@>oN-F`$Yysm1ethj_6jQ43zYDg@(UEyDcrbQaNU0JR@V;f8lPyRjw- z48A_&nk>(G6zye|-L?x_x!ssu#WU90Qyn5U&nR?AjuUG4?}eo6X-wX4F_@O`2M6~w z!7d{WisdVWM$devFzx}%_c#dZHkwR9_z6BruJCsPXI7tJqHWy~2#om+?k6tIUmN9>aH|(Ns9$Yc5fRvwKWYV(xLU4E+Tsigx!am+)>ai+%0JaliMnyxB9EynPZR7ep}&w+hfPy~FC2{;=5KKCJj!Qlb98 zL(J=dpI~=0gM~I76I=J{u(~j=&$m{IuZ{mO=Y9`(rnFtiGBgG+Zx8X?n%}&C`-SlB z4#0BGkdt@Y#J=sXfnck1Ag?bdOdS3iOqCK?O3QLqe`cK!a;cVitu7YQ+&aMW&lA?Z z>AjHE|Cm_+y$g6rK7jn04=Y~ql0~@WvaIrUmTDaX!H-RtztJh?u=p42wQU1LMQ;H~ zu9}^m!B7_a%#hDA^ZCD#iVd>AaOHR^=-GO)n#J!0f6hZRPJAdD$a7gY;~MbXe;E=7 zK4R)S^;yk_49E}r#$un2fPkcxkoWBuQ(xFaXiPm04)y%IeA!x%xZlZlZEazaeIJCZ z`Zt2pkku?+crSk29Sh=;I_PZp1ksB#1cj6|(0i}}5@)Yrk(VpM)YzC^JaLxyO+H`| zZ*7^wm~ug9d3U}KjsyRMYVpwAMu>OcBe?YaU&~z$Q$OqvDW5NhhjgDZ^*>5t;zt91 z9u1ZS-boUY9KL|z)lj}GPZo1?M~aQz>P5TFrC;xo|%k`0Po`G;4+5K%~kd+7JGoYrZzK>?S!(;Z$#02IJ9NwvhaUX z+1o4J@Aa@0D#!d5^E$6Ei7@qAlqvsQ4J%mX=g}gaDrO}cDj~Ob0kq!ad+D*6;H?qI zQhcphZB`EN>2v`T%N8(x)6S$Lqhv0&ADG9$FHC1k8>C$L1}@ek#k3a^sB6mswbf0~ zrcn=p3QGk?d;>Z|uL`U3&oZOoC2;6Z2hUBb6-x8376$tF66DqnLPSR&S?ILWqHR5o z#5^x!odK_y{QGvWIPw5onnK{BTPf7vJ1Yy^%inEo>%if$KIdZVh+aYGWVU+~Aj0jI zT~^g=At$^S)7p{__5Oa)aONxA3(97;^{2$v*a57R&oJ#`gIKe*mERY>Lei~UkT5X` zERtFwp}R4w&)mTBlaGlm2Qwf79|;qMDaRIT_?l30p5J(gk+r9m92vU5j#kaGj^1ZY9*hcSPLh(n5tW$e0d;7wXIjcuQuBs!9 z$@&h5KDNM>yaZ_ZdI4?}9|H?Dft^c9V##&{3aiSQvG766{d|xaf1AhND!4IQX@(ej z;Vb*mzn10hYiAdCUxec9BC)uCqM%rCk=4t7v!=OUpm=?iOc=3=`Lu98+|pX1bu!=k zd7WhSYaN7|x&e?|+`_be`9t~oi%|XI0dv@I02NmjFw={-p=RM9G4Vwg2$(a3mAJ0~ z<4-EE>SPg2Hogw><%7i6D(9frv^5al<|>qU-Q#D&2*G7Pu=o?ju;2L|l!Z)$UF|1X z{)0D6$8S00-yY2jwZDUM-60|N`WKeSv&dC3napG8Gnl2ni=}*dE5m_rfIgLQEuy+G z>w_ng@3WQ_*MDY>FU|?Zg_VL@&21ZamYB9^qK3As$V(d z`L>Wa@DhtZz8kXEeiTeoTiFY>d*E8~4%++f5%q>|hsGmAc-P+!nNA%EE@xf|g8l>M z;UCCsHB4cAN(b}meM)w3?_SopI#aN?|3h4Lkmrw8{vT=Q!dFxGwc$-kCkd5QsDva` zl4`FxNi|rzN6c@|t6NaP7`#b3Rd9^6<`6HBGLDq3%nP5M-i|OfxVrTzOF}nO> z#p$HO%y-vDjIP{N@m=LRD_UX0o=&)huJ`7$+M^gJ+YRt~G0^Fs7KbCpFVbus^PJJ#lMN%Sfz#G5nzv6#Qxndnl?TrV9EALVUl z2^YSz@M(7i!@m#M(@hDirgRkgx4vP~16;B3S&}Fzp2EGL9zyPbw@fv%fJKUbnQC4! z%CZcERNvl0S(78$?m5a`N)H9qa|=%93Avp2mGnibAs#_v+zV?8g;Fyh|y zVdf~G)14V=x3JTiZ&~uyBId-u7Js9GtXlaKW^3(3&*GIBp7lyB;arC7Pt{`U-Uw{3 zzRJ4Bc`*M+6Ij>RGZ>xzhZP-QEL>9;O}|`Z#cu-yYmM7NPW(OgLH;1qe^koescply zF%2(W{%pnvGL2aNm10`n8htvObFG>K3eih^;88wHNiZ zZwVUVR+wONRuE=;py$MQm=M(qJNrIl%{l+1+_fs|nFV3)1YcD6@SRPr&gMS&u|j6> zBy?Do#p*Nvp#1f5=yRU$M)Z$L{a1}fS(vXFHk&_dhn^ACufN3a9!03VH~?K$-BH#r zvqDy%gR#mxuz1#foIGql%MSc2q-sb}3_Zr*Z&u87ia9?UEn>kj8iFdnJ9<8OBGf0Z z5<03Mi&clb(D+xpD4W|GL-<`Nci9AXIlBf;wU4v1q(iK}%35rm)>6@UTnp7M@LBS6 zCsr-H!oC{veRa$!*6P!U5?w24c6gW6S7{+;tt~*c3huf!{V24Rl!~9jI+*2R9W0+g z*cizjfJ?@pyW>qvmuaxr``yr4t2fgb&;1nhZlkHi5oUbwteDMtHS#wnV$siMO#f$+ zP{o-`hMMOvGXi;Dl|Lg)4zai^+xUCjwOn@Ygw+4P)8c58H_Y_SXI8bRRumOCv#Qsr zm^b|w>Z}*}jkv19_sCnkR(yrkde(>q>;Ox(E@bb{j$yT*e+iB0gZR%%gP9%QC1fhe zm}}8uv18Z@Sa@*>gp z`$j8@AMOqni7IyHqtt1f(GGajryK6%`L2NPS=2rC6Re)Gm}%YP&g)Od=xN0zydPcy z20EG4^Uq?)|1QO%Tk`DcfZsG;zmG7mt&lCZnh67~FIDh*C(X#N#n%|` zkRN|qfNaiHy<*H24QbR)Z9?qIQ9K``1s8i)(UOiDIu_wfKl>dJSJqVHL$HJ^%X`|T z7k;h?8XH3oGO94{>3UQ?qf0(raB9fq{IhARh38IZd`?0vtH|*lBDEvnZPlTXV=*qvnz6I4%w(mW|uPgX_G zThrjkQB^b;=#1%IE_87GUc7Xthz6%s5Wg&9LPK}E;^FPMb3!ngC4Qygtz+y$UM#PC zpw-iEz=RDH-?6IFdQK)>$sAtQeZx?@ttWd_m5x)e3%AS^_RqL!_iXwQeHm6eYTxR8> z_8GLM{TQpY8&-M9w$|>gUMjn1*lMRcqrcrj%MzIT6Dm~;uU5YQG10C|zq9gx?dvX9 zC?TzdvEtr~O+uIPc6#eJ2$Z56VEW5ps4cGy?LI+ZYkdY6S4(Nu@;uDm<|-NvzsyF< zPo+z%CDhAe2@Y_$iVc?hK7Rfv(;t6C)VS$I^{yY$clJbTE#83}G8dBnxFuA--Yj=84JFc zA%4%&q}W?hx^J=p6fbXNSr#kF_~&4BoHmc-0jT>D4=T-df#So(Lh3b5*4d?l(?+d^ zFrH_0Zm@z_yW8BM)C0s9+`G8sDASJJj<)(X)W+x3?Yu9TVdY2eVfy5v*bf3qIoC*S zh1Bh$Dftj*+nK48`rs6??tK+2i#MQf*P)c@tVyzAuSBzB1EHBSmQGLi0Oc){DU5q) zWEugK#63zyFIIs4s$j8|cZxa_nz2RF3rwGxgZs93EON3Eg{@cu?|!wiz^>)gcFqwp z_}(h>zsXRL+lxC6gIT(g3`|GcLEg4tYMxXidUYF5WsMRFt_VcQHcMMy(TvQRSE1vF zl~8KggVMjBMalkeqSnMIq%VwMjx&B^$ieR5W-^PW?E|o%8((3vZ~@4&=LrcnR$})7 zN)W!9bAclFR`^};gU>(BsqHTRy4)x6z0_bz_VI>1&q(fvO2nc(6)JTMA;)bk=+;q4 zOS*5P1UET4o--3w?S{aEh8&PAwW3|W^wEFkJXq?Kz`4tpnT}n8c=TNYTXlu!jwgOb zEPuJfudaj%|jzQF$14Z#VrM zkV$U_X)x@;^P{sSzzuu8v-wz6zKi!Yr${!?^B1u+ZDTz4qJhv$^Az0F%q9z0Zz22Z z7B)}*C&nB#g4p}J;ci$ym>uearEiYF>GvmSfN3FI&B&vv2ei??r#y|`=7UcM9U~)y zCbs;`Jm_0r!p7TafZD`gOwc}nFOF`5{R>WkFd`akY@q&$C&qBsm|hg^i@Uee>asX(G89WC-R{)S+m#0exGU2%pt7NMgJVPmPy@mf#4u zA~I2q?~>fmfW%;g^Ne2c)$|)^zg3uXpSYEcu>cT zM(;-prx+%f8_~Y4^{o6@F5a^HgiT}b3ThXhvO_J4aPHgrurcZYy=_f~A+?tHqge*Q z)y2$ncrP@INfF>(5PTjnpE?4Q#q`SA@b%s{blmj}8^-)U`??G}P1a(zZwD)CY{m`q z?z4o$XE7p+Gw9|evs9yARG!Y+%c3Dg-#Ex5cRy6rna?NNugk${#cKHbpf9C1cH=Hy z6~5c{mYThO!#+5yhDQkjAc@&sERV^yw$>caU?0%jb zXgqd+-NJJEz&+BbyDY)u{z8hLohQwx)IqaPqal#)V&sTWXwfu)u7VhreQ%~XEl8Ur zaqpzfQw?FmWu99w`h%8VQ&?NXY36=WfoKqCz5@4L%$CFOr~R@Sl)d~%s zqo8TOA^8W)qq-x{vH0B*@H*1OI&N2^ea(ATJ1_&ktINp$JKy=J*pq^(8#Pv&Lg4u_ zrWLXRire{fDs3D*{bWGJ-DXkq(iwv7>=9t-V-B|U6R0XKwIX|-JE)%b0`+Idg|?)3 z=)P?`OFvW3_jNiH>mA1g(Gnu0ZsZiXhUL2#Fk5R|_`Gro3D}o9XE_QR znQ_p1Qdj5@3|lNH$KD>G&5^T>4zkqcxD-K{Gi zJ$o1#-gN_~)Ei9x+d#;;z*((dLU3|=PtJ{<&x93>yh3YPS=A>d$Tza^8g~kE(F0-n z7-}4N0X?5~qxu^k(D5kWXT{8gWSB$}&oku;--nWuZ#N8U)Tg$H!7PFE%)?{lm}>eN zROmjPs`ySN)PEaGck5uSD-U5u^gT42c?LTsOk?Vyx)}4zm4ult;NUS1%0fa})1p)~ zKF9a0{-d~)Y(LsxpTg2l5=z#ziW#})G^acJ}Fr9lY zohj8nNfdO`nat=vRx?L|RExGS`_>M$S`Xx$QO@kWN03XAC3g@XXBk!TXf(M7=daZv z%jTCXJk^&*hdc7)z5~RCjDyzsC7g-LSjVC+!S)1aGFvJ^mX#4i{?-M(qqCsl=^m6V z>J5r9ew0=;fjpQtbo7rEa_5YPx(L2|)$C68JepE-;ABW9B4=GUxju`-3UxI(P%BFKHW5LCG*rR8@I2;V9p$PvJ- z|8{gTJHSj|A*+q!-UK%fseagWmUm7Os?9ah(irRzy(C%8g_;aw1yxk3q9Q zOLWE6ET-Ckyw>PIkwQPnm0sg4xo8wOa|gJk19kql2Rn9i_V!~53-1l++OP*54M?WgdU6sD$^uUIi|L>@SL9YQhWd-)d$0Uq_0nF8^4hxizRa`LP7INP0BO zitKx2R7_S>hp;JQLH?RL_YxaY^Kp09cCV1-UTH@4TZcu}<3K+A*_ydOLa{I_pH&(tNLzIP~j4!psFIqRtIc?t&l9YssqJ51($l5IG0i(QW8 zSIgh8nBUn*NLnlhxr+^8{;?}ZGlxWO^8;%DHd@~t+CZEtl!iiN?x?v=7x=jw3R)e&N7oV4l`uJ z+=CbhDq!i)GaK4Nq_XsRqVd;Ql>J%FB#T^xpc97Bc6AV#Oq&fJaxNf-ZlE@s6KJYv zOOn6urS_Etn6{L=wLVT3olCnxAlrtuj~q$;SQ+b>_)XC7wGq=egCww<5;ZRFWP0)X z5chlrrSI)a$<@SMYYb5F%y>u``w1l%je(?IA0`QW@e7fly` zz{YwnQ2b>N!8cEc#a0I3xlaYVuC1o@8<&`Vj4nzHlSDtBLu~%EmRvhZP#BoYs&o6o zJ|inI&6z?bC+9(S!!N#Dd&~UYH#3Fg*`VVsM-5Rg*okH>kbll|2Xzb3HT)SycsY{K zDW0wDzmOD84}d0@=UBdBBy>Jnhn*h%DZ@J+<5Z)laZ4PlzqtoHw;DqIz*xNeX(@Q$ z>!pffBy(;L1!a|$WIM%} zTxag#Gv^8+&yGLG?j8UCb2=+tg<`As7Uo~Fp5^*Xh49LQtfXNTWDK53qle9)XyYZU zr635BPHdz>h1}g%{)xFJ96;ZY{mfJ20rOgzg#q0AV!t(7G=4P^suXut==rSz%k!z2 z^jrfh`R7A_{RwPadXD+-tVNfPkx<6-i_P1OMahs9>CI3}km&Xl$xHynfgMT3_NY1|7JooE9RLv@UpFpcg<`-635BC9&1Ot~I>KHg{v zjz89ehk(YwR956#g~i9Zu{%QysH2}h z%Fp0u)4NJg|5FclA2la=tGB4$hqHcr#q*4a19_JRQmm&Q<^PO9YMe^L+GkS3b7M$u z_=J`bJo7O>wmkCfI+DnLk}CGUz^W=;#45X(3Rxc`v3cnR!R%op+TS|I+#iQQXZ3oX zb$p2tmJ=z}bqu*ze?bQ=HL8Bro9g_zmw>xFG*+s^oh!2`I8}?eYv_T@dO0=5M$zue z@{pc$i^<;|1==Sk!&6BVG-o&StmitEO*$_)@+`)%Ba0}1Yba%BR0?AHKJ?ii1?4B* z!D@;f>7P*$jL)8B?#K1OUX~_i4UmxQgXh8>4R1<65C9g(7EoWmK47N25!Fxd&#L{7 z3jG{w!6`+LN>+%RbFf>GbuSY-&ur(M?z3XmqhO2{hk*a&`xrjK2-C-YW7kILk$pL5 zBCoBMYE4`~MNij~A!p2I_t=BYhO7Aw^{nW4FA?*1J23^G$@7q^g5`b>c(*r&rI#o0 z9MExGwgvBrVlS-}xLlh9rb1wZD{hnt+&+q)Z?^*zpt>vII(o+Clh znIt_uk%7dlQizMOAq_QUu)q0R+8n|iCOI$Jodivge|3NvcT5DcZ*S0NRS(iyu$cTq zOjxc`Au67?fcztV*lamc3^rRy`Tryo`!5kCpNHGJ+#3a6M`h3)au!3T8-t?qRMdCu zB}$%gkHW+n!FWJ83Jdr%T<;zWE#W@{j7Q^9FEDl2UQx@}8PaZRLG85Rpd8tY?Ejo+ z&FVIsf1ZZ!uH3K79KK38Ir%`YqLi~dN9V*g>?=YCYR%8&O}6r_?L3W9&88ht&w89yj% zlT*1jl_dH@WBgd`JiMRj>>o`hdToUJUB*y`Q_1v=ABcHLoG*Mp*d{lSf_=L%&A|Yi z?s0aC%Q~=EF=tk~1E^^C9@c@~v9J3W&<=cqe~)TY^y*}3wv2nIGt)3Zza4!;6Ik|d z5Ad=GCy8%TxtUHi>hJOs=leK9NBR~q`<@T;Ecb>kn_QeW*oJ~5dPDu`D@-zCkFIX7KU$9;0eWpqp{;cm`!#hg zs84(>2Is#({X^3Q{jbBs)0f9UBW+`ng4q?ma`T~X`dkWqm4Su}_%q2bM~HrfJ{tjA z8Y+|;6UmIt%`m(#pTQs0!Gs|XS*NZRwCzaX9V7m?qcIthXY%t?L?jf=X+g(FW2nrL zzlW}(n0?qsD0ayaY%Rx8cI6c;&0P=r$G-}-RUz1Ut{=%Ju4TV`_~%uwK={fXdZtZh zF}p%V$UVOX?1z1n7N2@4*8Rtyf02bqJ7gqpZ3msOm+9+y3r-tsP$K&z1d|0SD@_J( z``%!0v4OqwpA6Q^mt*ZOp3vLqK%T2tQs&jS%q;Dj@axfHlH}`^hgSSY&ItzOW9Lj} zJeSqge=lk*)uNjx!l7!=YQfBLEQ1acDBGh?RrPbk;?y#x^Q|X@-I@%=mfS5W$;Q$i zqbcu{4Y)qr$(&Tfn0DDnR`+Zk?JZSU4W?Yga4K2MHw ziEjvMa{QifEtFh;%%||eaj4DYm~3nh^u($7Ymz3sYILJkn?{VzGnaZV(xxWc1FY)N zRcW-9X8=#7J{#Q9`wr-v=K^`Q0Y7@E8y0+P<~(a|m~ zkX*{GaJ}O-S+9^OARiy}^OXM@*yZ1Xh1?+Zn?mKoSAiA6Ez^dtmVvxN)L|W^@{ej%qr@H{Vzvhnet5rhd%o~`SVgQ{Jj^kxLP3oL7oq0|z zXGxY(q*wR~ea&ll@5x!Pn&U&+Z)*e>?#F1fI*U!ymVu-EJr?xX64bSC2zs6jJgc3_ z@%R+TnGglR?F&HvPESx>=|dXPd(g*wJqbaak$JEH-PT2rpl(V5Ner|jX2QD~IqqfR z`HIMkD4BOtXg|rjv|aKfito^D)EZKJ#=q9%%7QazD5qW-$|N0ng1XUDv1(~Le{c9z zl&x4zijPuQ_G*9HH$atsYP+MTx%>hv17GH(IqZMtm;Ns^p#T53CF>7hMx+n;UC{#n&eKf$=S+0EF%B-T zT0;%n?qk^Z8%s7O932bF!pz|L8JAc1|;<}8CJGCRTV=W}~lMEQ=l*#PrYJ5jxrqiU032%#GCL{-Xy%nK}U8V)|7aG-*iKAIu*+Fjod|b=kAofy_Zn6xlBm9 zG#!%fXoKxn56YIQ@H6#e%uDSCS>vZtc}*D9EKK7r${?&6`~{;|&A|F`yM*Qq z6R9&Y44e{zv1X_rY@O1Sjwt8wiulw5hmtk7#LI#HaFn;5_~RpJ%i6 z;1n#+w7&J>cZz*1@Xr?Z{lo&kQ`eyEfMv{D$%PgCu%zT$SFj~n3i)nUFj zo?isj4jBX)RbDLcbSo?VQo%a@I8#lIE@aP}fPHntASrP?I4{pF|ETUNLxR0BiWS8ajny40Ua0l9tEJo}$2cW(%uct4>}gZ?I`f3gGl?208q4 zqd})a$#>2`DqY}48@je&_Up|;^X61>x5)GRRWp!n7l%xV{5@beTQJ4*?L8|IYlWzO>Irju;#Una9~ zVP?NKv*G|hp1BI-_YNh9Zch>e zwr12n7KY9HvIK>xi$QF0<#(%H!l2r|Ap6oKRH^yYqoN3YDzrUGW&OwsFAFf{!Tq zZUXts1WM)4&tvAdPcbX+sF_oZlA1d4D5otQt()GL8JN?qPcF zb0MBad(hYiXWS-7D2_0Ja69b zo#=E-i;_HaNix5yqAX?@HWd9uPU$4qPgdmk{2bk{t}F)o#_r@9$2;ZSbx7NOBpCBPt^TcptnDLr6ThBLlCA2} z(FewX!)9xKe@e%q1B0pT=u#HF%~|S}twDc_gGlDSL;Te+giNEC!ij<0U!>{_o!b`j zj_eo^8^1FD#{Q%@aX!789s%1fsgaL+FvJdtMHdY*+R=bRYu2)}p4{-!$9FAwI8xpa#_e*S_V9gmF>sauOPG+yg z{l5_wcz>EU$h)r~^~9anp{7Qv6$8oI<_y-?JZ3plHbUqT0jhGY3&z(*;5*(|^tw48 zzF!;zmT!=KSnUlmJ}bZcZ764Hjf3h3vuMziV9Gja1lFYk^Es>Lqc5L7UKS}hl6Srk^U)=3cP*ti8*2#a+q)US_p&D82i=34{0fVr;rHU2D~Zh9k=0Iy*(|Td@pWM)ZQd zjaK|j^N$6~mQ(2*&Y$pJNAH;%t7*TlNA84ug6hnER z)mmjHdY(GRU1;e-b7GwsesdHnjtN0Cai@?R!n;uG>oCZP@9(bgzU;BF?5pntvg~mf zowxCfx};WAd}7JnWHVXx884}6w^&Rw8AuH&aollcPL>DGvf$-sQNv{}_1!;%9Bo5L z<`V`Qp5?q_?TkrL77!irge81jMs2nexI^qVsy9AHr!)7k>%I)Kg?Kbm(qM|6Ptm&D zObQyr8N61nFsq*_XZOavmzU|K%6B=3Ppb6M5g<_1t&l+?m*;7!j#S9ru5;zQYV5Foo~NTvvhXwNrxA_0dc} zT%Z1yPovJup;)eKOsQ8j$l3oFbN(ZNlFEyyotc8Z7cC)=h12(l6;Q(O+`%K>u%{{7 zlws?FfkE?N*vJsjk9;F!7Hfd`Rz%sRq1;Pu$#Q%nz-{nyN?lwesP|e!p?>=Cwa$o= zz2foLH#1OeFJjT0iCy(_mgu@n6P@2a#?*It!en_f@Zrxr%hmIs(Sv)qvcgzy&n2W9 za-PM4J>;JLhE}^akcqhpq@BjDvjcqy&xf#?+(JD5uFQXfUjT!;PJyS63tULIMJbQXqjp;o;&SI1KE^Uhz3(tFm zrWuUNya)4{u`+ZzFXujs@o0Z$E$YP2BDFAn&!A@&>P(&DW+^~wcT+02^8oD*RTk~G zj0SztBzukVf|InE$)6oU`h$Lp>O&$ZnY&m$?ZT;9D_bH<%BL(H9Zr9)6eqRpbiJRJdEn2>lxo)Tu~F%4sV%5w_?%SdaUWm2`#Q{udx z9%rOAJ#V4W?5q5lyNF_?1K_?V_d|NjrVoEO%c?F{$cs}a?b!!e$80$!N&RM9cgU1P zgQe8nW>tJQ4FSGpn8dz}D7EJ%vV)RHE{^GNyOvCYDpMFSeu~UY+<1(@gXk<^# z^}unaJm|l$Cim{;7?wAILb6q$apW9KUviRlSC^7by(>LEHk6e0yOV26tx(l-nBbm1 zj@0uwGi0+l%C=1wJL^2L>cBU&zITnuL~BvM;W*Yy`%*zhD16=J2vyO~q`}`5(R%4b zC@wX@eb)t2oH`QBytA>))|%a^T}G1JuR>LaBSYJE+GevrsHz+Ozro@>DR>I}-{ov*X);yKC}pmk)n`;wfklfGF?``@w655S-UimB zb8rMz<)uh#c|K95A0jp%(G{v%3xyW<^<)+kMiPx;TlXaH#_Mh`whqt$#eJ@<v_*#_MKV#N$^HIH^ z2<7)~!m67&V)P{o%J-I$nb*t8q#b#cPA*cr!>i2UcZIfnb>3|lr@GCqZNsw41nUdn zn$QTP-?~*+6c4q#Bk5^pKlE5-?6LpsdTh8vxyd3;93fXZ@=-hKN=+%+?*aAuph*+$ zzgNDybVvMoqR8&aaj5KeT4vX$erKib!7lRgeG2C1&R4obq(R}OyX^e$y>?OoeTLdqAQgR(*NhY(l>8E&RID{M{zs&ZfwByJ1p?Z)>xRQr-7GhXHwmbS;jl1eyp&1 z-HtlV{GKy(Jd8_CrGk);>>2mS3`#vhvz`rv{7MyCn7fnwZnU7vY!kS(>@Hr}bU|uf z=mlO0{s2Q9K&a4ye@FLIlnrlsC2odu)Q#GAszajM5&G}XYy4}zoBdE&&SH!VsQ&R@ zn7HYa=+gV8@GEg8KE7!NrvKK{?T52i=B7lrQGErQR_qdw?B}P91Zx!vt_j1v2`=`j*q3L+1{V4r-ACAw4pQ1#W1w6fWlm^W#;_PC1>bAfi z)`-p&c4iDarnH>ewR_Q3qe-w+DM|Qz@f5TlJPCT*iy`iFKDCVu#)Ya2;Mp@p(%F?p zdPU{9dW$k)Z5FKEP)rKbSAqSq&rB{o5jHNcB>U(^V)?%w@LJLn9KFUu{n=4q_)0>m z84HB<`_*vF`8Y~DQN*q^w4l6pH8ZkQlv=FbK=;f|;c=xC)`|zn>WBwj{gFa#QM0)h zYao@mW3hi?X>aRBC7Z^9!9AsVncK;oXAdsG?h&h74RHSd_7Fw^@`;G`Y0QA zV>`1Nyn|v7IfBOPhZv=q&V~-rr=_b7L3Z9>9Fr4Cg~>&9F8(02?((BWz53CmhIM4a z_k=eG%oNYzCVG@`L5P@pnr&Whh=I%P;aK(n@EW4a)Ls=~`k*}cn);5t>exaL zUXJ}|Rzj!KQ=s^X18FKGp*SUgmO1nnmAZ`}&6y6AyJRQiSkuWDSU|y49!l1mbKnY`SM}mZ+aHZnKByA>BPaw+CuJu z`p2GEuZ01r9(40xH?UdM4eYWu!p<>yRBZ8}T(52;+o4j3{akaY&x39e5*Qre`&0{9$L~W&{k=%KB`3%UvNT$YMPl#Ty z8lqDe>~Gq_=8o5(eIw%G^7*}VG3X$Lj~h<+=M~_~Ut3{G*;wW}b|TChWk$o>Y-szx zFN|sHVaD#oU@RFAJD;2&O|9V&cqIUaoz&MyN@`bSGrY=R#ZN`_nzToowqwrtcR@j%eh_bD^!?J&m z*lyjEv}N8G+j-+Pq@U+2z+kBj9dAEKYdmczL~;y*^b25--eC~>Ud2;adcply$@EHV z1e|;^h3PptP_hER3BDI=wc0^{t1?LOY6c{29097K7jg6ZJghji4L2l~ z5%dN>AK3#l_8t({zE6PH1J-cP+-cISx2LJI4pSfQcUhh{i4HsorIhFbYItld-c)#o zye|qScOT&xm0hI%)r2H7hS9nHC*e+pJvK3L-if>BDd+m6t4_HlF8TDe1nfy#YVKjZ% zYHGX@(8+iYA)!0&?ZaOfxH9e2*uaJ=-rIntgPCF)~tJi(m7H(x#Bb)I{B<(*?JQQ zOw?bm%yXR$6?g>81UnF+S4BK)Sj^y*sATcr8pg41zZu%N%uh> zXYo5;odvhbG9hZq8v4zht;Gd%VZy#NXs|g3*Sk%C`B^cfY}}j9{gFXlWDeDt$w4<8 z8Lj%al_u6dz>jM(;LMDX)NBg21c!CPKsDwtjVKxb zkB>#e16e8!v1!ILeim@^*(LF~vlH|;oX%z$hm%^=OqjoN3aIyAg|hFT*t6T$aZQXi zF6*01dh`B+xnT!r)XI1`^6-PS;(#f9YE* zYBU;yeT*$3RQ|bex3dnbc;E4@-EvrVxCwK2r%=G)2>6kg2Jvd!Ve07R@H}@3brsl> z;Q=FZNZtXO-Lt7E{s?Kd#j{h!!yrAP1ZE9Pg*f>`g3ju_a4E_fFDv~Jm+bx|UD7NM z|9$<&g8NJa%hyYo>^Ku1&}80k>B+KEchT*r`ei#Q;Q0l?IC4IX>}E^f zuW`0E_edJtSOj~nS;0L+HGF$Knk|}hn0gj)U)n5j4{90Db@B0CM*> zW8ZbbLeIi(FmOvAxotavqt%Pq`w^Uhcr*Z}9UBbMyc4)Oeu^-AbT&-d)QgrrQ=^4b z7Q?48&&8J?JlKTsnJ~K}kR6Q}O*%VDXk2X`9Mn+;`eVcFY(ugC%KdO$-xCh3+etev z!p9mgq4R%=T`Bq{Au{G$Ebb`1#w&I~@#v>QTfVH+Xk4 zhw6vzkUGhIE^m%afgU|q39k%~(b4SR?6TD%P+pXYR@SBH)AIn$F}0xb`+krxC;{c| z^GO=APh6fGN!hNaX}#>7$dUaB%PeSgK-Cad2Rk^w*^jplj#F^u*Uf!h~oXkg@{|wU4Mv>>LI@|n>v#NNN&9xO(h(cDBW62QMc4?JT^wja&r}%r=Q;cR z-Wp;GqS)lISrD6|Mz){iY0H~acx!bwI*pk`;iCqFTIf>pD)j=(sJ~bMT6FqV8U(H$ z0SjMmC)uH$bpE^+Eqbj8#A^aqclM;c&oscbmm}=no(duQIn+O-5T@Ra5elESV&#xM zv}C3QWGyg(@$vVi$1i8o+>=I7zkMxa9x$UNAGPTAQCGI@l?4{rjHf27cNj{H60r2D{#chT`=Qr z8l3Uc1p5cw+3d3KOs{M`?Bi~pIWha$(4paE{pBQt4l7}HylZ=OSt6}ib_OO^R*JKY z=D|3pZ)^?!Uamji&oh7?SR+9mSjw)E`w0mepZ_% z+l_<~o&PYv<`9fsLAbytpH6%pgw8uN*~>nM$uQ8F&rLLG?cg@HqkAJ;bYln8D&@Y0 zDid1kYXaf>`OZx1C6nnTupWGOVJELb3)T;Sk?BK0Ww~|5{)=(q{mZY|OUvQ3%DRN_ z*L2AJn<+(XSjJWy{K@?9#+3I|U5#IO4pR5&C>Z}bl3lOu3DZ`!V(*^EG2}vTs#X~R zmKB>gV{iaDC2K+NN7lmdYg*W9@ z;L5A5*imSXx`iy^d=h)=wVz$;{*c9` zs?dr-{z6(bKMOh*qH&*ID6;;v6h}aL^H%o4g>%q&Z`$|ONbsN6fS20j#I5T$vD$wp zrTdg6oYQ-ZDc1NfNu`CbBB}tDHa-#6F`hjhGC-KAFo-|M~*LB z@zQ{A_&UH0jEZGSURzI5GZyHqFMt3ar*3@@bKXj)_%g4wf@oAe|CGm)^f zpI4yV6C-Gf=03i~y3iPVL>g1a*=YOHSjx_k*vYdk_x_H-5iuHIJ+zSBJ};q(IhE|N z+!g?cJLx%Uk;SNe|CCc`KsS6>L|Wz3nU@^olH z0{&lV=l>RC6ZYYaRFYJZBt0u3L@PBQs&RjF|`JCtF21|P2z(siV;x~@{X+w2)9ucb= z{{^<5@@FTHA}42<(5=5U__VS6@S|OKg8PHGio~8IH(ZUL5k0Ag<~meY$8zJJMZnXZ zWB9A-r(mJ}PsTb_fTqVNYVR-+%eQNT>G|G#pzjz`Gi@mb1|Q}99(?64OFT1!wZ!`I)h9A%&`x#g9=`qGv z?&Sws4W-#-X5?JiXkrm`5hj&f0qqfkA$5c!`XuQ==a>CBW&KC&v->6}dJ zc=dzO;xp=(T(H2DL~3*^4otjC&i-{K1wp-v2c8~L+~dMgn%>=xOtP3Hv3wg#);2uB zn~fLfeZ3#JrFt51;}_A=1K)5hd#_JUmtkB5nwMHqn%< z6g3c=yd-k3ES~snspk**)uWHkD9Mr2D*7m?lv~kH&0TE2Oxt%{qxpL%OlPxp&Gi}7 zsR!E^QHDD`zR+>kXOa<i|*qC`f7TMjNgSrDpf%67G~1$v^QFa{FxqJJvglP8HDiA)BaM)N%Ax z=~6x?2Q#cjkRw_~q}$^DtY7{cZ_UW3qxz4r>(lp|# zC8m#*nWS&^DH8bRDz*G>MjqQuhtCI7N&Eh@^zG|C^tmXLSj_H4gWTNd$d~KEzG*nO zF{T48161D7R9qiVvRB#S@dt_Y!;CEQ({(I8zD}Qf&)-4iwti%r*CaAA@HmF;tA;5b@&hsbpKD@zoH=g3Q!s9q7{w$d=Kb8Ah5J&pzO6k{O z-yn3%c-pWgoo<|zOZwb&qLq+L506I)fE9sKMkU@zZ`{x z5BVfA^)2V}auF>*S&x5{F455}$3~L<;a-LIzpP5Q9s)V} z)W3}DaF!AOI|rCs`6PL&V)=v92_&^g4vn!phprFJ$j1(A>c0099A23~Pn7JTUe6c8 zve-1bWx*^sA09=fr%r(Vsi(=RuS1A_rZMgAlMGJ|f5QKYb`f*G1Jp%Tg~xRF;ZOGO z`C6k%cEm(dqC7&q9!8TdYa;1p=?O9l6M2tKwj|@<>&p70Y)ws;kT-5$@sMaX{+@l7 zt~#_BmW-Z7cAgBQe#$)Rmw1?zCq|O@eQTiF*ns6w7SaSSU0AbLXQH8zfSfwH8eGGs z(QvOIYWwsky`A2N1l0XQTRS<0a{^xH11E*vT8Y(*(_6bUk9ae zn~2EH8r71^(4ohAjK%X&RLNXKnF5*+XG6IbmO)C;Lg_0_Y$o-PGgyllm|e!ow%vT? z*hVh;r^V%8I>FI%7!(@qMIWQNRJ8mP`#e56eYNjL+*&x|lp~;?V`h;E*8M%t z@^CTDIp{M}gCuIEfhaZv150;My`>t|OhboMikonCwE*ngF+;&o6bX&f97(I(g{EcbW5=&zT-~5e9UnEr=1;Rp@Xx_SaHZKv z)H{rJb!TsbcaC72X)p;W+DdVdDU7 z4ygq7bAM`j<5{8Vz+6)9o&tYIEF)Dh(bU&mgA_SG7PqD4MZ~q&QrnsU22i z?aD(mdOGWD-@A`*zKla%!_P3_QW{w{Z5lV}c^c6U*hD9{C6TXD(KOf~fm*zOja|Vh z(BA7Hom)_bb-I*$%6XA3W?}TB#p3_-X1M)R+ieeVx+c_ng$RT{4np$?mJ6h>Ab)}pjTpF;l$@AO!cEQT^H*y~ zVAN}j*%^vndT&t>{TrRzb`bgXb7<;niWNzLM7HKDDh!>t+@1<_na;R>s*S`*#PTm5 z1Mrs7A}T0xMXRl+!NrPYrq+Hf%6w``t5%#uhpgEofptYipR1u^g%eiW*piTjVT3f# zC*F1=Na*5j#CXkj@SU_1rY_7y;e%4X>|zhrL;b^_9~3|hCKp4%pu@lg450=xZ_JUe zAaeBpOv*is9aS~pQ5c3chJT^?of&#W?<R5p{UGh6X-eNo9ww^4_I7%=xtl$0W1N z$JW1~*}>)-_kEn)D35piW(?s^?tp*5IJ!Z|ayR$p5%rDrXg@X|1J)jf++obC7d(l) zYGZFiN+VF@mvYXcaF(~84d(riV(j#6Ok&JjFU2IH=;+VugneYWmbp~5^Db8ZGa8bu zhEtbA{U~E0pw70rfnwsh|OS0^eewS-JXB}U&BQmt0T#_$_S zsvj9rvo#}WWo=^OU`86Md(k*UYx2r!BlAPD zeLO1z3bk&4Vp#|8zJd9m42@{^BPWc#=uHBrvOZY1+t{MFkV-4r`eQqg-rB8COtZsS zmT(j=Fy6>H4qAk=H7TH4trQozvmBK{7&bKJu-_^3v&@%a^E7K(oxd9*{Pd`KTqhQ^ zE+t&xRw7FlFh}+_;$|CzQrApn?zbS4H)=d_+Y^b7mj_^)!#OS}o%Jd;z1iEta@uKl z3Ob%mCp8_PAz(`%5^`Y*QTMIK=;anvu)I?#YG>VH!57#d+)S((C#5!iCW+ZI41G%0 z(Zbt05@QI}=nFDc8FMrble z^Exn%&qLLTbO_lrk>m%=C!7EEq)q3N;FX1doG;i&bk?|F%>B<8Kl&ji{5OocOz;E2 zI3F%&gfsO@Q$yfBXO^+fM!m5dRgB%GEL^gl`a0HPM!FfhcVB@URu~XhKOU}>zX=P zgKG8{H0>CO?tcf+%{xq~L$A4XA8sKgBfF8LtNPT{v?q~fTvG=VlxYfeo&9XapN&YPn(!J%tcj1 zjW{=IA$nR&AR|NiGjCTer$4$sP1BK6tLj=%HMo+=SxUqoUVJ0W!4e3G$iBx!Cn zCZ+0An3i41`OX?g9Rub-^>0(Ctd|fk)}kuGI8G=Dpfy3=N#L45@YiR4#y4Ih&?6nAZZb|x zKp&EJ!iD!w?M}49ZAod_1o$N!K%H$j689O4NqmlgV}yc6pLCM<@_~c5<;sPkCl@h(pp~U2dn5u*4a}5*b z5fAn@X72nG(^lSu4$%xMx<3KWPhLg+EJrg>*B)rP@)PS=zdwfc&PHz2BBG8)$X%EY zvD&UgKA@88vRp`gbalzp-ZwChF*b}#!fC>SO@v>J=(@w4&6#Ylg{NU}h;I1cG z(Zu>d!G=_5$kveS?P&1$8U$B(kZ|Tk?zGMVw`g7RwR{o@h#EjU_Wnd+x3L&QXA(!@ z0u-cl6IX{V1oN}^v0=17+SQ7P*8IVwnX%hMn{RTpxmvVh@j@!FY$@^{n8EIqSdOrE zEXmD(4Wjo;spp_Mv})FGwC6i8xQ%&bI%bm^>*HwmFobv=y9}F+9I4)lT9C&c;8qup zVwo&4uVc!bB$lC&INXBB=L|=;F%qhCtsRZn+jTzAJbz=~awX^PLB*%ObeNK5-)v8C z%rHxwQ>{t0`(rfPG?WOv_VQK9Um-C$8k5ehqq+%mX=&ASbYndX>7#TmddXxe-DXF+ zMvkX8JFhbymJbnl{ddFKZXnt{vLp%F1F2a%%PH2!qV=49RPkFbRyqBE_%Howeff6k zSU7^VCcJ{&x{nyu22{4)ifgSOK{MkHp-oL5He~gtc8iA-;k5-^hY#z6AV+LOQ>n=L zIyPPRA@Q!8(AWG2-bxrttRDBK8F@>H;B5t;w|zX3_4$p$m9IHtbqcs}uNc?wDNdd- zm^MlbiFK75k^NealXqDWp?WnnIlw$+RocYhyA4=hb|>ncgCQ=$h3u;vLL27`Xyx*) zWW`iJYEWCsbMoWp{34uI>dgQ>PnPB8A8@KwjhvsQ32pXO^VZY6uS$;7mP5aG!< z$e(i$q;pEZ!1z6lMDAjw~V$cnY0GHMVcU9&^M=viXZ z1F_JoznsSZR6<+pcH(V2hlnQjfLNuF2)`NA(0f|6;b1mSUNV6SO@{I+o3EJL)`lTI zZ4h(y2ILGMSx+iwC1R}pESh6jgxS-cV7WsI zD#rC>{RVc&%5t@_cL!5bxjE!bUqg}xxq-`54eYitm^%3MWAkh`BH*TQ()o>I^?_B? zsfG1T54e!|N|uLbk<}v0fzL%s8bRIU1mT^yYC0B(hyR!U_4di6LGXU zlq9V>hbFfc(T-dfsLmhE=EE(-=Zgz*&RjxNqZ&Z9P#5)wjG}6-t;E3a4i`9UHPzg4 z3`85g@Eua-X9(3s-_va78a>zSL_p`Lrr&R3>753sP#4XPJaG4~wfsYJ!V(Xge& z+l}?tYy`A%<3?h0q(7~WD8i<`Ds0I7105;jNyyl4)HMAfW^P$TIQL$*xxylny<4ZBd0{rUn32v`iPQ~-Uz_Jw=(qF zs!QiD-9{W8x6!&~%s)}K6a&0%sh-@4+I^lznwYcNYV|KPdaOgjt1U_F9bam^`y~e5 z+fG$U=TY$CATQjyi*F9-271gv-L-TZ)pMCcZLL@{TfIK^f^n_ZcM_G~hz_mMdk??iG6K1Uh; z;SeflA6{6dNjwrZ6O;3vG-bXyDOxa@2siX*9W!(A-EsnA!YpZ1xA9cGw|8=7U(agKa`a@-trqgoPE6^IWgG&4D;q-lHlE}p)savlltjB-D z>D5IGD)jRQwQ4a|-qIoE%QR?wj0iJz8T+o63DuBj0wss5c9L|_Y zv&NIIyIZJC$DQ|>sSP5><>)wmG}T;YLd`mKh$zmF%3E0GsnU}Qzqs>bMolLw-%e=C zIE}Gg{v_(*MADIDLhN(q(%j2>RItTJtgh$}DZ!eozivgEEMq|+R27xJs=?=1H-@A5&iS|}NN37cP^;`O&OCgR_V;G^qA z8r}b)83{JD@jq{pw{`~6Sv-&W)}CRVz2U^4b@@aQ;plXVb!w+MVBrk{fsDOby>}K- zud4v*Hg7)ERz!7o&mpB64Uj#6F~c;SKv>cfrFCP_qOUy_{A`-k>WD1*bar81X?)utiI$PA?Tl_EF^MZr%MJ6UU zLA2)v-+Ymsqc=vdcWf;#@tH2PevL!pjCH6O^rtBEzdskHXCz9iUs( zhB-OYpyvJzmS>khyyYoA<|gAP9+}0toVv&xj9t%}Dp+P+5ht$qJ`9;r)bH z%-fT!6mFP>8Oy4$Id}wb6kGx>nY+N=w3)&+-e8mz!&_bfEH936>@cRx9Qe#WUDUr<0_Ko?`&t!SX2n)nCo^cvN(8uaud3^5ivFP#dC@$bwGpGhm zK%<~e$Qh#od955nKL5tf*An2Yd$NAv0hC&LaxtI@n%f^ShZyivQx%YC%4Vj4p0G77G>-4~8Y+u8PPpv=1eGpC;gBe?d&xEQ+8Y4!30|Dbu_?@*ahr>k?uCGWE)*s< z@R{jr&?d10RBB)GlSKs>e`6CCr0hpw%x6BzhMleU14w5ED@CWOAa5n}zKzO5O@oi< z;qjTzzPblx{p!UrWop)M$-@nkzk<=ol|g)e?!xA@3JHRGQNY$s%8jp z?F7?(RhYXlkdr;X!rR>|fWZFyxzgKT_`;a$s2_Kn#@DL3P?5 z;8q)9*UKa<{nW;lTU^I#@gKhQLku`RMlR{+bzU{5Mr^qz7v67-gOI6#P!hPGG2#>$ zzx)liFQyaI{5?6v?Lw!tV`6mKpUc&!wqWPm|G=a4H%9JIKy&vT@EtrIogN)y_q_v{ zcjPxD=KDa5uP+D!)K02{72KK9f1!LxJ*X}*Ce)1HAiR5vlc@(m=Ctu_7AXLOGBw1% z%YcHiGzgwn0ttr>gFE~4HCY5*iUlaR6)u(=o#Aqp8iDb`EjU%?if)INLhjBDkbBRb zlWx19tlqAPvIzysurKj&bY?UNT8E3JAzv`WzZ+KWddiW**HCEtNi1w;e3+Drpy@15G=rK;!i5=XTAp2 z(ZO8VwpPq`cZ9Mfg{XK~%LVQ7goY7c(Bl#7KxF)c%v*oZ`&I)dFTamP*(cColocn_ zOi+f@JcWcsQ5ajl6Ox2yxXdY|Kr4-^7?z|ICqCDbbJHSXPQdywGkkDzL0Bu z!nhFY$MNPzLm^w|Ja7H)PM*;npy{7j6#UxdWSsgG1p$jE3aASRrVQi*GOXd!%tPqA zbvImMeSbw;6zE;;j^?rsP%JG+4`n_Gk6Ux{hbCO*ur5B(^DFOIz+A0o$3bw9s~}jV za!UK>tawG|3lJ)_KyX9B#U4L`o_@c<=DrfL)9*ohn459aCL(^ zp`^zFxI6kdn$$55*#ih zxoERcUrz-tNv)hT$6Z;N`kKr96^RY=;=!n5KUB4M;Yf`O_`JtS=*l^TbtyYg_{;>f z*nVTSG8ILqeOO;tA1b^5{<^}s8$>R!FTCR!KWld-x z9{|ezzd;bie@_C1!kMM7mkGXLsk zCx{q>s&rW#FK83;BbVNR@=aMNGxX%7FD8i-7~d+&CJ;BNXJA{2p zt-<4W7Qb)C0k%hE;IE%eP}zKsd;jAeB+1VJ_jx%gevRT1m$0*wp>L5q{ssD1pN3L# zGIxH!H?}_8qG?ecU-yVP(|+kelYAShdQh=_kNN2JG9J}KOJJn(JCq2rAxAPC6_-cx zDX*5ImxBacyw+o7(*s`C{SB%tCo3~Q9><`vMD$&30Q#f8K!dL)>gb$ccj;?5lLhBM z{!Ef~tYYwG12u+xBX9yk1jmdr%VSYC?i73Y+ZVb$Pq^*LWu zcoX)qe4wB?@J7lrVBXahWwv)LMtI~vbABe5=&sK)hIi4bO@{KrGhph;C62jCazeF?xYs0 zJYRt7!r3Wislqy~jR1 z06UG;cydr2suRzPL%B3aS=@$te|KYeFSf^gd%y|JPV+gJHe$!)M-cq@9RAYD#nrv5 zuA3-iGzz~Ba=xykM1`FA&IS4o_@ijD9 zehob44(7^*y@4ON7+3Fo2A$O(z__Ik>Xy6*ll%wZ*j3LL>d(O-J`8F{J;Pt8(!pxd zew_a;2d7?YM(GkS?$szZS0n|onYM(Bw;c;Dd*4Bk>Je6&E#+1-PPjnN2wmSOv2@QR zl<4(`yPpaj_ac^p4{d9d#8Xs zW3nigUsA4!P=n}KD65(Nhf!shVDmZV{#%zK<~{^LZN)o?j@t!}rd$Hi33I-6otVvu zf!OeIGWw-oz(m_s;2hbC9>vf2n3zcP_b6g>1dn69UW1L|H*!_lpsOfmR#0t)4u(cW$sln-!*(9TPIoca+se2!%6O+B7@il{D2RTkbk$AzCB0y^(c zaDt@4oN>PkP%~sVc<9M^56e*yGxIC&a%DAc7?j4Cf~8P1EDRLeY?ZRU8-RS@gQZci z=&@omZ?Bz-`O^w8#I-jjxkqz?4ogVsYQ+w_|DgJ>KBk>HCGHUR$Ixy$Tv~-a937en znX@7>X~1t(f4apD*d%nZV(aRUfxK*3C9h5#g3-~8@IwOPltCmNKk3PLXLf6eL_JC^1P6tD(u<#^(Tn|VNewZq`5dK~AEM!`- zMTMb)Xi`UHb+UX_@?lX@6)+ z@OpA2UQ92n?nOE-*KyO9t^%E<&rrEy5l(zL07BjuQjOD#Y5#uh*tVgF-}`AXq)%S} zdxZBvP?LhYm$buzUAyVU@3wUK-kIcc+X7lWzFV`L;iw*%aM*pGj4$(K&8=|y+={DQ36B66m^gm@pyB$9^x^nCAP0uu+2Jx5Q9 z@BH#6gU{W;exGbP)1R!5^w^VpPQ6D<%r?_5k9lN*ZVcbWr_ePr5m{vL2mY2XCku4g zyr-9iCf|76GDnYmnD!6-yIY%X-lI+ZJ!6PVSrzmfnobwZT>&jG2SSd$lqT=EN@q(J zpZp>3&71IIrtP3^d)-h`RJe@kdTT3=h*oT~|7cEK{Lbm+0piL=`#eM_l(A5EH z#IKJXt&vBO;c3H3pqP0Zj@>3jm7mBRy%Qv%ZW3Qov!1pr^rN%q2+3M~OH6Pa3HOF) zBNi!(EBgBtf0S*N_^w|h8TL#0lwxuOEA8_8L7fIFJtBj!;LvP)k ziL2+$A#W@lNaJ%?-q0?N@d3R^{SF}hnZ0SnwosBd$CzjyI1g^2ogi4*&ec^`fM|0C zc6;YVH@wlKvK=m@`Dr_Ezfnq~n5U*;g)dPYO~K7_0S#kL7I{_|Bxx>2{!kG#J89Al zXiQ7@6hfHBL>j-PiMbu>L1e{ny;Ijg)gZ>mxW{;lsk&U|u0)hQat6hLE6Q+pKT;Uo z3GpAHm4(A0&c5wMt^Qo_uIb#|Y zL4JiNH9cRB85rA~mO*HAccPlGj?YPulBgRE zsNZKIDOCAVfrqj%Y2QOMc-N2K&0K$oJ17!cp^Sp5Yo&dw3dsg&wqW|7EC5b_9bsGZJg$PW8s>kro_7!rI3XpMU$( zWD-tYA3GAiLKFIYfS3q=o#aABFC~teD^c1{1vyKuV{_?#zT&k9jXGdKLzien$C@8l z8efKidmFK|w+1~rdpi|uAEzw(uMY`&x0M9#+(^21j35Tw2<&+2PV683hHxtbdj6FY zouA@DOieFSohBdaCjy)soqVG#CPJZv?e^;Xh+X$KId*W z`cZwK%XIYB<BJN1=nXx0EwAC&(qlxJnnU-R?!as0n{taz1ZmdD!;rap%>* z+%BSCb|&nP8$qoP#L|Yu08+sd(&o7uES7D?<;OQsn=T7{B)x^)kmV3jHXFkX>`C#F zp|oVBlmqGnnlWE<3iqupHQ2pUGb&vye&+p8F1Xh`I*B!tmtXkjYL zG-!s>Z010>Hr6E;Rzrx-)3LN}&_vSGAY#1gftahY5C!ii^E$`oL$x>;1+m^#Wn0P@ zruHRnk^P8ZlB!56ZYznb4<;qAPC~^_Gh#08Nd=B2PMT>lqVi&$_iXm2JJFhN4LX8S z4KqxB&Jo4IcBR>fb#z6*LQ>viE@_Vak595*}|#<3~oK zjloM$Kh5Jj-Y;eMSAImbBbw`cb_J@gw4v|ky{I-CMbh*Zf-G|!4SlMO9<4J#X2OGG z=x6Zw@tBi|40!pJRA_y363ypcK*6;f%p9&swXVz~vc9*lde#miGJlA1ye_p%@TbC% zCA^oY6J%#n`8qj!(^@eUT~nsfjQO^tO>a3h?dpk@EU)(K!#L*QT0q?DzJXQFWoVvv zm-Cpd