IQ1_XS/IQ1_XXS/IQ1_XXXS quants, x86 IQ1_XXXS dot product, finetune + YaRN server fixes

common/common.cpp

@@ -1642,10 +1642,21 @@ struct llama_context_params common_context_params_to_llama(const common_params &
     cparams.n_outputs_max_per_seq = std::max(params.n_outputs_max_per_seq, 0);
     cparams.n_batch           = params.n_batch;
     cparams.n_ubatch          = params.n_ubatch;
+    // Hybrid GDN rollback snapshots for MTP/EAGLE must stay in one ubatch.
+    if (cparams.n_rs_seq > 0) {
+        const uint32_t need = cparams.n_rs_seq + 1;
+        if (cparams.n_ubatch < need) {
+            cparams.n_ubatch = need;
+        }
+        if (cparams.n_batch < cparams.n_ubatch) {
+            cparams.n_batch = cparams.n_ubatch;
+        }
+    }
     cparams.n_threads         = params.cpuparams.n_threads;
     cparams.n_threads_batch   = params.cpuparams_batch.n_threads == -1 ?
                                 params.cpuparams.n_threads : params.cpuparams_batch.n_threads;
     cparams.embeddings        = params.embedding;
+    cparams.ctx_type          = params.ctx_type;
     cparams.rope_scaling_type = params.rope_scaling_type;
     cparams.rope_freq_base    = params.rope_freq_base;
     cparams.rope_freq_scale   = params.rope_freq_scale;

common/common.h

@@ -451,8 +451,10 @@ struct common_params {
     int32_t grp_attn_n            =     1; // group-attention factor
     int32_t grp_attn_w            =   512; // group-attention width
     int32_t n_print               =    -1; // print token count every n tokens (-1 = disabled)
+    enum llama_context_type ctx_type = LLAMA_CONTEXT_TYPE_DEFAULT; // OPT = training context (no-cache attention)
     float   rope_freq_base        =  0.0f; // RoPE base frequency
     float   rope_freq_scale       =  0.0f; // RoPE frequency scaling factor
+
     float   yarn_ext_factor       = -1.0f; // YaRN extrapolation mix factor
     float   yarn_attn_factor      = -1.0f; // YaRN magnitude scaling factor
     float   yarn_beta_fast        = -1.0f; // YaRN low correction dim

examples/training/finetune.cpp

@@ -14,6 +14,34 @@
 #pragma warning(disable: 4244 4267)  // possible loss of data
 #endif
 
+struct finetune_cb_ctx {
+    struct llama_model * model   = nullptr;
+    int64_t              save_every = 0; // save every N train iterations (0 = only at end)
+    int64_t              iter    = 0;
+    std::string          ckpt_fmt;      // printf pattern, e.g. "out/ckpt-%06d.gguf"
+};
+
+static finetune_cb_ctx g_cb;
+
+static void finetune_epoch_callback(
+        bool               train,
+        ggml_opt_context_t opt_ctx,
+        ggml_opt_dataset_t dataset,
+        ggml_opt_result_t  result,
+        int64_t            ibatch,
+        int64_t            ibatch_max,
+        int64_t            t_start_us) {
+    ggml_opt_epoch_callback_progress_bar(train, opt_ctx, dataset, result, ibatch, ibatch_max, t_start_us);
+
+    if (train && g_cb.save_every > 0 && ibatch % g_cb.save_every == 0) {
+        g_cb.iter++;
+        char fname[512];
+        snprintf(fname, sizeof(fname), g_cb.ckpt_fmt.c_str(), (int) g_cb.iter);
+        llama_model_save_to_file(g_cb.model, fname);
+        fprintf(stderr, "\nsaved checkpoint %s\n", fname);
+    }
+}
+
 int main(int argc, char ** argv) {
     std::setlocale(LC_NUMERIC, "C");
 
@@ -42,6 +70,7 @@ int main(int argc, char ** argv) {
     llama_backend_init();
     llama_numa_init(params.numa);
     // load the model and apply lora adapter, if any
+    params.ctx_type = LLAMA_CONTEXT_TYPE_OPT; // no-cache attention for clean backward gradients
     auto llama_init = common_init_from_params(params);
 
     auto * model = llama_init->model();
@@ -66,6 +95,16 @@ int main(int argc, char ** argv) {
             ggml_opt_optimizer_name(params.optimizer), (double) lr.lr0, (double) lr.wd, (double) lr.lr_min, (double) lr.decay_epochs,
             (unsigned) lr.epochs, (double) params.n_batch / params.n_ubatch, (double) params.val_split);
 
+    // checkpoint configuration: save every N training iterations via env vars
+    // FINETUNE_CKPT_EVERY=100 FINETUNE_CKPT_FMT="out/ckpt-%06d.gguf"
+    {
+        const char * ckpt_every = getenv("FINETUNE_CKPT_EVERY");
+        const char * ckpt_fmt   = getenv("FINETUNE_CKPT_FMT");
+        g_cb.model      = model;
+        g_cb.save_every = ckpt_every ? atoll(ckpt_every) : 0;
+        g_cb.ckpt_fmt   = ckpt_fmt ? ckpt_fmt : (params.out_file + ".ckpt-%06d");
+    }
+
     struct llama_opt_params lopt_params{
         /*n_ctx_train     =*/0,
         /*param_filter    =*/llama_opt_param_filter_all,
@@ -83,7 +122,7 @@ int main(int argc, char ** argv) {
 
     for (lr.epoch = 0; lr.epoch < lr.epochs; ++lr.epoch) {
         llama_opt_epoch(ctx, dataset, result_train, result_eval, idata_split,
-                        ggml_opt_epoch_callback_progress_bar, ggml_opt_epoch_callback_progress_bar);
+                        finetune_epoch_callback, finetune_epoch_callback);
         fprintf(stderr, "\n");
 
         ggml_opt_result_reset(result_train);

ggml/include/ggml.h

@@ -430,7 +430,13 @@ extern "C" {
         GGML_TYPE_NVFP4   = 40, // NVFP4 (4 blocks, E4M3 scale)
         GGML_TYPE_Q1_0    = 41,
         GGML_TYPE_Q2_0    = 42,
-        GGML_TYPE_COUNT   = 43,
+        // IQ1_S's 8-element ternary codebook with a narrower index; see ggml-common.h.
+        // 42..63 stay reserved for upstream (42 is already Q2_0), so fork-local types start
+        // at 64 and a stock build rejects these files instead of misreading them.
+        GGML_TYPE_IQ1_XS   = 64, // 1.4375 bpw, 1024-entry grid
+        GGML_TYPE_IQ1_XXS  = 65, // 1.3125 bpw,  512-entry grid
+        GGML_TYPE_IQ1_XXXS = 66, // 1.1875 bpw,  256-entry grid
+        GGML_TYPE_COUNT   = 67,
     };
 
     // precision
@@ -475,6 +481,10 @@ extern "C" {
         GGML_FTYPE_MOSTLY_NVFP4   = 26, // except 1d tensors
         GGML_FTYPE_MOSTLY_Q1_0    = 27, // except 1d tensors
         GGML_FTYPE_MOSTLY_Q2_0    = 28, // except 1d tensors
+        // 28..39 are reserved for upstream ggml, which is already using 28 for Q2_0.
+        GGML_FTYPE_MOSTLY_IQ1_XS   = 40, // except 1d tensors
+        GGML_FTYPE_MOSTLY_IQ1_XXS  = 41, // except 1d tensors
+        GGML_FTYPE_MOSTLY_IQ1_XXXS = 42, // except 1d tensors
     };
 
     // available tensor operations: