[feat] tombstone evicted dynamic embedding keys in delta dump

tzrec/main.py

@@ -79,7 +79,12 @@
 from tzrec.protos.model_pb2 import Kernel as KernelProto
 from tzrec.protos.model_pb2 import ModelConfig
 from tzrec.protos.train_pb2 import TrainConfig
-from tzrec.utils import checkpoint_util, config_util, predict_util
+from tzrec.utils import (
+    checkpoint_util,
+    config_util,
+    dynamicemb_util,
+    predict_util,
+)
 from tzrec.utils.delta_embedding_dump import DeltaEmbeddingDumper
 from tzrec.utils.dist_util import (
     DistributedModelParallel,
@@ -689,6 +694,12 @@ def train_and_evaluate(
     is_local_rank_zero = int(os.environ.get("LOCAL_RANK", 0)) == 0
     acc_utils.allow_tf32(train_config)
     enable_delta_embedding_dump = train_config.HasField("delta_embedding_dump_config")
+    # Arm evicted-key retention before feature/model building so the dump's
+    # pop_evicted_keys drain has a buffer to consume on every dynamicemb table.
+    dynamicemb_util.set_auto_retain_evicted_keys(
+        enable_delta_embedding_dump
+        and train_config.delta_embedding_dump_config.dump_evicted_tombstones
+    )
 
     data_config = pipeline_config.data_config
     # Build feature

tzrec/protos/train.proto

@@ -108,6 +108,16 @@ message DeltaEmbeddingDumpConfig {
     // even embedding_dim.
     optional DeltaEmbeddingQuantType quant_type = 6
         [default = DELTA_EMBEDDING_QUANT_NONE];
+    // Publish all-zero tombstone rows for dynamicemb keys evicted since the
+    // last dump; the polling processor detects them and deletes the key from
+    // NvEmbeddings, reclaiming its memory (FeatureStore has no delete). While
+    // enabled, delta dump mode also arms dynamicemb's evicted-key recording
+    // (evicted_item_mode=RETAIN_KEY) automatically; the retained keys live in
+    // per-rank GPU memory and grow with eviction volume until each dump
+    // drains them, so keep the dump interval short on high-churn tables.
+    // Requires a dynamicemb build with EvictedItemMode; older builds degrade
+    // to a warning.
+    optional bool dump_evicted_tombstones = 7 [default = true];
 }
 
 message TrainConfig {