feat(index): make the 1 MB max file size configurable via CSP_MAX_FILE_BYTES

.please/docs/references/semble.md

@@ -165,8 +165,11 @@ Same contract as semble `tokens.py`:
 - Content-type partition: `DOC_LANGUAGES`, `CONFIG_LANGUAGES`, `DATA_LANGUAGES`; code = all minus
   those. `get_extensions(types, extra)` inverts the map; the **`extra`** param (custom extensions)
   is a small Rust-side API addition.
-- File gating: `MAX_FILE_BYTES = 1_000_000` (in `create.rs`); empty/whitespace and too-new
-  (mtime) files are skipped (`FileStatus`).
+- File gating: `DEFAULT_MAX_FILE_BYTES = 1_000_000`, overridable per process via
+  `CSP_MAX_FILE_BYTES` (`get_max_file_bytes()` — upstream `SEMBLE_MAX_FILE_BYTES`, #252; a
+  malformed/non-positive value warns once and falls back). `create_index_from_path` collects the
+  paths skipped for size and prints one stderr warning (count + first 5 paths). Empty/whitespace
+  and too-new (mtime) files are skipped (`FileStatus`).
 
 ### 4.6 `indexing/dense.rs` — Model2Vec embeddings (real + stub)
 
@@ -316,8 +319,8 @@ Ported faithfully (`LazyLock<Regex>` for the static patterns, `RefCell<HashMap>`
   a local source root is read lazily per returned result with a memo (misses memoized too), where
   upstream `_compute_file_sizes` runs eagerly over every indexed file in `SembleIndex.__init__`.
   Git sources still capture eagerly at clone time (the temp checkout is gone by search time).
-  Because the read now happens inside a live search, it is bounded by `MAX_FILE_BYTES` (the same
-  ceiling the indexer applies) — upstream, running at construction time, has no such bound.
+  Because the read now happens inside a live search, it is bounded by `get_max_file_bytes()` (the
+  same ceiling the indexer applies) — upstream, running at construction time, has no such bound.
   Decoding matches upstream `read_file_text` (`errors="replace"`) and the csp indexer
   (`String::from_utf8_lossy`), so a non-UTF-8 file that got indexed still gets sized.
 

README.ko.md

@@ -407,6 +407,8 @@ csp find-related src/auth.ts 42 ./my-project
 
 `csp search`나 `csp find-related`를 `--index` 없이 실행하면, `csp`는 소스와 콘텐츠 선택을 키로 하여 글로벌 캐시 `~/.csp/index/`에 자동으로 인덱싱·캐시합니다. 다음 실행 때 캐시를 재사용하며, 소스 파일이 바뀌면 콘텐츠 해시로 자동 무효화되므로 수동으로 다시 인덱싱할 필요가 없습니다. 무효화된 캐시는 증분으로 다시 만들어집니다. 내용이 바뀐 파일만 다시 청킹·임베딩하고, 바뀌지 않은 파일은 기존 청크·벡터·BM25 포스팅을 그대로 유지합니다. `--index <경로>`를 지정하면 그 경로를 그대로 사용하고 자동 캐시를 우회합니다. `csp index -o <경로>`는 명시적 영속화 전용(`-o` 필수)이며 자동 캐시와는 독립적입니다.
 
+인덱스를 가볍게 유지하기 위해 1 MB보다 큰 파일은 인덱싱에서 제외됩니다. 제외된 파일은 인덱싱 시점에 stderr 경고로 앞쪽 몇 개의 경로와 함께 보고됩니다. 생성된 대용량 문서나 수집 문서를 다뤄야 한다면 `CSP_MAX_FILE_BYTES` 환경 변수(바이트 단위)로 이 한도를 올리거나 내릴 수 있습니다. 값이 잘못됐거나 0 이하이면 경고 후 기본값 1 MB로 돌아갑니다.
+
 <details>
 <summary>토큰 절약량 보기</summary>
 

README.md

@@ -407,6 +407,8 @@ csp find-related src/auth.ts 42 ./my-project
 
 When you run `csp search` or `csp find-related` **without** `--index`, `csp` automatically indexes and caches the source in a global cache at `~/.csp/index/`, keyed by the source and content selection. The cache is reused on the next run and invalidated automatically when the source files change (by content hash), so you do not need to reindex manually. A stale cache is rebuilt incrementally: only files whose content changed are re-chunked and re-embedded, and unchanged files keep their existing chunks, vectors, and BM25 postings. Passing `--index <path>` uses that exact path instead and bypasses the auto-cache. `csp index -o <path>` is for explicit persistence only (`-o` is required) and is independent of the auto-cache.
 
+Files larger than 1 MB are skipped during indexing to keep index builds lean. Skipped files are reported as a warning on stderr at index time, naming the first few paths. If you work with large generated or ingested documents, you can raise (or lower) this limit with the `CSP_MAX_FILE_BYTES` environment variable (in bytes); a malformed or non-positive value warns and falls back to the 1 MB default.
+
 <details>
 <summary>Savings</summary>
 

crates/csp/src/indexing/cache_orchestrator.rs

@@ -9,11 +9,11 @@ use crate::chunking::source::DESIRED_CHUNK_LENGTH_CHARS;
 use crate::indexing::cache::{
     compute_content_hash_from_paths, ensure_cache_dir, resolve_cache_dir, CacheLocation,
 };
-use crate::indexing::create::MAX_FILE_BYTES;
 use crate::indexing::dense::SelectableBasicBackend;
 use crate::indexing::dense::DEFAULT_MODEL_NAME;
 use crate::indexing::file_walker::walk_files;
 use crate::indexing::files::get_extensions;
+use crate::indexing::files::get_max_file_bytes;
 use crate::indexing::index::{
     normalize_content, parse_manifest, read_chunks, CspIndex, IndexManifest, LoadOptions,
     INDEX_SCHEMA_VERSION,
@@ -42,11 +42,14 @@ fn collect_source_paths(root: &Path, content: &[ContentType]) -> Vec<(String, Pa
     let resolved = get_extensions(content, None);
     let ext_refs: Vec<&str> = resolved.iter().map(String::as_str).collect();
     let mut files = Vec::new();
+    // Same ceiling as `create_index_from_path`, so the fingerprint tracks
+    // exactly the files that get indexed.
+    let max_file_bytes = get_max_file_bytes();
     for file_path in walk_files(root, &ext_refs, &[]) {
         let Ok(meta) = std::fs::metadata(&file_path) else {
             continue;
         };
-        if meta.len() > MAX_FILE_BYTES {
+        if meta.len() > max_file_bytes {
             continue;
         }
         let rel = file_path.strip_prefix(root).unwrap_or(&file_path);

crates/csp/src/indexing/create.rs

@@ -15,14 +15,21 @@ use crate::chunking::source::chunk_source;
 use crate::indexing::cache::sha256_hex;
 use crate::indexing::dense::{embed_chunk_refs, Model, SelectableBasicBackend};
 use crate::indexing::file_walker::walk_files;
-use crate::indexing::files::{detect_language, get_extensions};
+use crate::indexing::files::{
+    detect_language, get_extensions, get_max_file_bytes, DEFAULT_MAX_FILE_BYTES, MAX_FILE_BYTES_ENV,
+};
 use crate::indexing::sparse::{enrich_for_bm25, Bm25Index};
 use crate::indexing::types::{make_chunk_id, FileManifest, FileManifestEntry, PreviousIndex};
 use crate::tokens::tokenize;
 use crate::types::{Chunk, ContentType};
 
 /// 1 MB max file size to read and index.
-pub const MAX_FILE_BYTES: u64 = 1_000_000;
+#[deprecated(
+    since = "0.1.10",
+    note = "use `indexing::files::DEFAULT_MAX_FILE_BYTES` or `get_max_file_bytes()` \
+            (the limit is now overridable via `CSP_MAX_FILE_BYTES`)"
+)]
+pub const MAX_FILE_BYTES: u64 = DEFAULT_MAX_FILE_BYTES;
 
 /// Options for [`create_index_from_path`].
 pub struct CreateIndexOptions<'a> {
@@ -33,6 +40,28 @@ pub struct CreateIndexOptions<'a> {
     pub content: Option<Vec<ContentType>>,
     /// When set, chunk file paths are stored relative to this root.
     pub display_root: Option<PathBuf>,
+    /// Max file size (bytes) to read and index; larger files are skipped with a
+    /// warning. `None` resolves `CSP_MAX_FILE_BYTES` (default 1 MB) via
+    /// [`get_max_file_bytes`].
+    pub max_file_bytes: Option<u64>,
+}
+
+impl<'a> CreateIndexOptions<'a> {
+    /// Options for `model` with every other field at its default: code-only
+    /// content, no extra extensions, chunk paths as walked, and the size limit
+    /// resolved from `CSP_MAX_FILE_BYTES`. Prefer
+    /// `CreateIndexOptions { extensions: .., ..CreateIndexOptions::new(model) }`
+    /// over a bare struct literal so a future option field does not break
+    /// callers.
+    pub fn new(model: &'a Model) -> Self {
+        Self {
+            model,
+            extensions: None,
+            content: None,
+            display_root: None,
+            max_file_bytes: None,
+        }
+    }
 }
 
 /// Result of [`create_index_from_path`].
@@ -45,6 +74,51 @@ pub struct CreateIndexResult {
     pub files: FileManifest,
 }
 
+/// A repository-controlled path rendered for a stderr diagnostic, with control
+/// characters (ANSI escapes, newlines) escaped so a hostile file name in a
+/// cloned repo cannot rewrite the terminal or forge diagnostic lines.
+fn escape_control(path: &str) -> String {
+    let mut out = String::with_capacity(path.len());
+    for c in path.chars() {
+        if c.is_control() {
+            out.extend(c.escape_default());
+        } else {
+            out.push(c);
+        }
+    }
+    out
+}
+
+/// The warning for files skipped for exceeding `max_file_bytes` — the count
+/// plus the first five paths — or `None` when nothing was skipped. Port of
+/// upstream `_warn_skipped_large` (semble #252): a silent skip left unexplained
+/// gaps in results. `from_env` selects the hint: the env var when the limit
+/// was resolved from it, else the `max_file_bytes` option the caller pinned.
+pub(crate) fn skipped_large_warning(
+    skipped: &[String],
+    max_file_bytes: u64,
+    from_env: bool,
+) -> Option<String> {
+    if skipped.is_empty() {
+        return None;
+    }
+    let shown: Vec<String> = skipped.iter().take(5).map(|p| escape_control(p)).collect();
+    let knob = if from_env {
+        MAX_FILE_BYTES_ENV
+    } else {
+        "max_file_bytes"
+    };
+    Some(format!(
+        "Skipped {} file(s) exceeding the maximum file size of {} bytes \
+         (raise {} to include them): {}{}",
+        skipped.len(),
+        max_file_bytes,
+        knob,
+        shown.join(", "),
+        if skipped.len() > 5 { " ..." } else { "" },
+    ))
+}
+
 /// Replace a file's BM25 postings: remove its old slots (if any), then add its
 /// new ones.
 fn reindex_file(
@@ -169,6 +243,7 @@ pub fn create_index_from_path(
         .unwrap_or_else(|| vec![ContentType::Code]);
     let resolved = get_extensions(&content, options.extensions.as_deref());
     let ext_refs: Vec<&str> = resolved.iter().map(String::as_str).collect();
+    let max_file_bytes = options.max_file_bytes.unwrap_or_else(get_max_file_bytes);
 
     let (mut bm25_index, previous_files, mut previous_chunks, mut previous_vectors) =
         open_previous(previous);
@@ -181,14 +256,16 @@ pub fn create_index_from_path(
     let mut vectors: Vec<Option<Vec<f32>>> = Vec::new();
     let mut fresh_rows: Vec<usize> = Vec::new();
     let mut files = FileManifest::new();
+    let mut skipped_large: Vec<String> = Vec::new();
 
     for file_path in walk_files(path, &ext_refs, &[]) {
         let language = detect_language(&file_path.to_string_lossy());
         let size = match std::fs::metadata(&file_path) {
             Ok(meta) => meta.len(),
             Err(_) => continue,
         };
-        if size > MAX_FILE_BYTES {
+        if size > max_file_bytes {
+            skipped_large.push(display_path(&file_path, options.display_root.as_deref()));
             continue;
         }
         let Ok(bytes) = std::fs::read(&file_path) else {
@@ -205,7 +282,7 @@ pub fn create_index_from_path(
             eprintln!(
                 "csp: skipping {}: its display path collides with an already indexed file \
                  (non-UTF-8 file name)",
-                file_path.display()
+                escape_control(&file_path.display().to_string())
             );
             continue;
         }
@@ -248,6 +325,16 @@ pub fn create_index_from_path(
         }
     }
 
+    // Warn before the empty check so a tree of only oversized files explains
+    // itself rather than just reporting "no supported files".
+    if let Some(warning) = skipped_large_warning(
+        &skipped_large,
+        max_file_bytes,
+        options.max_file_bytes.is_none(),
+    ) {
+        eprintln!("csp: {warning}");
+    }
+
     if chunks.is_empty() {
         return Err(format!(
             "No supported files found under {}.",

crates/csp/src/indexing/create/tests.rs

@@ -5,10 +5,8 @@ use tempfile::tempdir;
 
 fn opts(model: &Model, display_root: Option<PathBuf>) -> CreateIndexOptions<'_> {
     CreateIndexOptions {
-        model,
-        extensions: None,
-        content: None,
         display_root,
+        ..CreateIndexOptions::new(model)
     }
 }
 
@@ -53,6 +51,7 @@ fn respects_extensions_override() {
         extensions: Some(vec![".txt".to_string()]),
         content: Some(vec![ContentType::Docs]),
         display_root: Some(dir.path().to_path_buf()),
+        max_file_bytes: None,
     };
     let result = create_index_from_path(dir.path(), &options, None).unwrap();
     assert_eq!(result.chunks.len(), 1);
@@ -76,6 +75,67 @@ fn skips_files_over_max_bytes() {
     assert!(!paths.contains(&"big.ts"));
 }
 
+#[test]
+fn honors_configured_max_file_bytes() {
+    let dir = tempdir().unwrap();
+    std::fs::write(dir.path().join("mid.ts"), "a".repeat(200)).unwrap();
+    std::fs::write(dir.path().join("small.ts"), "export const x = 1\n").unwrap();
+    let model = make_stub_model(4);
+    let indexed = |limit: u64| -> Vec<String> {
+        let options = CreateIndexOptions {
+            max_file_bytes: Some(limit),
+            ..opts(&model, Some(dir.path().to_path_buf()))
+        };
+        let result = create_index_from_path(dir.path(), &options, None).unwrap();
+        result.chunks.iter().map(|c| c.file_path.clone()).collect()
+    };
+
+    // A limit under the default still gates: 200 bytes > 100.
+    let paths = indexed(100);
+    assert!(paths.iter().any(|p| p == "small.ts"));
+    assert!(!paths.iter().any(|p| p == "mid.ts"));
+    // Raising it lets the same file through.
+    assert!(indexed(1_000).iter().any(|p| p == "mid.ts"));
+}
+
+#[test]
+fn skipped_large_warning_names_first_five_paths_and_count() {
+    assert_eq!(skipped_large_warning(&[], 1_000_000, true), None);
+
+    let two = vec!["a.ts".to_string(), "b.ts".to_string()];
+    let msg = skipped_large_warning(&two, 1_000_000, true).unwrap();
+    assert_eq!(
+        msg,
+        "Skipped 2 file(s) exceeding the maximum file size of 1000000 bytes \
+         (raise CSP_MAX_FILE_BYTES to include them): a.ts, b.ts"
+    );
+
+    let seven: Vec<String> = (1..=7).map(|i| format!("f{i}.ts")).collect();
+    let msg = skipped_large_warning(&seven, 500, true).unwrap();
+    assert!(msg.starts_with("Skipped 7 file(s) exceeding the maximum file size of 500 bytes"));
+    assert!(
+        msg.ends_with("f1.ts, f2.ts, f3.ts, f4.ts, f5.ts ..."),
+        "{msg}"
+    );
+    assert!(!msg.contains("f6.ts"));
+
+    // A caller-pinned limit points at the option, not the env var.
+    let msg = skipped_large_warning(&two, 100, false).unwrap();
+    assert!(
+        msg.contains("(raise max_file_bytes to include them)"),
+        "{msg}"
+    );
+    assert!(!msg.contains("CSP_MAX_FILE_BYTES"));
+}
+
+#[test]
+fn skipped_large_warning_escapes_control_characters_in_paths() {
+    let hostile = vec!["evil\x1b[31m\nfake: ok.ts".to_string()];
+    let msg = skipped_large_warning(&hostile, 10, true).unwrap();
+    assert!(msg.ends_with("evil\\u{1b}[31m\\nfake: ok.ts"), "{msg}");
+    assert!(!msg.contains('\n'));
+}
+
 #[test]
 fn descends_into_subdirectories() {
     let dir = tempdir().unwrap();

crates/csp/src/indexing/file_sizes.rs

@@ -11,7 +11,7 @@ use std::io::Read as _;
 use std::path::{Path, PathBuf};
 use std::sync::Mutex;
 
-use crate::indexing::create::MAX_FILE_BYTES;
+use crate::indexing::files::get_max_file_bytes;
 
 /// UTF-16 character counts per repo-relative file path, resolved eagerly
 /// (captured) or lazily (read from a local root on demand).
@@ -97,7 +97,7 @@ impl FileSizes {
 /// the file walker never follows symlinks, and a path that has since become a
 /// symlink, FIFO, or device must not be able to redirect or stall the read.
 ///
-/// The read is bounded by [`MAX_FILE_BYTES`], the same ceiling
+/// The read is bounded by [`get_max_file_bytes`], the same ceiling
 /// `create_index_from_path` applies — lazily, this runs inside a live search,
 /// so a file that has grown past the indexing limit since it was chunked must
 /// not be slurped whole on the query path. Decoding is lossy, matching the
@@ -137,12 +137,15 @@ pub(crate) fn read_file_chars(root: &Path, file_path: &str) -> Option<u64> {
     // a local writer who already controls the indexed tree.
     let file = std::fs::File::open(&canonical).ok()?;
     let meta = file.metadata().ok()?;
-    if !meta.is_file() || meta.len() > MAX_FILE_BYTES {
+    let max_file_bytes = get_max_file_bytes();
+    if !meta.is_file() || meta.len() > max_file_bytes {
         return None;
     }
     let mut bytes = Vec::with_capacity(meta.len() as usize);
-    file.take(MAX_FILE_BYTES + 1).read_to_end(&mut bytes).ok()?;
-    if bytes.len() as u64 > MAX_FILE_BYTES {
+    file.take(max_file_bytes.saturating_add(1))
+        .read_to_end(&mut bytes)
+        .ok()?;
+    if bytes.len() as u64 > max_file_bytes {
         return None;
     }
     Some(String::from_utf8_lossy(&bytes).encode_utf16().count() as u64)
@@ -260,7 +263,7 @@ mod tests {
     #[test]
     fn lazy_skips_files_larger_than_the_indexing_ceiling() {
         let root = tempdir().unwrap();
-        let big = vec![b'a'; MAX_FILE_BYTES as usize + 1];
+        let big = vec![b'a'; get_max_file_bytes() as usize + 1];
         std::fs::write(root.path().join("grown.ts"), &big).unwrap();
         let sizes = FileSizes::lazy(root.path().to_path_buf());