feat: add --filter-file and --prefix-file for large filter sets

CHANGELOG.md

@@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file.
 
 ## Unreleased changes
 
+### New Features
+
+* Added `--filter-file` (JSON) and `--prefix-file` (newline text) flags to `monocle parse`
+  and `monocle search` for loading large filter sets from files. File filters merge with
+  CLI flags — union within each dimension (OR), AND across dimensions. Supports the
+  RIB-extract → filter-updates workflow at scale (#117).
+
 ### Bug Fixes
 
 * Fixed `--time-format rfc3339` being ignored for `json`, `json-line`, and `json-pretty`

README.md

@@ -20,6 +20,7 @@ See through all Border Gateway Protocol (BGP) data with a monocle.
 - [Usage](#usage)
   - [`monocle parse`](#monocle-parse)
     - [Output Format](#output-format)
+    - [Filter Files](#filter-files)
   - [`monocle search`](#monocle-search)
   - [`monocle rib`](#monocle-rib)
   - [`monocle time`](#monocle-time)
@@ -348,6 +349,12 @@ Options:
 
           [default: unix]
 
+      --filter-file <PATH>
+          Load filters from a JSON file (prefixes, origin_asns, peer_asns, etc.) Merged with CLI filter flags (AND across dimensions, union within each)
+
+      --prefix-file <PATH>
+          Load a newline-delimited list of prefixes from a file Lines starting with # and blank lines are ignored
+
   -o, --origin-asn <ORIGIN_ASN>
           Filter by origin AS Number(s), comma-separated. Prefix with ! to exclude
 
@@ -426,6 +433,76 @@ monocle parse file.mrt -o '!13335,!15169'
 
 Note: Cannot mix positive and negative values in the same filter.
 
+#### Filter Files
+
+For large filter sets that exceed command-line argument limits, or for reusable
+filter definitions, use `--filter-file` (JSON) or `--prefix-file` (newline text).
+
+**`--prefix-file`** — a plain text file with one prefix per line. Blank lines and
+`#` comments are ignored. This is the most ergonomic option for the common
+RIB-extract → filter-updates workflow:
+
+```bash
+# Step 1: Extract prefix list from a RIB dump
+monocle parse -o 64496 rib.gz | cut -d'|' -f5 | sort -u > prefixes.txt
+
+# Step 2: Filter subsequent updates using the prefix file
+monocle parse --prefix-file prefixes.txt updates.gz
+```
+
+Prefix file format:
+
+```text
+# Prefixes originated by AS64496 (extracted from RIB)
+192.0.2.0/24
+198.51.100.0/24
+
+# IPv6 prefixes
+2001:db8::/32
+```
+
+**`--filter-file`** — a structured JSON file supporting all filter dimensions:
+
+```json
+{
+  "prefixes": ["192.0.2.0/24", "2001:db8::/32"],
+  "origin_asns": ["64496"],
+  "peer_asns": ["174", "6939"],
+  "communities": ["64496:100", "*:200"],
+  "as_path_regex": "174 64496$",
+  "elem_type": "w",
+  "include_sub": true,
+  "start_ts": "2025-01-01T00:00:00Z",
+  "end_ts": "2025-01-01T01:00:00Z"
+}
+```
+
+All fields are optional — include only the dimensions you need. String-typed
+fields use the same syntax as CLI flags, including `!` prefix for negation.
+
+**Combining with CLI flags** — file-based filters merge with CLI filter flags.
+Within each dimension, values are unioned (OR logic). For example:
+
+```bash
+# CLI prefix + file prefixes are unioned
+monocle parse -p 10.0.0.0/8 --filter-file filters.json updates.gz
+```
+
+For scalar fields (`as_path_regex`, `elem_type`, time fields), CLI flags take
+precedence over file values. Boolean flags (`include_super`, `include_sub`)
+are OR-ed.
+
+**Using both file types together**:
+
+```bash
+monocle parse \
+  --filter-file as_filters.json \
+  --prefix-file extra_prefixes.txt \
+  updates.gz
+```
+
+Prefixes from both sources are unioned into a single prefix filter.
+
 #### Field Selection
 
 Use `-f` or `--fields` to select which columns to display:
@@ -597,6 +674,12 @@ Options:
 
           [default: unix]
 
+      --filter-file <PATH>
+          Load filters from a JSON file (prefixes, origin_asns, peer_asns, etc.) Merged with CLI filter flags (AND across dimensions, union within each)
+
+      --prefix-file <PATH>
+          Load a newline-delimited list of prefixes from a file Lines starting with # and blank lines are ignored
+
       --use-cache
           Use the default XDG cache directory ($XDG_CACHE_HOME/monocle) for MRT files. Overridden by --cache-dir if both are specified
 

src/bin/commands/parse.rs

@@ -5,6 +5,7 @@ use bgpkit_parser::encoder::MrtUpdatesEncoder;
 use bgpkit_parser::BgpElem;
 use clap::Args;
 
+use monocle::lens::parse::filter_file::{load_prefix_file, merge_prefix_file, FilterFile};
 use monocle::lens::parse::{ParseFilters, ParseLens};
 use monocle::utils::{OrderByField, OrderDirection, OutputFormat, TimestampFormat};
 
@@ -43,6 +44,16 @@ pub(crate) struct ParseArgs {
     #[clap(long, value_enum, default_value = "unix")]
     pub time_format: TimestampFormat,
 
+    /// Load filters from a JSON file (prefixes, origin_asns, peer_asns, etc.)
+    /// Merged with CLI filter flags (AND across dimensions, union within each)
+    #[clap(long, value_name = "PATH")]
+    pub filter_file: Option<PathBuf>,
+
+    /// Load a newline-delimited list of prefixes from a file
+    /// Lines starting with # and blank lines are ignored
+    #[clap(long, value_name = "PATH")]
+    pub prefix_file: Option<PathBuf>,
+
     /// Filter by AS path regex string
     #[clap(flatten)]
     pub filters: ParseFilters,
@@ -57,9 +68,28 @@ pub fn run(args: ParseArgs, output_format: OutputFormat) {
         order_by,
         order,
         time_format,
-        filters,
+        filter_file,
+        prefix_file,
+        mut filters,
     } = args;
 
+    // Load and merge file-based filters into CLI filters
+    if let Some(ref pf) = filter_file {
+        if let Err(e) = FilterFile::load(pf).and_then(|ff| ff.merge_into(&mut filters)) {
+            eprintln!("ERROR: {}", e);
+            std::process::exit(1);
+        }
+    }
+    if let Some(ref pf) = prefix_file {
+        match load_prefix_file(pf) {
+            Ok(prefixes) => merge_prefix_file(prefixes, &mut filters),
+            Err(e) => {
+                eprintln!("ERROR: {}", e);
+                std::process::exit(1);
+            }
+        }
+    }
+
     // Parse and validate fields (false = parse command, no collector in defaults)
     let fields = match parse_fields(&fields_arg, false) {
         Ok(f) => f,

src/bin/commands/search.rs

@@ -10,6 +10,7 @@ use bgpkit_parser::encoder::MrtUpdatesEncoder;
 use bgpkit_parser::BgpElem;
 use clap::Args;
 use monocle::database::MsgStore;
+use monocle::lens::parse::filter_file::{load_prefix_file, merge_prefix_file, FilterFile};
 use monocle::lens::search::SearchFilters;
 use monocle::utils::{OrderByField, OrderDirection, OutputFormat, TimestampFormat};
 use monocle::MonocleConfig;
@@ -59,6 +60,16 @@ pub struct SearchArgs {
     #[clap(long, value_enum, default_value = "unix")]
     pub time_format: TimestampFormat,
 
+    /// Load filters from a JSON file (prefixes, origin_asns, peer_asns, etc.)
+    /// Merged with CLI filter flags (AND across dimensions, union within each)
+    #[clap(long, value_name = "PATH")]
+    pub filter_file: Option<PathBuf>,
+
+    /// Load a newline-delimited list of prefixes from a file
+    /// Lines starting with # and blank lines are ignored
+    #[clap(long, value_name = "PATH")]
+    pub prefix_file: Option<PathBuf>,
+
     /// Use the default XDG cache directory ($XDG_CACHE_HOME/monocle) for MRT files.
     /// Overridden by --cache-dir if both are specified.
     #[clap(long)]
@@ -541,11 +552,39 @@ pub fn run(config: &MonocleConfig, args: SearchArgs, output_format: OutputFormat
         order_by,
         order,
         time_format,
+        filter_file,
+        prefix_file,
         use_cache,
         cache_dir,
-        filters,
+        mut filters,
     } = args;
 
+    // Load and merge file-based filters into CLI filters
+    if let Some(ref pf) = filter_file {
+        if let Err(e) =
+            FilterFile::load(pf).and_then(|ff| ff.merge_into(&mut filters.parse_filters))
+        {
+            eprintln!("ERROR: {}", e);
+            std::process::exit(1);
+        }
+    }
+    if let Some(ref pf) = prefix_file {
+        match load_prefix_file(pf) {
+            Ok(prefixes) => merge_prefix_file(prefixes, &mut filters.parse_filters),
+            Err(e) => {
+                eprintln!("ERROR: {}", e);
+                std::process::exit(1);
+            }
+        }
+    }
+
+    // Validate merged parse filters (prefixes, ASNs, communities) early so file-based
+    // mistakes fail fast with actionable errors instead of failing per-file later.
+    if let Err(e) = filters.parse_filters.validate() {
+        eprintln!("ERROR: {}", e);
+        std::process::exit(1);
+    }
+
     let cache_dir = match cache_dir {
         Some(cache_dir) => Some(cache_dir),
         None => use_cache.then(|| PathBuf::from(config.cache_dir())),