diff --git a/.gitignore b/.gitignore index 3f0f372..2e2b782 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,11 @@ dist /metrics /reports +# AI Landscape tool for interaction +/ai-landscape-tools + + +# WSL / windows file, not needed *Zone.Identifier* server.log diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cb1e68..0fd7125 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,93 @@ All notable changes to NuAnalytics are recorded here. Format loosely follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); the project uses semantic versioning. +## [0.4.1] — 2026-06-04 + +This release is additive — no breaking changes from 0.4.0. It introduces a +unified JSON degree format and the tooling around it, parallel/process-isolated +batch analysis, and two new MCP tools, plus two engine fixes that make +machine-converted catalogs analyzable. + +### Added + +- **Unified JSON degree format.** Degree programs can now be authored and + consumed as JSON (the `DegreeProgram` model serialized directly) alongside + YAML. Every `degree` subcommand auto-detects the format on load (content + starting with `{`/`[` → JSON, otherwise YAML), and raw ai-landscape JSON + shapes are converted on the fly. Prerequisites serialize as a symmetric + tagged structure (`{"and"|"or": [...]}`, with a bare string as a leaf), and + `tags` are available on degrees, requirements, and courses. + +- **`degree convert`** — convert ai-landscape program JSON into the unified + format. Category lists and picklists map to requirements, AND-of-OR + prerequisites flip into the internal `PrereqExpr` tree, and missing credits + default to 3 (with warnings). ai-landscape *cluster* pipeline files + (`course_verifier`/`course_scraper..results`) expand into one unified + file per program with collision-safe `__.unified.json` names. + `-o ` accepts a file (single input) or directory; `--pretty` pretty-prints. + +- **`degree schema`** — emit the unified-degree JSON Schema + (`src/assets/degree.schema.json`), the same schema the MCP server serves, to + stdout or `-o `. The schema now documents the `from` clause + (`fromClause`: courses / pattern / include / exclude / groups), confirming the + unified format supports the same wildcard gen-ed/elective pools as YAML + (e.g. `"CS:2500+"`, `"*:*"`). + +- **Parallel `degree analyze` (`-j`/`--jobs`, default 8).** A multi-file analyze + now runs as a rolling pool of worker processes, one file per OS process. A + pathological degree (e.g. a full-catalog scrape) is contained to its own + process: if it OOMs or crashes, the kernel kills only that child, the parent + records it in `/failures.log` with its exit status (so a + `SIGKILL` is distinguishable from a non-zero exit), and the rest continue. + Single-file, `--school`, and `-j 1` runs stay in-process with full per-degree + output. + +- **`--school ` on `degree analyze`** — treat all inputs as programs of + one school and emit a combined `_school_report.json` rolling up + degree-level metrics across the programs. + +- **`scripts/analyze-batch.sh`** — process-isolated batch analyze with a + per-process virtual-memory cap (`ulimit -v`) and a timeout, for running large + directories of degrees without the OS OOM-killer taking down the whole batch. + +- **JSON input for `degree trim`.** Trim now accepts unified (and raw + ai-landscape) JSON and writes the trimmed program back in the input's format — + a `.json` input yields a trimmed `.json`; YAML stays YAML. + +- **Metrics-rich report JSON.** Output JSON now opens with the degree block and + is laid out degree → analysis → requirements → selected plans → courses. Each + selected plan carries its courses, credits, course count, critical path, and a + term-by-term schedule (mirroring the MCP `analyze_degree` shape). `total_credits` + surfaces at the top. + +- **New MCP tools.** `convert_degree` (ai-landscape JSON → unified JSON + + warnings, caching the result for chaining by `degree_id`; a cluster file + returns a bounded program inventory) and `get_degree_json_schema` (returns the + machine JSON Schema). The existing degree tools + (`validate_degree` / `analyze_degree` / `audit_degree` / `trim_degree` / + `get_course_detail`) now accept unified and ai-landscape JSON content — and the + `cache:` handle from `convert_degree` — in addition to YAML, via a + content-level format sniff; `validate_degree` surfaces any + `conversion_warnings`. + +### Fixed + +- **Out-of-memory on large select pools.** `RequirementResolver` materialized + every `C(n, k)` combination of a select pool — a "choose 15 of 42" pool + (~10¹¹) could allocate tens of GB and get OOM-killed even for an otherwise + tiny program. Combination generation is now bounded: when `C(n, k)` exceeds + 2000, it deterministically down-samples to that cap. Peak memory on the worst + catalog programs drops from >6 GB to ~25–120 MB. + +- **Converted programs collapsed to a single plan.** Elective-category selects + were excluded from the plan space (`ENUMERABLE_CATEGORIES` was `["major"]` + only), so converted programs produced one plan with `std_dev = 0`. Electives + are now enumerated (plan-count estimation uses saturating multiplication so a + capped pool can't overflow), restoring real metric spread. + +- **JSON parse errors** now report as a distinct `JsonError` rather than + "YAML Parse Error". + ## [0.4.0] — 2026-05-20 ### Breaking changes diff --git a/Cargo.lock b/Cargo.lock index 7738c6d..7f131ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1369,7 +1369,7 @@ dependencies = [ [[package]] name = "nu-analytics" -version = "0.4.0" +version = "0.4.1" dependencies = [ "chrono", "clap", diff --git a/Cargo.toml b/Cargo.toml index 5ca9aba..3dab1c4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "nu-analytics" -version = "0.4.0" +version = "0.4.1" edition = "2021" authors = ["Albert Lionelle "] description = "A CLI tool for computing Curricular Analytics metrics. Based off metrics from CurricularAnalytics.org. Currently, only basic metrics and additional reporting features." diff --git a/Readme.md b/Readme.md index 4d3e3b8..052cdbb 100644 --- a/Readme.md +++ b/Readme.md @@ -1,6 +1,6 @@ # NuAnalytics -**Version 0.4.0** +**Version 0.4.1** NuAnalytics is a Rust-based tool for analyzing computer science curricula. It computes detailed metrics about curriculum structure including complexity, blocking relationships, delay paths, and centrality measures to help understand how courses are organized and their impact on students. @@ -16,7 +16,9 @@ It is based off the work of Greg Heileman, and CurricularAnalytics.org. Current - **Curriculum Analysis**: Parse CSV-formatted curriculum files and analyze course dependencies - **Degree Program Validation**: Validate YAML-based degree programs with comprehensive checks - **Degree Plan Generation**: Generate all possible degree plans and compute aggregate statistics -- **Degree Trim** *(new in 0.4.0)*: Collapse a degree YAML to one walkable shortest-path-per-course variant — useful for visualisation and downstream tools that don't reason about alternatives. Protected major subjects keep all options; pattern pools survive orphan pruning. +- **Unified JSON Degree Format** *(new in 0.4.1)*: Author and analyze degrees as JSON as well as YAML — every `degree` subcommand auto-detects the format on load. `degree convert` turns ai-landscape program JSON into the unified format, and `degree schema` emits its JSON Schema for downstream validation. +- **Parallel Batch Analysis** *(new in 0.4.1)*: `degree analyze -j N` runs a process-isolated worker pool (default 8) so a pathological degree (e.g. a full-catalog scrape) can't take down the whole batch; failures are logged and the run continues. +- **Degree Trim** *(new in 0.4.0)*: Collapse a degree to one walkable shortest-path-per-course variant — useful for visualisation and downstream tools that don't reason about alternatives. Protected major subjects keep all options; pattern pools survive orphan pruning. Accepts YAML or JSON and round-trips the input format. - **Gen-Ed Tracking**: Track how major courses satisfy general education requirements - **Graph Analysis**: Build and analyze course prerequisite graphs - **Curriculum Auditing**: Identify issues like missing prerequisites and complex prerequisite chains @@ -31,7 +33,7 @@ It is based off the work of Greg Heileman, and CurricularAnalytics.org. Current - **Markdown**: Text-based reports for documentation - **Term Scheduling**: Automatic course scheduling respecting prerequisites and credit limits - **Configuration Management**: Flexible configuration system with CLI overrides -- **MCP Server** (optional): AI model integration for interactive degree building via Model Context Protocol, including a `trim_degree` tool that pipes a fresh `cache:` handle back for chained `validate_degree` / `audit_degree` calls. +- **MCP Server** (optional): AI model integration for interactive degree building via Model Context Protocol. Degree tools accept YAML, unified JSON, or raw ai-landscape JSON; `convert_degree` *(new in 0.4.1)* converts ai-landscape JSON to the unified format and `get_degree_json_schema` *(new in 0.4.1)* returns the machine schema. The `trim_degree` tool pipes a fresh `cache:` handle back for chained `validate_degree` / `audit_degree` calls. - **Authenticated Database Access** *(behavior change in 0.4.0)*: Supabase reads and writes both require a logged-in user (`nuanalytics db login`); session tokens auto-refresh. ## Quick Start diff --git a/docs/degree.md b/docs/degree.md index 2977211..060d84e 100644 --- a/docs/degree.md +++ b/docs/degree.md @@ -31,8 +31,20 @@ nuanalytics degree print-graph samples/degrees/csu-cs-bscs-general.yaml # Trim alternatives down to a single shared shortest path nuanalytics degree trim samples/degrees/csu-cs-bscs-general.yaml + +# Convert ai-landscape program JSON to the unified degree JSON +nuanalytics degree convert program.json -o converted/ + +# Emit the unified-degree JSON Schema +nuanalytics degree schema -o degree.schema.json ``` +> **Input formats:** every subcommand accepts both YAML and the unified +> JSON degree format, and auto-detects which on load (content starting +> with `{` or `[` is parsed as JSON, otherwise YAML). Raw ai-landscape +> program JSON is detected and converted automatically. See +> [Input File Format](#input-file-format). + > **Migration note:** prior versions exposed these as flags > (`degree --validate`, `degree --analyze`, etc.). Every flag has moved > to a subcommand of the same name. There is no implicit default action; @@ -66,6 +78,8 @@ This generates: | `--report-dir ` | Override reports output directory | from config | | `--metrics-dir ` | Override metrics output directory | from config | | `--include ` | Courses to always include in all plans (comma-separated) | none | +| `-j, --jobs ` | Files to analyze concurrently when multiple are given, each in its own process (see [Parallel analysis](#parallel-analysis)) | 8 | +| `--school ` | Treat all inputs as programs of one school and also emit a combined `_school_report.json` rollup | none | **Examples:** @@ -112,6 +126,32 @@ nuanalytics degree analyze --include "CS3500,STAT301" samples/degrees/csu-cs-bsc `degree trim` reuses the same flag — see the **Trim** section below for trim-specific semantics. +### Parallel Analysis + +When you pass **multiple** files to `analyze`, the command runs a rolling +pool of worker processes — one file per OS process, `-j/--jobs` at a time +(default 8). Process isolation matters: a pathological degree (for example a +full-catalog scrape with thousands of courses) is contained to its own +process, so if it exhausts memory or crashes, the kernel kills only that +child. The parent records the failure in `/failures.log` — one +`pathstatus` line, so an OOM kill shows as `signal: 9 (SIGKILL)` and is +distinguishable from a non-zero exit — and the remaining files carry on. The +parent prints a progress line and a summary; per-worker stdout/stderr is +suppressed. + +```bash +# Analyze a directory of degrees, 12 at a time +nuanalytics degree analyze samples/degrees/*.yaml -j 12 --metrics-dir out/ + +# Roll the per-degree metrics up into one school report +nuanalytics degree analyze cs-programs/*.json --school "Example University" +``` + +Single-file runs, `--school`, and `-j 1` run in-process with full per-degree +console output. For an externally throttled variant (a per-process +`ulimit -v` memory cap and a timeout), see `scripts/analyze-batch.sh`; the +in-process pool deliberately imposes no ulimit or timeout. + ### Validation (`validate`) ```bash @@ -231,6 +271,60 @@ prints the protected-subject set and the list of removed orphans. - `type: one_of` concentrations are preserved as a whole; trim recurses into each concentration's nested requirements. +### Convert (`convert`) + +Convert program file(s) into the unified degree JSON format. + +```bash +nuanalytics degree convert path/to/program.json +``` + +The input may be raw ai-landscape program JSON (auto-detected and +converted), an existing unified JSON file, or YAML; the output is always +unified JSON with structured prerequisites. The converter maps ai-landscape +category lists and picklists to requirements, flips their AND-of-OR +prerequisites into the internal expression tree, and defaults missing course +credits to 3. Data-quality issues (such as assumed credits) are reported and +embedded as a `conversion_warnings` array in the output. + +ai-landscape *cluster* pipeline files (`course_verifier` / +`course_scraper..results`) are expanded into **one unified file per +program**, using collision-safe `__.unified.json` names. + +| Option | Description | Default | +|--------|-------------|---------| +| `-o, --out ` | Output file (single input) or directory (one `.unified.json` per input) | next to each input as `.unified.json` | +| `--pretty` | Pretty-print the JSON (default is compact, one line) | false | + +```bash +# Convert a directory of ai-landscape programs into ./converted/ +nuanalytics degree convert ai-landscape-tools/validation_jsons/*.json -o converted/ + +# Convert one program, pretty-printed +nuanalytics degree convert program.json --pretty -o program.unified.json +``` + +> This converter is transitional — once upstream emits unified JSON +> directly it is no longer needed. + +### Schema (`schema`) + +Print the JSON Schema for the unified degree format — useful for validating +unified JSON files in other tools or pipelines. This is the same +`degree.schema.json` the MCP server serves via `get_degree_json_schema`. + +```bash +# Print to stdout +nuanalytics degree schema + +# Write to a file +nuanalytics degree schema -o degree.schema.json +``` + +| Option | Description | Default | +|--------|-------------|---------| +| `-o, --out ` | Write the schema to this file instead of stdout | stdout | + ## Analysis Output ### Console Output @@ -332,6 +426,15 @@ nuanalytics config set metrics_dir "./metrics" Degree programs are defined in YAML files with three main sections: +> **Unified JSON.** The same three-section model can also be expressed as +> *unified JSON* — the degree model serialized directly to JSON, with +> prerequisites written as a symmetric tagged structure +> (`{"and"|"or": [...]}`, a bare string being a single prerequisite) and an +> optional `tags` array on degrees, requirements, and courses. Every +> subcommand auto-detects YAML vs. JSON on load, and `degree schema` emits +> the JSON Schema for the format. The sections below describe the YAML form; +> the JSON form mirrors it field-for-field. + ### Degree Metadata ```yaml diff --git a/docs/json-dump-feedback.md b/docs/json-dump-feedback.md new file mode 100644 index 0000000..d047c77 --- /dev/null +++ b/docs/json-dump-feedback.md @@ -0,0 +1,209 @@ +# AI-Landscape JSON Dump — Review & Feedback + +_Review of the `degree analyze` JSON output (`.debug/metrics/json-dump/`) produced +from the ai-landscape `cluster_outputs/` scrape, cross-checked against the +human-validated set (`validation_jsons_stripped/`). Date: 2026-06-03._ + +## What was reviewed + +| | count | +|---|---| +| Converted unified programs (from `cluster_outputs/`, 643 pipeline files) | 1,033 | +| Reports successfully produced (scraped) | 930 | +| Scraped programs that **failed** (OOM / SIGKILL) | 35 | +| Validated programs converted + analyzed (from `validation_jsons_stripped/`) | 42 | +| **Reports now in `json-dump/` (scraped + validated merged)** | **972** | + +The 42 validated reports were generated and merged into the dump (see +[§5](#5-validated-set-merged-into-the-dump)). They are **additions**, not +overwrites — see [§2.5](#25-scraped-vs-validated-naming-divergence). + +--- + +## 1. NuAnalytics engine issues (these block the dump's usefulness) + +These two are **our** bugs, not ai-landscape's. They are the reason the dump is +currently low-value, and they should be fixed before another full run. + +### 1.1 OOM on large elective pools — *eager combination materialization* + +**All 35 failures are `signal: 9 (SIGKILL)` = out-of-memory.** The cause is **not** +catalog size and **not** plan enumeration — it is that +`RequirementResolver::resolve_select_requirement` calls `generate_combinations(pool, count)`, +which **eagerly materializes every C(N, k) combination into a `Vec>`**, +for *every* `select` requirement — including `category: "elective"` ones that are +later discarded from the plan space (see §1.2). + +A program with one large "choose k of N" elective explodes regardless of how few +courses it has: + +| Program | Courses | Largest `choose k of N` | C(N,k) | Result | +|---|---|---|---|---| +| Iowa State CS BS | **78** | choose 15 of 42 | 9.9×10¹⁰ | **OOM (>6 GB)** | +| Missouri-Columbia | 64 | choose 24 of 41 | 1.5×10¹¹ | OOM | +| UW-La Crosse | 64 | choose 12 of 51 | 1.6×10¹¹ | OOM | +| USF AI | 16,012 | choose 6 of 16,000 | 2.3×10²² | OOM | + +**Proof it's the combination Vec, not the course graph:** Iowa State (78 courses) +peaks at **>6 GB and aborts**. Shrinking just that one requirement to "choose 2 of 42" +(C=861) drops peak memory to **22 MB** and it completes. The 78-course graph is +trivial; the 9.9×10¹⁰-element `Vec` is the hog. + +`--jobs 8` (the default pool) then stacks ~8 of these allocations on a 15 GiB box, +so the OS OOM-killer fires and takes out multiple concurrent workers — including +small innocent ones running at the same time. + +**Recommended fix (NuAnalytics):** never materialize the full combination set. +Either (a) sample `min(C(N,k), max_plans)` combinations lazily, or (b) cap the +materialized set and warn, or (c) skip materialization entirely for +non-enumerable categories (since electives are discarded anyway — see §1.2). Option +(c) alone would have prevented all 35 failures. + +### 1.2 Every report has `variations_run: 1` — *degenerate, single-plan metrics* + +**All 971 reports** (min = median = max = 1) analyze exactly **one** plan, so every +degree-level distribution is collapsed: `std_dev = 0`, and +`median = mean = min = max = q1 = q3`. The whole point of the metrics output +(variations, spread) is lost. + +**Root cause:** `plan_generator.rs` enumerates only requirements whose category is in +`ENUMERABLE_CATEGORIES = ["major"]`. The converter labels **every** picklist/elective +`select` requirement `category: "elective"`, so none of a converted program's choice +points ever enter the plan space → exactly 1 plan. + +**Proof:** flipping American University's picklist categories `elective → major` takes +it from `Estimated total plans: 1` to **74,256 estimated / 986 analyzed, complexity +std_dev 7.25** — a real distribution. + +> Note the tension with §1.1: simply enabling enumeration of electives would +> immediately trigger the OOM in §1.1 for big pools. **Both must be fixed together** — +> enumerate elective choice points *and* sample combinations instead of materializing +> them. + +**What is still meaningful today:** the **per-course** metrics (complexity, centrality, +blocking, delay) are derived from the prerequisite graph, not from plan sampling, so +they are valid even at `variations_run = 1`. Only the degree-level distributions are +degenerate. (E.g. CSU Fullerton: 26/31 courses carry non-zero graph metrics.) + +--- + +## 2. ai-landscape data-quality issues + +### 2.1 Picklist `[N]` is ambiguous: course-count vs. credits → 316 impossible requirements + +`[N]` in a picklist tag is interpreted inconsistently in the source data: + +- `"Calculus [1]"` over 2 courses → clearly **choose 1 course**. +- `"Artificial Intelligence Concentration [12]"` over **5** courses → **impossible** as + a course count (can't choose 12 of 5); only makes sense as **12 credits** (≈ 4 courses). + +Because the converter treats `[N]` as a course count, **316 `select` requirements across +225 of 1,033 files (20%)** are emitted with `count > pool_size`, which are unsatisfiable. + +**Recommendation (ai-landscape):** make `[N]` unambiguous — either always "number of +courses" or always "number of credits", or tag it explicitly +(e.g. `[3 courses]` vs `[12 credits]`). This is the single highest-value data fix. +(NuAnalytics should also defensively clamp `count` to the pool size.) + +### 2.2 Unparseable picklist tags (missing `[N]`) — 481+ occurrences + +The most common conversion warning is **`Unparseable picklist tag`**: a picklist label +with no `[N]` annotation at all, so no choice count can be derived. Top offenders: + +``` +481× "Computer Science Electives …" +136× "Technical Electives …" +101× "CSC 300-400 level …" + 85× "Artificial Intelligence …" + 77× "Upper Division …" +``` + +**Recommendation (ai-landscape):** every picklist tag should carry the `[N]` count, or +be omitted if it isn't actually a choice group. + +### 2.3 Catalog-scrape outliers (whole-catalog ingestion) + +A handful of programs captured far more courses than a degree could contain — the +scraper appears to have swallowed an entire course catalog: + +``` +16,012 University of South Florida — Artificial Intelligence (B.S.A.I.) + 2,046 Kean University — Artificial Intelligence + 1,991 University of Vermont — B.A. in Computer Science + 1,926 CUNY Brooklyn College — Computer Science, B.S. + 1,031 University of North Florida — Computer Science (BS) +``` +(Median program course count is **29**.) These are almost certainly scraper errors and +should be flagged/re-scraped. + +### 2.4 Missing `course_hours` → defaulted to 3 credits + +Of 1,033 converted files, **452 carry conversion warnings (11,162 total)**; a large +share are `missing course_hours; assuming 3 credits`. The 3-credit assumption is +reasonable but silently skews credit totals. **Recommendation:** populate +`course_hours` (we can also backfill from the IPEDS/Supabase catalog before defaulting). + +### 2.5 Scraped vs. validated naming divergence + +The scraped and validated records for the *same* program use different degree-name +strings (scraped: `"Computer Science, B.S."`; validated: +`"Bachelor's of Science Computer Science"`), so their report filenames never collide. +23 of the 41 validated universities also appear in the scraped set, but as +differently-named files. **Recommendation:** converge on a canonical +`university` + `degree` + `degree_type` identity so records can be matched and de-duplicated. + +--- + +## 3. What's working well + +- **Conversion is robust:** all 1,033 programs converted without crashing; the cluster + pipeline format (`course_verifier..results`) is correctly unwrapped. +- **Prerequisites** import losslessly into the tagged `{and|or}` AST. +- **Tags** (`ai`, `ai-major`/`ai-concentration`/`ai-minor`) are carried onto the degree + and surface correctly in reports. +- **Per-course graph metrics** are sound and useful today (see §1.2). +- The validated set is clean and bounded (max branching 4.1×10⁴ vs the failed-set + median 3.4×10⁸) — **human validation already fixes the elective explosion.** + +--- + +## 4. Recommendations, prioritized + +**NuAnalytics (we own these):** +1. **Stop materializing combinations** in `generate_combinations` (fixes all 35 OOMs). — §1.1 +2. **Enumerate elective choice points**, not just `category == "major"` (fixes + `variations_run = 1`). Must land with #1. — §1.2 +3. Defensively **clamp `select.count` to the pool size** + warn. — §2.1 + +**ai-landscape (data):** +1. Disambiguate picklist **`[N]` (courses vs credits)**. — §2.1 +2. Ensure every **picklist tag carries `[N]`**. — §2.2 +3. Fix **whole-catalog scrapes** (USF, Kean, UVM, CUNY Brooklyn, UNF). — §2.3 +4. Populate **`course_hours`**. — §2.4 +5. Emit a **canonical program identity** for matching/dedup. — §2.5 + +--- + +## 5. Validated set merged into the dump + +The 42 `validation_jsons_stripped/` programs were converted +(`degree convert … -o .debug/converted_validated/`) and analyzed +(`degree analyze … -j 4`, 0 failures) into the dump. Notably, the **scraped** +CSU-Fullerton CS BS was one of the 35 OOM failures, while its **validated** version +analyzes cleanly — a concrete win for using validated data. + +The ai-landscape directories under `ai-landscape-tools/` were **not modified**. + +## 6. Sample reports + +Five reports were generated with the tool into `.debug/metrics/json-reports/` +(report JSON + HTML + plan CSVs), anchored on the requested CSU CS BS: + +1. California State University, Fullerton — CS, B.S. _(the CSU csbs)_ +2. Northeastern University — CS concentration +3. Florida International University — CS, B.S. +4. Drexel University — CS (AI concentration) +5. Georgia Institute of Technology — CS minor + +All five are valid JSON with populated per-course metrics; all show +`variations_run: 1` per §1.2. diff --git a/docs/mcp.md b/docs/mcp.md index f9df0ed..4599964 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -136,6 +136,14 @@ The server uses stdio transport, compatible with any MCP client. Configure your ## Available Tools +> **Input formats.** The degree tools (`validate_degree`, `audit_degree`, +> `trim_degree`, `analyze_degree`, `get_course_detail`) accept YAML, unified +> JSON, or raw ai-landscape JSON in their `yaml_content` parameter — the +> format is auto-detected, and ai-landscape shapes are converted on the fly. +> They also accept a `cache:` handle (from `cache_yaml` or +> `convert_degree`) as `degree_id`. When the input is ai-landscape JSON, any +> assumptions made during conversion are reported as `conversion_warnings`. + ### `get_degree_schema` Returns documentation about the degree YAML format. @@ -150,6 +158,24 @@ Returns documentation about the degree YAML format. - "What fields go in the degree section?" → calls with `section: "degree"` - "Give me an example degree YAML" → calls with `section: "examples"` +### `get_degree_json_schema` + +Returns the machine-validatable JSON Schema (draft 2020-12) for the unified +degree format — the structure `convert_degree` produces and that +`validate_degree` / `analyze_degree` / `trim_degree` accept. Use it to +validate a unified degree document or to understand the format +programmatically, including wildcard `from` pools (e.g. pattern `"CS:2500+"` +or `"*:*"`). It is the same schema the CLI emits via `degree schema`. + +This is distinct from `get_degree_schema`, which returns the *human-readable +YAML* reference. + +**Parameters:** none. + +**Example Prompts:** +- "Give me the JSON Schema for a degree" → calls `get_degree_json_schema` +- "What does a valid unified degree document look like?" → calls `get_degree_json_schema` + ### `validate_degree` Validates a degree program YAML string and returns detailed feedback. @@ -279,6 +305,46 @@ prune even when no requirement names them explicitly. - "Also keep all MATH alternatives" → calls with `keep_all: ["MATH"]` - "Force MATH241 wherever possible" → calls with `include: ["MATH241"]` +### `convert_degree` + +Converts an ai-landscape program JSON into the unified NuAnalytics degree +JSON (the same format `analyze_degree` / `validate_degree` accept). Maps +category lists and picklists to requirements, flips AND-of-OR prerequisites +into the internal expression tree, and defaults missing credits (reported as +`conversion_warnings`). + +**Parameters:** +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `json_content` | string | One of `json_content` / `json_path` | Inline ai-landscape (or already-unified) program JSON | +| `json_path` | string | One of `json_content` / `json_path` | Path to a JSON file on the MCP server's filesystem | +| `program` | string | No | For a multi-program *cluster* pipeline file: the program to convert. Omit to get the cluster's program inventory instead. | +| `pretty` | boolean | No | Pretty-print the `unified_json` body (default compact) | + +**Response Format:** +```json +{ + "success": true, + "kind": "single", + "program_count": 1, + "unified_json": "{ ... unified degree JSON ... }", + "conversion_warnings": ["Course 'CS101' missing credits; assumed 3"], + "cache_id": "cache:9f3a…", + "note": "Pass cache_id as degree_id to validate_degree / analyze_degree to chain." +} +``` + +The `cache_id` is a `cache:` handle for the converted body — pass its +value to `validate_degree` / `analyze_degree` / `audit_degree` / `trim_degree` +(as their `degree_id`) to chain without re-pasting the JSON. For a cluster +pipeline file with no `program` set, `kind` is `"cluster"` and a `programs` +array lists the available program names (pass one back as `program`). + +**Example Prompts:** +- "Convert this ai-landscape program to the unified format" → calls with `json_content: "…"` +- "What programs are in this cluster file?" → calls with `json_path: "…"`, no `program` +- "Convert the BSCS program from that cluster and analyze it" → `convert_degree` then `analyze_degree` with the returned `degree_id` + ### `analyze_degree` Runs full degree analysis: generates all possible course plans, computes aggregate diff --git a/scripts/analyze-batch.sh b/scripts/analyze-batch.sh new file mode 100755 index 0000000..7ac35b9 --- /dev/null +++ b/scripts/analyze-batch.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Analyze many unified degree JSONs — one OS process per file, run JOBS at a +# time, each capped with a virtual-memory limit and a timeout. +# +# Why one process per file: an out-of-memory degree (e.g. a full-catalog scrape +# with thousands of courses) aborts *itself* under the ulimit instead of letting +# the OS OOM-killer take down the whole batch. Threads can't give this — they +# share one address space — so isolation requires separate processes. +# +# Usage: +# scripts/analyze-batch.sh [jobs] [-- extra nuanalytics flags] +# +# Examples: +# scripts/analyze-batch.sh .debug/converted .debug/metrics/json-dump 8 +# scripts/analyze-batch.sh '.debug/converted/*.unified.json' out 4 -- --max-plans 50000 +# +# Tunables (env): +# NA_BIN analyzer binary (default: ./target/release/nuanalytics) +# NA_MEM_KB per-process vmem cap, KB (default: 8000000 ~ 8 GB) +# NA_TIMEOUT per-process timeout, secs (default: 300) +# +# Keep JOBS * NA_MEM_KB at or below your RAM so concurrent workers don't +# collectively exhaust memory. +set -uo pipefail + +INPUT="${1:?usage: analyze-batch.sh [jobs] [-- extra flags]}" +NA_METRICS="${2:?metrics-dir required}" +JOBS="${3:-4}" + +# Everything after a literal `--` is passed through to `degree analyze`. +shift 3 2>/dev/null || shift "$#" +NA_EXTRA_STR="" +if [ "${1:-}" = "--" ]; then shift; NA_EXTRA_STR="$*"; fi + +NA_BIN="${NA_BIN:-./target/release/nuanalytics}" +NA_MEM_KB="${NA_MEM_KB:-8000000}" +NA_TIMEOUT="${NA_TIMEOUT:-300}" + +if [ ! -x "$NA_BIN" ]; then + echo "error: analyzer binary not found at '$NA_BIN'." >&2 + echo " build it first: cargo build --release (or set NA_BIN=...)" >&2 + exit 1 +fi + +mkdir -p "$NA_METRICS" "$NA_METRICS/logs" +FAILLOG="$NA_METRICS/failures.log" +: > "$FAILLOG" + +# Worker: analyze one file in an isolated, memory-capped, time-bounded subshell. +# Success drops its stderr log; failure keeps it and records the path. +analyze_one() { + local f="$1" base + base="$(basename "$f")" + # shellcheck disable=SC2206 # passthrough flags are simple space-free tokens + local extra=($NA_EXTRA_STR) + if ( + ulimit -v "$NA_MEM_KB" + timeout "$NA_TIMEOUT" "$NA_BIN" degree analyze "$f" \ + --no-report --metrics-dir "$NA_METRICS" "${extra[@]}" + ) >/dev/null 2>"$NA_METRICS/logs/$base.err"; then + rm -f "$NA_METRICS/logs/$base.err" + else + echo "$f" >> "$FAILLOG" + fi +} +export -f analyze_one +export NA_BIN NA_METRICS NA_MEM_KB NA_TIMEOUT NA_EXTRA_STR FAILLOG + +# Collect inputs (a directory, or a glob / space-separated list) and run the pool. +if [ -d "$INPUT" ]; then + find "$INPUT" -maxdepth 1 -name '*.json' -print0 +else + # shellcheck disable=SC2086 # intentional word-splitting of a glob argument + printf '%s\0' $INPUT +fi | xargs -0 -r -P "$JOBS" -I{} bash -c 'analyze_one "$@"' _ {} + +produced=$(find "$NA_METRICS" -maxdepth 1 -name '*_report.json' | wc -l | tr -d ' ') +failed=$(wc -l < "$FAILLOG" | tr -d ' ') +echo "----------------------------------------" +echo "reports produced : $produced" +echo "failed/skipped : $failed (see $FAILLOG; per-file stderr in $NA_METRICS/logs/)" diff --git a/src/assets/degree.schema.json b/src/assets/degree.schema.json new file mode 100644 index 0000000..13aa1b1 --- /dev/null +++ b/src/assets/degree.schema.json @@ -0,0 +1,182 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/Khoury-NeuCurricularAnalytics/NuAnalytics/degree.schema.json", + "title": "NuAnalytics Unified Degree Program", + "description": "The unified degree JSON: degree metadata, a requirements map, and a courses map. Prerequisites use the symmetric tagged form (a course string, or {\"and\"|\"or\": [...]}). Optional model fields are permitted (additionalProperties is not restricted).", + "type": "object", + "required": ["degree", "requirements", "courses"], + "properties": { + "degree": { "$ref": "#/$defs/degree" }, + "requirements": { + "type": "object", + "additionalProperties": { "$ref": "#/$defs/requirement" } + }, + "courses": { + "type": "object", + "additionalProperties": { "$ref": "#/$defs/course" } + }, + "analysis": { "$ref": "#/$defs/analysis" }, + "selected_plans": { "type": "array" }, + "conversion_warnings": { + "type": "array", + "items": { "type": "string" } + } + }, + "$defs": { + "prerequisite": { + "description": "Recursive prerequisite expression. A leaf is a course key string; an AND node is {\"and\": [...]}; an OR node is {\"or\": [...]}.", + "oneOf": [ + { "type": "string" }, + { + "type": "object", + "required": ["and"], + "properties": { + "and": { "type": "array", "items": { "$ref": "#/$defs/prerequisite" } } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": ["or"], + "properties": { + "or": { "type": "array", "items": { "$ref": "#/$defs/prerequisite" } } + }, + "additionalProperties": false + } + ] + }, + "metricStats": { + "type": "object", + "properties": { + "min": { "type": "number" }, + "max": { "type": "number" }, + "mean": { "type": "number" }, + "std_dev": { "type": "number" }, + "median": { "type": "number" }, + "q1": { "type": "number" }, + "q3": { "type": "number" } + } + }, + "degree": { + "type": "object", + "required": ["name"], + "properties": { + "name": { "type": "string" }, + "degree_type": { "type": "string" }, + "cip_code": { "type": ["string", "null"] }, + "system_type": { "type": "string", "examples": ["semester", "quarter"] }, + "id": { "type": ["string", "null"] }, + "institution": { "type": ["string", "null"] }, + "catalog_year": { "type": ["string", "null"] }, + "source_url": { "type": ["string", "null"] }, + "total_credits": { "type": ["integer", "null"] }, + "tags": { + "type": ["array", "null"], + "items": { "type": "string" }, + "description": "Program-level classification tags, e.g. [\"ai\", \"ai-concentration\"]." + } + } + }, + "course": { + "type": "object", + "required": ["name", "prefix", "number"], + "properties": { + "name": { "type": "string" }, + "prefix": { "type": "string" }, + "number": { "type": "string" }, + "credit_hours": { "type": "number" }, + "prerequisites": { "$ref": "#/$defs/prerequisite" }, + "corequisites": { "type": "array", "items": { "type": "string" } }, + "strict_corequisites": { "type": "array", "items": { "type": "string" } }, + "tags": { + "type": ["array", "null"], + "items": { "type": "string" }, + "description": "Course-level tags, e.g. [\"ai\"]." + }, + "metrics": { + "type": "object", + "description": "Per-course metric statistics (present in analysis report output).", + "properties": { + "complexity": { "$ref": "#/$defs/metricStats" }, + "centrality": { "$ref": "#/$defs/metricStats" }, + "delay": { "$ref": "#/$defs/metricStats" }, + "blocking": { "$ref": "#/$defs/metricStats" } + } + } + } + }, + "requirement": { + "type": "object", + "required": ["type"], + "properties": { + "type": { "enum": ["all", "select", "one_of"] }, + "name": { "type": ["string", "null"] }, + "category": { "type": ["string", "null"] }, + "courses": { "type": ["array", "null"], "items": { "type": "string" } }, + "count": { "type": ["integer", "null"] }, + "credits": { "type": ["integer", "null"] }, + "from": { "anyOf": [{ "$ref": "#/$defs/fromClause" }, { "type": "null" }] }, + "options": { "type": ["array", "null"] }, + "tags": { + "type": ["array", "null"], + "items": { "type": "string" }, + "description": "Requirement-group tags, e.g. [\"core\"], [\"ai\",\"required\"], [\"elective\"]." + } + } + }, + "fromClause": { + "type": "object", + "description": "Course pool a `select`/`one_of` requirement draws from. Combine an explicit `courses` list with subject `pattern`s (wildcards). A pure-wildcard pool with `credits` (e.g. pattern \"*:*\") resolves to placeholder electives, so general-education / free-elective buckets can be modeled WITHOUT enumerating every real course. The same wildcard support exists in the YAML format.", + "properties": { + "courses": { + "type": ["array", "null"], + "items": { "type": "string" }, + "description": "Explicit course keys in the pool." + }, + "pattern": { + "type": ["string", "null"], + "description": "Subject pattern / wildcard, e.g. \"CS:2500+\", \"CS:300-479\", or \"*:*\" for any course.", + "examples": ["CS:2500+", "CS:300-479", "*:*"] + }, + "include": { + "type": ["array", "null"], + "items": { "type": "string" }, + "description": "Additional patterns to fold into the pool, e.g. [\"CS:300-479\", \"MATH:300-479\"]." + }, + "exclude": { + "type": ["array", "null"], + "items": { "type": "string" }, + "description": "Courses or patterns to remove from the pool." + }, + "groups": { + "type": ["array", "null"], + "description": "Grouped selection: pick from N named groups of courses." + }, + "groups_required": { + "type": ["integer", "null"], + "description": "How many groups to satisfy (null = all)." + }, + "per_group": { + "type": ["integer", "null"], + "description": "How many courses to pick from each chosen group." + } + } + }, + "analysis": { + "type": "object", + "description": "Analysis block in report output.", + "properties": { + "variations_run": { "type": "integer" }, + "sample_type": { "type": "string", "examples": ["shuffled", "sequential", "stratified"] }, + "metrics": { + "type": "object", + "properties": { + "complexity": { "$ref": "#/$defs/metricStats" }, + "delay": { "$ref": "#/$defs/metricStats" }, + "credits": { "$ref": "#/$defs/metricStats" } + } + } + } + } + } +} diff --git a/src/cli/args.rs b/src/cli/args.rs index c2e8ac5..164f394 100644 --- a/src/cli/args.rs +++ b/src/cli/args.rs @@ -6,6 +6,11 @@ use std::path::PathBuf; use nu_analytics::config::ConfigOverrides; use nu_analytics::logger::Level; +/// Default number of concurrent worker processes for `degree analyze` on a +/// multi-file batch. A small, machine-independent default that keeps memory +/// bounded while still overlapping I/O-bound scrape conversions. +pub const DEFAULT_ANALYZE_JOBS: usize = 8; + /// CLI log level argument /// /// Represents log levels that can be passed via CLI arguments. Converts to lowercase @@ -243,6 +248,19 @@ pub enum DegreeSubcommand { /// Example: --include "CS3500,MATH2331,PHIL1145" #[arg(long, value_name = "COURSES", value_delimiter = ',')] include: Option>, + + /// Number of files to analyze concurrently, each in its own process so + /// a pathological degree (e.g. a full-catalog scrape) can't take down + /// the whole batch. Applies only when multiple files are given; use + /// `-j 1` to run sequentially in-process with full per-degree output. + #[arg(short = 'j', long, value_name = "N", default_value_t = DEFAULT_ANALYZE_JOBS)] + jobs: usize, + + /// Treat all input files as programs of one school and also emit a + /// combined `_school_report.json` rolling up degree-level + /// metrics across the programs. The value is the school name. + #[arg(long, value_name = "NAME")] + school: Option, }, /// Trim a degree program to a single entry path per course. @@ -300,6 +318,41 @@ pub enum DegreeSubcommand { #[arg(long, value_name = "COURSES", value_delimiter = ',')] include: Option>, }, + + /// Convert program file(s) to the unified degree JSON. + /// + /// Accepts raw ai-landscape program JSON (auto-detected and converted), + /// existing unified JSON, or YAML, and emits unified JSON with structured + /// prerequisites. Data-quality issues (e.g. assumed credits) are reported + /// and embedded as `conversion_warnings` in the output. + /// + /// This converter is transitional — once upstream emits unified JSON + /// directly it is no longer needed. + Convert { + /// Source program file(s). Shell wildcards are expanded by the shell. + #[arg(value_name = "FILES", num_args = 1..)] + files: Vec, + + /// Output destination. Without `-o`, each file is written next to its + /// input as `.unified.json`. With `-o`, the value is a + /// file (single input) or a directory (one output per input). + #[arg(short, long, value_name = "PATH")] + out: Option, + + /// Pretty-print the JSON output (default is compact, one line). + #[arg(long)] + pretty: bool, + }, + + /// Print the JSON Schema for the unified degree format. + /// + /// Useful for validating unified JSON files in other tools/pipelines. + /// Writes to `-o ` if given, otherwise prints to stdout. + Schema { + /// Write the schema to this file instead of stdout. + #[arg(short, long, value_name = "PATH")] + out: Option, + }, } #[derive(Debug, Subcommand)] @@ -379,25 +432,29 @@ pub enum Command { #[arg(long)] no_report: bool, }, - /// Validate, analyze, audit, or trim a degree program YAML file. + /// Validate, analyze, audit, trim, or convert a degree program (YAML or JSON). /// - /// Dispatches to one of several actions via subcommand. Circular - /// prerequisites are automatically broken by removing optional edges - /// to create a valid DAG for analysis. + /// Dispatches to one of several actions via subcommand. Inputs may be YAML + /// or unified-JSON degree files (raw ai-landscape JSON is auto-converted). + /// Circular prerequisites are automatically broken by removing optional + /// edges to create a valid DAG for analysis. /// /// # Examples /// ```sh /// # Validate one or more files /// nuanalytics degree validate samples/degrees/neu-khoury-bscs-boston.yaml /// - /// # Print the prerequisite graph - /// nuanalytics degree print-graph samples/degrees/csu-cs-bscs-general.yaml - /// /// # Run full plan-enumeration analysis /// nuanalytics degree analyze samples/degrees/csu-cs-bscs-general.yaml /// /// # Trim alternatives down to a single shared shortest path /// nuanalytics degree trim samples/degrees/neu-khoury-bscs-boston.yaml + /// + /// # Convert ai-landscape program JSON to unified JSON (glob into a dir) + /// nuanalytics degree convert ai-landscape-tools/validation_jsons/*.json -o converted/ + /// + /// # Emit the unified-degree JSON Schema + /// nuanalytics degree schema -o degree.schema.json /// ``` Degree { #[command(subcommand)] diff --git a/src/cli/commands/degree.rs b/src/cli/commands/degree.rs index 2c545a6..d0fbf37 100644 --- a/src/cli/commands/degree.rs +++ b/src/cli/commands/degree.rs @@ -7,8 +7,9 @@ use nu_analytics::core::degree::audit::{ detect_lowest_course_level, find_deep_chains, find_upper_level_without_prereqs, }; use nu_analytics::core::degree::{ - load_degree_from_yaml, PlanGenerator, PlanGeneratorConfig, PlanSelector, PlanSelectorConfig, - PlanValidator, PlanValidatorConfig, PlanVariant, SamplingStrategy, + load_degree_from_json, load_degree_from_yaml, DegreeParseError, PlanGenerator, + PlanGeneratorConfig, PlanSelector, PlanSelectorConfig, PlanValidator, PlanValidatorConfig, + PlanVariant, SamplingStrategy, }; use nu_analytics::core::metrics::compute_all_metrics; use nu_analytics::core::models::course_graph::{CourseNode, PrerequisiteEdge, PrerequisiteType}; @@ -47,7 +48,7 @@ pub fn validate_degree(degree_path: &Path, verbose: bool) -> Result<(), String> } // Load the degree program - let program = load_degree_from_yaml(degree_path).map_err(|e| { + let program = load_degree_auto(degree_path).map_err(|e| { format!( "Failed to load degree program from {}: {}", degree_path.display(), @@ -134,7 +135,7 @@ fn load_and_build_graph( eprintln!("Loading degree program from: {}", degree_path.display()); } - let program = load_degree_from_yaml(degree_path).map_err(|e| { + let program = load_degree_auto(degree_path).map_err(|e| { format!( "Failed to load degree program from {}: {}", degree_path.display(), @@ -267,7 +268,7 @@ pub fn audit_degree(degree_path: &Path, config: &Config, verbose: bool) -> Resul } // Load the degree program - let program = load_degree_from_yaml(degree_path).map_err(|e| { + let program = load_degree_auto(degree_path).map_err(|e| { format!( "Failed to load degree program from {}: {}", degree_path.display(), @@ -477,6 +478,11 @@ pub struct AnalyzeOptions { pub verbose: bool, /// Courses to always include in all plans pub include_courses: Option>, + /// Concurrent worker processes for a multi-file batch (1 = in-process). + pub jobs: usize, + /// When set, treat the inputs as programs of one school and also emit a + /// combined `_school_report.json` rolling up degree-level metrics. + pub school: Option, } /// Run `degree validate` over one or more files. @@ -494,9 +500,300 @@ pub fn run_audit(files: &[PathBuf], config: &Config, verbose: bool) { run_batch(files, |path| audit_degree(path, config, verbose)); } -/// Run `degree analyze` over one or more files. +/// Environment marker set on spawned worker processes so they run a single +/// file in-process instead of recursively spawning their own pool. +const WORKER_ENV: &str = "NU_ANALYZE_WORKER"; + +/// Run `degree analyze`. A multi-file batch is processed as a pool of isolated +/// worker processes (`--jobs`, default 8) so one pathological degree can't take +/// down the whole run; single-file, `--school`, `-j 1`, and worker-mode +/// invocations run in-process. pub fn run_analyze(files: &[PathBuf], options: &AnalyzeOptions, config: &Config) { - run_batch(files, |path| analyze_degree(path, options, config)); + let in_worker = std::env::var_os(WORKER_ENV).is_some(); + if in_worker || options.jobs <= 1 || options.school.is_some() { + run_analyze_inprocess(files, options, config); + return; + } + + let inputs = filter_degree_inputs(files); + match inputs.len() { + 0 => { + eprintln!("Error: No degree files to process after filtering."); + process::exit(1); + } + // A single file gains nothing from a worker process; run it in-process + // so the user gets the full per-degree output. + 1 => run_analyze_inprocess(files, options, config), + _ => run_analyze_parallel(&inputs, options), + } +} + +/// Poll interval for reaping finished worker processes. +const WORKER_POLL: std::time::Duration = std::time::Duration::from_millis(50); + +/// The metrics output directory from `options`, defaulting to `metrics/`. +fn metrics_dir_or_default(options: &AnalyzeOptions) -> PathBuf { + options + .metrics_dir + .clone() + .unwrap_or_else(|| PathBuf::from("metrics")) +} + +/// Analyze a multi-file batch as a rolling pool of up to `options.jobs` worker +/// processes (each re-invokes this binary on one file). Worker output is +/// suppressed; a progress line and final summary are printed, and any failures +/// (path + exit status) are written to `/failures.log`. +/// +/// Isolation here is reactive: a worker that exhausts memory is killed by the +/// OS and recorded as a failure. Unlike `scripts/analyze-batch.sh`, the pool +/// imposes no per-process memory cap or timeout — use that script when a hard +/// `ulimit -v` / `timeout` guard is required. +fn run_analyze_parallel(inputs: &[&Path], options: &AnalyzeOptions) { + use std::io::Write; + + let exe = match std::env::current_exe() { + Ok(p) => p, + Err(e) => { + eprintln!("Error: cannot locate the nuanalytics executable: {e}"); + process::exit(1); + } + }; + let metrics_dir = metrics_dir_or_default(options); + + // Write the index.csv header once so concurrent workers only append rows. + if !options.no_csv { + if let Err(e) = + nu_analytics::core::report::plan_export::write_index_csv_header(&metrics_dir) + { + eprintln!("Warning: could not initialize index.csv: {e}"); + } + } + + let jobs = options.jobs.max(1); + let child_flags = analyze_child_flags(options); + let total = inputs.len(); + println!("Analyzing {total} programs with {jobs} worker process(es)…"); + + let mut next = 0usize; + let mut running: Vec<(PathBuf, std::process::Child)> = Vec::new(); + let mut done = 0usize; + let mut failed: Vec<(PathBuf, String)> = Vec::new(); + + loop { + while running.len() < jobs && next < total { + let f = inputs[next]; + next += 1; + match spawn_analyze_worker(&exe, f, &child_flags) { + Ok(child) => running.push((f.to_path_buf(), child)), + Err(e) => { + failed.push((f.to_path_buf(), format!("spawn error: {e}"))); + done += 1; + } + } + } + if running.is_empty() { + break; + } + + let reaped = reap_finished(&mut running, &mut failed); + if reaped > 0 { + done += reaped; + print!("\r {done}/{total} done ({} failed) ", failed.len()); + let _ = std::io::stdout().flush(); + } else { + std::thread::sleep(WORKER_POLL); + } + } + println!(); + + report_pool_outcome(total, &failed, &metrics_dir); +} + +/// Reap every worker that has finished, removing it from `running` and +/// recording non-success exits in `failed` (path + status string). Returns the +/// number reaped this pass (0 ⇒ nothing finished yet). +fn reap_finished( + running: &mut Vec<(PathBuf, std::process::Child)>, + failed: &mut Vec<(PathBuf, String)>, +) -> usize { + let mut reaped = 0; + let mut i = 0; + while i < running.len() { + match running[i].1.try_wait() { + Ok(Some(status)) => { + let (f, _) = running.remove(i); + if !status.success() { + // ExitStatus Display includes the signal on Unix, so an + // OOM-killed worker reads e.g. "signal: 9 (SIGKILL)". + failed.push((f, status.to_string())); + } + reaped += 1; + } + Ok(None) => i += 1, + Err(e) => { + let (f, _) = running.remove(i); + failed.push((f, format!("wait error: {e}"))); + reaped += 1; + } + } + } + reaped +} + +/// Spawn one isolated worker process to analyze `file` (output suppressed). +fn spawn_analyze_worker( + exe: &Path, + file: &Path, + flags: &[String], +) -> std::io::Result { + std::process::Command::new(exe) + .arg("degree") + .arg("analyze") + .arg(file) + .args(flags) + .env(WORKER_ENV, "1") + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn() +} + +/// Print the batch summary; on any failure, write `failures.log` (one +/// `pathstatus` line each) and exit 1. +fn report_pool_outcome(total: usize, failed: &[(PathBuf, String)], metrics_dir: &Path) { + use std::fmt::Write as _; + + let succeeded = total - failed.len(); + println!( + "✓ analyzed {succeeded}/{total} programs ({} failed)", + failed.len() + ); + if failed.is_empty() { + return; + } + let faillog = metrics_dir.join("failures.log"); + let mut body = String::new(); + for (path, reason) in failed { + let _ = writeln!(body, "{}\t{reason}", path.display()); + } + if std::fs::write(&faillog, body).is_ok() { + println!(" failures listed in {}", faillog.display()); + } + if let Some((first, _)) = failed.first() { + println!( + " re-run one for details: nuanalytics degree analyze {} -j 1", + first.display() + ); + } + process::exit(1); +} + +/// Reconstruct the `degree analyze` flags for a worker child from `options`. +/// Excludes `--jobs` (the worker marker prevents re-pooling) and `--school` +/// (the pool only runs when school mode is off). +/// +/// This must mirror every *result-affecting* flag on the `Analyze` subcommand +/// in `src/cli/args.rs`: a flag added there but omitted here is silently dropped +/// for pooled runs, so workers would analyze with different settings than the +/// user asked for. Covered by `test_analyze_child_flags_*`. +fn analyze_child_flags(o: &AnalyzeOptions) -> Vec { + let mut a: Vec = Vec::new(); + if let Some(d) = &o.metrics_dir { + a.push("--metrics-dir".into()); + a.push(d.display().to_string()); + } + if let Some(d) = &o.report_dir { + a.push("--report-dir".into()); + a.push(d.display().to_string()); + } + if o.no_report { + a.push("--no-report".into()); + } + if o.no_csv { + a.push("--no-csv".into()); + } + if let Some(n) = o.max_plans { + a.push("--max-plans".into()); + a.push(n.to_string()); + } + if let Some(n) = o.sample_plans { + a.push("--sample-plans".into()); + a.push(n.to_string()); + } + if let Some(s) = &o.sampling_strategy { + a.push("--sampling-strategy".into()); + a.push(s.clone()); + } + if let Some(s) = &o.calc_strategy { + a.push("--calc-strategy".into()); + a.push(s.clone()); + } + if o.full_run { + a.push("--full-run".into()); + } + if let Some(courses) = &o.include_courses { + if !courses.is_empty() { + a.push("--include".into()); + a.push(courses.join(",")); + } + } + a +} + +/// Run `degree analyze` in-process (single file, school mode, `-j 1`, or as a +/// spawned worker). Without `--school`, each file is analyzed independently; +/// with `--school`, a combined `_school_report.json` is also written. +fn run_analyze_inprocess(files: &[PathBuf], options: &AnalyzeOptions, config: &Config) { + let Some(school_name) = options.school.clone() else { + run_batch(files, |path| { + analyze_degree(path, options, config).map(|_| ()) + }); + return; + }; + + // School mode: collect per-program rollups across the batch. + let inputs = filter_degree_inputs(files); + if inputs.is_empty() { + eprintln!("Error: No degree files to process after filtering."); + process::exit(1); + } + + let total = inputs.len(); + let mut rollups = Vec::new(); + let mut had_failure = false; + for (idx, path) in inputs.iter().enumerate() { + if total > 1 { + if idx > 0 { + print_separator(); + } + println!("=== [{}/{}] {} ===", idx + 1, total, path.display()); + } + match analyze_degree(path, options, config) { + Ok(rollup) => rollups.push(rollup), + Err(e) => { + eprintln!("Error: {e}"); + had_failure = true; + } + } + } + + if !rollups.is_empty() { + let metrics_dir = options + .metrics_dir + .as_ref() + .map_or_else(|| std::path::PathBuf::from("metrics"), Clone::clone); + match nu_analytics::core::report::unified_report::export_school_report_json( + &school_name, + &rollups, + &metrics_dir, + ) { + Ok(path) => println!("✓ School report: {}", path.display()), + Err(e) => eprintln!("Error: Failed to write school report: {e}"), + } + } + + if had_failure { + process::exit(1); + } } /// Run `degree trim` over one or more input files. @@ -522,19 +819,19 @@ pub fn run_trim( process::exit(1); } - let yaml_inputs = filter_yaml_inputs(inputs); - if yaml_inputs.is_empty() { - eprintln!("Error: No YAML files to process after filtering."); + let degree_inputs = filter_degree_inputs(inputs); + if degree_inputs.is_empty() { + eprintln!("Error: No degree files to process after filtering."); process::exit(1); } let dir_mode = out.is_some_and(looks_like_directory); - if let Some(file_out) = out.filter(|_| yaml_inputs.len() > 1 && !dir_mode) { + if let Some(file_out) = out.filter(|_| degree_inputs.len() > 1 && !dir_mode) { eprintln!( "Error: -o {} is a file path, but {} input files were given; pass a directory (or end the path with '/') instead", file_out.display(), - yaml_inputs.len() + degree_inputs.len() ); process::exit(1); } @@ -549,9 +846,9 @@ pub fn run_trim( } } - let total = yaml_inputs.len(); + let total = degree_inputs.len(); let mut had_failure = false; - for (idx, input) in yaml_inputs.iter().enumerate() { + for (idx, input) in degree_inputs.iter().enumerate() { if total > 1 { if idx > 0 { print_separator(); @@ -584,9 +881,9 @@ where process::exit(1); } - let yaml_files = filter_yaml_inputs(files); + let yaml_files = filter_degree_inputs(files); if yaml_files.is_empty() { - eprintln!("Error: No YAML files to process after filtering."); + eprintln!("Error: No degree files to process after filtering."); process::exit(1); } @@ -611,16 +908,20 @@ where } } -/// Pick out the YAML inputs from a mixed list of paths, warning to stderr -/// about anything skipped. Shared between [`run_trim`] and [`run_batch`]. -fn filter_yaml_inputs(files: &[PathBuf]) -> Vec<&Path> { +/// Filter `files` to those `accept`ed, warning to stderr (with `expected` naming +/// the wanted kind) about each path skipped. Shared by the degree subcommands. +fn filter_inputs<'a>( + files: &'a [PathBuf], + accept: fn(&Path) -> bool, + expected: &str, +) -> Vec<&'a Path> { files .iter() .filter_map(|p| { - if is_yaml_path(p) { + if accept(p) { Some(p.as_path()) } else { - eprintln!("Skipping non-YAML file: {}", p.display()); + eprintln!("Skipping non-{expected} file: {}", p.display()); None } }) @@ -634,6 +935,25 @@ fn is_yaml_path(path: &Path) -> bool { .is_some_and(|ext| ext.eq_ignore_ascii_case("yaml") || ext.eq_ignore_ascii_case("yml")) } +/// Returns `true` if the path has a `.json` extension (case-insensitive). +fn is_json_path(path: &Path) -> bool { + path.extension() + .and_then(|ext| ext.to_str()) + .is_some_and(|ext| ext.eq_ignore_ascii_case("json")) +} + +/// A degree input is a YAML or JSON file (JSON may be unified or raw +/// ai-landscape — both handled by [`load_degree_auto`]). +fn is_degree_input_path(path: &Path) -> bool { + is_yaml_path(path) || is_json_path(path) +} + +/// Filter inputs to degree files (YAML or JSON), warning about the rest. +/// Used by batch commands that accept both formats. +fn filter_degree_inputs(files: &[PathBuf]) -> Vec<&Path> { + filter_inputs(files, is_degree_input_path, "degree (.yaml/.yml/.json)") +} + /// Filename suffix appended to the input stem when `degree trim` runs /// without an explicit `-o`/`--out` path. const TRIM_OUTPUT_SUFFIX: &str = "_trimmed"; @@ -668,19 +988,37 @@ fn looks_like_directory(p: &Path) -> bool { s.ends_with('/') || s.ends_with(std::path::MAIN_SEPARATOR) } -/// Resolve the on-disk output path for `input` given the user's `-o` -/// argument and whether we determined it to be a directory destination. -fn resolve_trim_output(input: &Path, out: Option<&Path>, dir_mode: bool) -> PathBuf { +/// Resolve an output path from a `-o` argument: `None` writes `filename` next to +/// the input, a directory destination joins `filename` under it, and a file +/// destination is used verbatim. Shared by `trim` and `convert`. +fn resolve_output_path( + input: &Path, + out: Option<&Path>, + dir_mode: bool, + filename: &str, +) -> PathBuf { match out { - None => default_trim_output(input), - Some(dir) if dir_mode => { - let (stem, ext) = trim_output_stem_ext(input); - dir.join(format!("{stem}{TRIM_OUTPUT_SUFFIX}.{ext}")) - } + None => input.with_file_name(filename), + Some(dir) if dir_mode => dir.join(filename), Some(file) => file.to_path_buf(), } } +/// Resolve the on-disk output path for `degree trim` given the user's `-o` +/// argument and whether we determined it to be a directory destination. +fn resolve_trim_output(input: &Path, out: Option<&Path>, dir_mode: bool) -> PathBuf { + if out.is_none() { + return default_trim_output(input); + } + let (stem, ext) = trim_output_stem_ext(input); + resolve_output_path( + input, + out, + dir_mode, + &format!("{stem}{TRIM_OUTPUT_SUFFIX}.{ext}"), + ) +} + /// Load `input`, apply /// [`trim_program`](nu_analytics::core::degree::trim_program) with the /// given options, and write the result to `out_path`. Prints a success @@ -693,9 +1031,9 @@ fn trim_one( include: Option<&[String]>, verbose: bool, ) -> Result<(), String> { - use nu_analytics::core::degree::{save_degree_to_yaml, trim_program, TrimOptions}; + use nu_analytics::core::degree::{trim_program, TrimOptions}; - let program = load_degree_from_yaml(input) + let program = load_degree_auto(input) .map_err(|e| format!("Failed to load {}: {}", input.display(), e))?; let opts = TrimOptions { @@ -714,8 +1052,9 @@ fn trim_one( )); } - save_degree_to_yaml(&trimmed, out_path) - .map_err(|e| format!("Failed to write {}: {}", out_path.display(), e))?; + // Round-trip the input format: a JSON input yields a trimmed JSON file + // (resolve_trim_output preserves the extension), YAML stays YAML. + save_degree_auto(&trimmed, out_path)?; println!("✓ Trimmed degree written to: {}", out_path.display()); if verbose { @@ -1113,7 +1452,7 @@ fn analyze_degree( degree_path: &Path, options: &AnalyzeOptions, config: &Config, -) -> Result<(), String> { +) -> Result { let verbose = options.verbose; // Load and validate the degree program @@ -1196,10 +1535,354 @@ fn analyze_degree( // Print summary print_analysis_summary(&ctx, &aggregator, plans_processed); + Ok( + nu_analytics::core::report::unified_report::ProgramRollup::from_analysis( + ctx.program, + &aggregator, + ), + ) +} + +/// The JSON Schema for the unified degree format, embedded at build time. +const UNIFIED_DEGREE_SCHEMA: &str = include_str!("../../assets/degree.schema.json"); + +/// Emit the unified-degree JSON Schema to a file (`out`) or stdout. +pub fn run_schema(out: Option<&Path>) { + if let Some(path) = out { + match std::fs::write(path, UNIFIED_DEGREE_SCHEMA) { + Ok(()) => println!("✓ Schema written to: {}", path.display()), + Err(e) => { + eprintln!("Error: Failed to write {}: {e}", path.display()); + process::exit(1); + } + } + } else { + print!("{UNIFIED_DEGREE_SCHEMA}"); + } +} + +/// Run `degree convert` over one or more inputs, emitting unified JSON. +pub fn run_convert(files: &[PathBuf], out: Option<&Path>, pretty: bool, verbose: bool) { + // Directory mode when -o is a directory, or when multiple inputs share one -o. + let dir_mode = out.is_some_and(looks_like_directory) || (out.is_some() && files.len() > 1); + run_batch(files, |path| { + convert_file(path, out, dir_mode, pretty, verbose) + }); +} + +/// Convert one input. A cluster pipeline file (the full multi-stage ai-landscape +/// state) expands to one unified JSON per program; everything else is a single +/// unified file. +fn convert_file( + input: &Path, + out: Option<&Path>, + dir_mode: bool, + pretty: bool, + verbose: bool, +) -> Result<(), String> { + let contents = std::fs::read_to_string(input) + .map_err(|e| format!("Failed to read {}: {e}", input.display()))?; + + if is_json_path(input) { + // Only parse the Value to route by shape; malformed JSON falls through + // to `convert_single`, which surfaces a proper parse error. + if let Ok(value) = serde_json::from_str::(&contents) { + if let Some(programs) = nu_analytics::core::degree::extract_cluster_programs(&value) { + let out_dir = cluster_out_dir(input, out); + return convert_cluster(input, &programs, &out_dir, pretty, verbose); + } + // Valid JSON that is neither a cluster file, an ai-landscape program + // (`courses` category map), nor a unified degree (top-level `degree`) + // is skipped rather than failing the batch (e.g. pipeline sidecar + // files like checkpoint/metrics in a cluster dump). + if !is_landscape_value(&value) && value.get("degree").is_none() { + println!( + "• {}: skipped (not a degree/program/cluster file)", + input.display() + ); + return Ok(()); + } + } + } + + let out_path = resolve_convert_output(input, out, dir_mode); + convert_single(input, &out_path, &contents, pretty, verbose) +} + +/// Filename suffix for `degree convert` output (unified JSON). +const CONVERT_OUTPUT_SUFFIX: &str = ".unified.json"; + +/// Separator between school and program in a cluster output filename. +const CLUSTER_NAME_SEP: &str = "__"; + +/// File stem of `input` as a `&str`, or `default` when missing/non-UTF-8. +fn file_stem_or<'a>(input: &'a Path, default: &'a str) -> &'a str { + input + .file_stem() + .and_then(|s| s.to_str()) + .unwrap_or(default) +} + +/// Output path for `degree convert`: `.unified.json` (next to input, or +/// inside an `-o` directory), or the verbatim `-o` file for a single input. +fn resolve_convert_output(input: &Path, out: Option<&Path>, dir_mode: bool) -> PathBuf { + let stem = file_stem_or(input, "degree"); + resolve_output_path( + input, + out, + dir_mode, + &format!("{stem}{CONVERT_OUTPUT_SUFFIX}"), + ) +} + +/// Convert a single-program input (YAML, unified JSON, or a flat ai-landscape +/// program file) to one unified JSON at `out_path`. +fn convert_single( + input: &Path, + out_path: &Path, + contents: &str, + pretty: bool, + verbose: bool, +) -> Result<(), String> { + use nu_analytics::core::degree::json_parser::{ + parse_degree_json_with_warnings, to_unified_value, + }; + + if out_path == input { + return Err(format!( + "refusing to overwrite input file {}; pass an explicit -o path", + input.display() + )); + } + + let (program, warnings) = if is_json_path(input) { + parse_degree_json_with_warnings(contents) + .map_err(|e| format!("Failed to parse {}: {e}", input.display()))? + } else { + let program = load_degree_from_yaml(input) + .map_err(|e| format!("Failed to load {}: {e}", input.display()))?; + (program, Vec::new()) + }; + + let mut value = to_unified_value(&program) + .map_err(|e| format!("Failed to build unified JSON for {}: {e}", input.display()))?; + write_unified_value(&mut value, &warnings, out_path, pretty)?; + + println!("✓ Converted {} -> {}", input.display(), out_path.display()); + report_warnings(&warnings, verbose); + Ok(()) +} + +/// Expand a cluster pipeline file into one unified JSON per program, written as +/// `__.unified.json` under `out_dir`. +fn convert_cluster( + input: &Path, + programs: &[(String, nu_analytics::core::degree::LandscapeProgram)], + out_dir: &Path, + pretty: bool, + verbose: bool, +) -> Result<(), String> { + if programs.is_empty() { + println!("• {}: no convertible programs", input.display()); + return Ok(()); + } + + let school = file_stem_or(input, "school"); + let mut total_warnings = 0usize; + // Distinct program names can sanitize to the same stem; disambiguate so no + // program silently overwrites another. + let mut used_stems: HashSet = HashSet::new(); + for (name, prog) in programs { + total_warnings += write_cluster_program( + input, + school, + name, + prog, + out_dir, + &mut used_stems, + pretty, + verbose, + )?; + } + + let warn_note = if total_warnings > 0 { + format!(", {total_warnings} warning(s)") + } else { + String::new() + }; + println!( + "✓ {}: {} program(s){warn_note}", + input.display(), + programs.len() + ); Ok(()) } -/// Load and validate a degree program from YAML +/// Convert one cluster program to `__.unified.json` under +/// `out_dir` (disambiguating colliding stems via `used_stems`), returning its +/// conversion-warning count. +#[allow(clippy::too_many_arguments)] +fn write_cluster_program( + input: &Path, + school: &str, + name: &str, + prog: &nu_analytics::core::degree::LandscapeProgram, + out_dir: &Path, + used_stems: &mut HashSet, + pretty: bool, + verbose: bool, +) -> Result { + use nu_analytics::core::degree::{convert_landscape, json_parser::to_unified_value}; + + let result = convert_landscape(prog); + let mut value = to_unified_value(&result.program).map_err(|e| { + format!( + "Failed to build unified JSON for {} / {name}: {e}", + input.display() + ) + })?; + let base = format!( + "{}{CLUSTER_NAME_SEP}{}", + safe_filename(school), + safe_filename(name) + ); + let stem = unique_stem(base, used_stems); + let out_path = out_dir.join(format!("{stem}{CONVERT_OUTPUT_SUFFIX}")); + write_unified_value(&mut value, &result.warnings, &out_path, pretty)?; + if verbose { + println!(" ✓ {}", out_path.display()); + } + Ok(result.warnings.len()) +} + +/// Directory destination for a cluster file's per-program outputs: the `-o` +/// path (always a directory, since a cluster expands to many files; created on +/// demand), or next to the input when `-o` is omitted. +fn cluster_out_dir(input: &Path, out: Option<&Path>) -> PathBuf { + if let Some(dir) = out { + return dir.to_path_buf(); + } + // `-o` omitted: write next to the input. `Path::parent` is `Some("")` for a + // bare filename, so treat an empty parent as the current directory. + input + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .map_or_else(|| PathBuf::from("."), Path::to_path_buf) +} + +/// Embed `conversion_warnings` (when any), create parent dirs, then write +/// `value` to `out_path` as JSON (pretty or compact). +fn write_unified_value( + value: &mut serde_json::Value, + warnings: &[String], + out_path: &Path, + pretty: bool, +) -> Result<(), String> { + if !warnings.is_empty() { + if let Some(obj) = value.as_object_mut() { + obj.insert( + "conversion_warnings".to_string(), + serde_json::Value::Array( + warnings + .iter() + .map(|w| serde_json::Value::String(w.clone())) + .collect(), + ), + ); + } + } + if let Some(parent) = out_path.parent() { + if !parent.as_os_str().is_empty() { + std::fs::create_dir_all(parent) + .map_err(|e| format!("Failed to create {}: {e}", parent.display()))?; + } + } + // Emit with `degree` first for readability (see unified_value_to_string). + let text = nu_analytics::core::degree::unified_value_to_string(value, pretty) + .map_err(|e| format!("Failed to serialize JSON for {}: {e}", out_path.display()))?; + std::fs::write(out_path, text) + .map_err(|e| format!("Failed to write {}: {e}", out_path.display())) +} + +/// Print a conversion-warning tally, expanding each warning when `verbose`. +fn report_warnings(warnings: &[String], verbose: bool) { + if warnings.is_empty() { + return; + } + println!(" {} conversion warning(s)", warnings.len()); + if verbose { + for w in warnings { + println!(" - {w}"); + } + } +} + +/// Return a filename stem unique within `used`: `base`, else `base-2`, `base-3`, +/// … Records the chosen stem in `used`. +fn unique_stem(base: String, used: &mut HashSet) -> String { + if used.insert(base.clone()) { + return base; + } + let mut n = 2; + loop { + let candidate = format!("{base}-{n}"); + if used.insert(candidate.clone()) { + return candidate; + } + n += 1; + } +} + +/// True if `value` is an ai-landscape flat program: a `courses` object whose +/// values are arrays (category -> list). +fn is_landscape_value(value: &serde_json::Value) -> bool { + value + .get("courses") + .and_then(serde_json::Value::as_object) + .is_some_and(|m| m.values().next().is_some_and(serde_json::Value::is_array)) +} + +/// Replace filesystem-hostile characters with `_`. Local to this binary crate; +/// the library's equivalent (`plan_export::sanitize_filename`) is crate-private. +fn safe_filename(s: &str) -> String { + s.chars() + .map(|c| match c { + '/' | '\\' | ':' | '*' | '?' | '"' | '<' | '>' | '|' | ' ' => '_', + _ => c, + }) + .collect() +} + +/// Load a degree program, dispatching on file extension: `.json` uses the +/// unified-JSON loader (which also auto-converts raw ai-landscape files), and +/// everything else is treated as YAML. +fn load_degree_auto>( + path: P, +) -> Result { + let path = path.as_ref(); + if is_json_path(path) { + load_degree_from_json(path) + } else { + load_degree_from_yaml(path) + } +} + +/// Save a degree program, dispatching on file extension to mirror +/// [`load_degree_auto`]: `.json` writes unified JSON, everything else YAML. The +/// error carries the destination path. +fn save_degree_auto( + program: &nu_analytics::core::DegreeProgram, + path: &Path, +) -> Result<(), String> { + use nu_analytics::core::degree::{save_degree_to_json, save_degree_to_yaml}; + let result = if is_json_path(path) { + save_degree_to_json(program, path) + } else { + save_degree_to_yaml(program, path) + }; + result.map_err(|e| format!("Failed to write {}: {e}", path.display())) +} + fn load_degree_program( degree_path: &Path, verbose: bool, @@ -1209,7 +1892,7 @@ fn load_degree_program( eprintln!("Loading degree program from: {}", degree_path.display()); } - let program = load_degree_from_yaml(degree_path).map_err(|e| { + let program = load_degree_auto(degree_path).map_err(|e| { format!( "Failed to load degree program from {}: {}", degree_path.display(), @@ -1466,6 +2149,25 @@ fn generate_analysis_outputs( } } } + + // Unified metrics-rich report JSON (the whole degree structure plus + // degree- and course-level metrics) for downstream viz/DB. Grouped with + // the other metrics-dir exports, so `--no-csv` suppresses it too. + let sample_type = sampling_strategy_label(&ctx.gen_config.sampling_strategy); + match nu_analytics::core::report::unified_report::export_degree_report_json( + ctx.program, + aggregator, + selected, + sample_type, + &metrics_dir, + ) { + Ok(path) => outputs_generated.push(format!("Report JSON: {}", path.display())), + Err(e) => { + if ctx.verbose { + eprintln!("Warning: Failed to export unified report JSON: {e}"); + } + } + } } if ctx.verbose && !outputs_generated.is_empty() { @@ -1479,6 +2181,16 @@ fn generate_analysis_outputs( Ok(outputs_generated) } +/// Human-readable label for a sampling strategy (used as `sample_type` in the +/// unified report JSON). +const fn sampling_strategy_label(strategy: &SamplingStrategy) -> &'static str { + match strategy { + SamplingStrategy::Sequential => "sequential", + SamplingStrategy::Shuffled => "shuffled", + SamplingStrategy::Stratified => "stratified", + } +} + /// Generate HTML report fn generate_html_report( ctx: &AnalysisContext<'_>, @@ -2232,6 +2944,91 @@ fn print_separator() { mod tests { use super::*; + #[test] + fn test_unique_stem_disambiguates_collisions() { + let mut used = HashSet::new(); + assert_eq!( + unique_stem("School__CS".to_string(), &mut used), + "School__CS" + ); + // Same base again -> suffixed. + assert_eq!( + unique_stem("School__CS".to_string(), &mut used), + "School__CS-2" + ); + assert_eq!( + unique_stem("School__CS".to_string(), &mut used), + "School__CS-3" + ); + // A distinct base is untouched. + assert_eq!( + unique_stem("School__AI".to_string(), &mut used), + "School__AI" + ); + } + + #[test] + fn test_safe_filename_replaces_hostile_chars() { + assert_eq!( + safe_filename("Computer Science (BS): A/B"), + "Computer_Science_(BS)__A_B" + ); + // Clean names pass through untouched. + assert_eq!(safe_filename("CS-BS_2024"), "CS-BS_2024"); + } + + #[test] + fn test_cluster_out_dir_branches() { + // None -> next to the input (parent dir); bare filename -> ".". + assert_eq!( + cluster_out_dir(Path::new("clusters/uni.json"), None), + PathBuf::from("clusters") + ); + assert_eq!( + cluster_out_dir(Path::new("uni.json"), None), + PathBuf::from(".") + ); + // Any `-o` is the output directory (a cluster expands to many files, + // so it can't target a single file). + assert_eq!( + cluster_out_dir(Path::new("clusters/uni.json"), Some(Path::new("out"))), + PathBuf::from("out") + ); + } + + #[test] + fn test_is_landscape_value() { + // `courses` object whose first value is an array -> landscape. + let yes: serde_json::Value = + serde_json::from_str(r#"{"courses":{"cs_course_core":[{"course_code":"CS1"}]}}"#) + .unwrap(); + assert!(is_landscape_value(&yes)); + // Unified: course keys map to objects, not arrays. + let unified: serde_json::Value = + serde_json::from_str(r#"{"courses":{"CS1":{"name":"Intro"}}}"#).unwrap(); + assert!(!is_landscape_value(&unified)); + // No `courses`, or `courses` not an object. + assert!(!is_landscape_value(&serde_json::json!({"degree": "BS"}))); + assert!(!is_landscape_value(&serde_json::json!({"courses": []}))); + } + + #[test] + fn test_resolve_convert_output_branches() { + let input = Path::new("degrees/neu.json"); + assert_eq!( + resolve_convert_output(input, None, false), + PathBuf::from("degrees/neu.unified.json") + ); + assert_eq!( + resolve_convert_output(input, Some(Path::new("out")), true), + PathBuf::from("out/neu.unified.json") + ); + assert_eq!( + resolve_convert_output(input, Some(Path::new("out/explicit.json")), false), + PathBuf::from("out/explicit.json") + ); + } + #[test] fn test_is_yaml_path_yaml_extension() { assert!(is_yaml_path(Path::new("foo.yaml"))); @@ -2341,4 +3138,170 @@ mod tests { PathBuf::from("out/explicit.yaml") ); } + + #[test] + fn test_analyze_child_flags_default_is_empty() { + // A worker started from default options should carry no extra flags. + let flags = analyze_child_flags(&AnalyzeOptions::default()); + assert!(flags.is_empty(), "expected no flags, got {flags:?}"); + } + + #[test] + fn test_analyze_child_flags_excludes_jobs_and_school() { + // --jobs and --school must never be forwarded: the worker marker stops + // re-pooling and the pool only runs when school mode is off. + let opts = AnalyzeOptions { + jobs: 16, + school: Some("Northeastern".to_string()), + ..Default::default() + }; + let flags = analyze_child_flags(&opts); + assert!(!flags.iter().any(|f| f == "--jobs" || f == "-j")); + assert!(!flags.iter().any(|f| f == "--school")); + assert!(flags.is_empty(), "expected no flags, got {flags:?}"); + } + + #[test] + fn test_analyze_child_flags_propagates_result_affecting_options() { + let opts = AnalyzeOptions { + metrics_dir: Some(PathBuf::from("m")), + report_dir: Some(PathBuf::from("r")), + no_report: true, + no_csv: true, + max_plans: Some(500), + sample_plans: Some(50), + sampling_strategy: Some("shuffled".to_string()), + calc_strategy: Some("median".to_string()), + full_run: true, + include_courses: Some(vec!["CS3500".to_string(), "MATH2331".to_string()]), + ..Default::default() + }; + let flags = analyze_child_flags(&opts); + let joined = flags.join(" "); + for expected in [ + "--metrics-dir m", + "--report-dir r", + "--no-report", + "--no-csv", + "--max-plans 500", + "--sample-plans 50", + "--sampling-strategy shuffled", + "--calc-strategy median", + "--full-run", + "--include CS3500,MATH2331", + ] { + assert!( + joined.contains(expected), + "missing {expected:?} in {joined:?}" + ); + } + } + + #[test] + fn test_analyze_child_flags_empty_include_is_omitted() { + // An empty include list must not produce a dangling `--include`. + let opts = AnalyzeOptions { + include_courses: Some(Vec::new()), + ..Default::default() + }; + let flags = analyze_child_flags(&opts); + assert!(!flags.iter().any(|f| f == "--include")); + } + + #[test] + fn test_analyze_child_flags_roundtrips_through_clap() { + // The reconstructed flags must parse back on the `analyze` subcommand, + // guarding against a flag name drifting away from args.rs. + use crate::args::{Cli, Command, DegreeSubcommand, SamplingStrategyArg}; + use clap::Parser; + + let opts = AnalyzeOptions { + no_report: true, + max_plans: Some(1234), + sampling_strategy: Some("stratified".to_string()), + ..Default::default() + }; + let mut argv = vec![ + "nuanalytics".to_string(), + "degree".to_string(), + "analyze".to_string(), + "some.json".to_string(), + ]; + argv.extend(analyze_child_flags(&opts)); + let cli = Cli::try_parse_from(argv).expect("reconstructed flags should parse"); + match cli.command { + Command::Degree { subcommand } => match subcommand { + DegreeSubcommand::Analyze { + no_report, + max_plans, + sampling_strategy, + .. + } => { + assert!(no_report); + assert_eq!(max_plans, Some(1234)); + assert_eq!(sampling_strategy, Some(SamplingStrategyArg::Stratified)); + } + other => panic!("expected Analyze, got {other:?}"), + }, + other => panic!("expected Degree command, got {other:?}"), + } + } + + #[test] + fn test_metrics_dir_or_default_falls_back_to_metrics() { + assert_eq!( + metrics_dir_or_default(&AnalyzeOptions::default()), + PathBuf::from("metrics") + ); + let opts = AnalyzeOptions { + metrics_dir: Some(PathBuf::from("custom")), + ..Default::default() + }; + assert_eq!(metrics_dir_or_default(&opts), PathBuf::from("custom")); + } + + #[cfg(unix)] + #[test] + fn test_reap_finished_records_failed_exit_and_keeps_running() { + use std::process::Command; + // One child exits non-zero immediately; the other stays alive. + let dead = Command::new("/bin/sh") + .arg("-c") + .arg("exit 7") + .spawn() + .expect("spawn dead"); + let alive = Command::new("/bin/sh") + .arg("-c") + .arg("sleep 30") + .spawn() + .expect("spawn alive"); + let mut running = vec![ + (PathBuf::from("dead.json"), dead), + (PathBuf::from("alive.json"), alive), + ]; + let mut failed: Vec<(PathBuf, String)> = Vec::new(); + + // Poll until the short-lived child is reaped (it exits near-instantly). + let mut total = 0; + for _ in 0..200 { + total += reap_finished(&mut running, &mut failed); + if total >= 1 { + break; + } + std::thread::sleep(std::time::Duration::from_millis(10)); + } + + assert_eq!(total, 1, "the exited child should be reaped exactly once"); + assert_eq!(failed.len(), 1, "non-zero exit must be recorded"); + assert_eq!(failed[0].0, PathBuf::from("dead.json")); + assert!( + !failed[0].1.is_empty(), + "a status string should be recorded" + ); + assert_eq!(running.len(), 1, "the still-running child must remain"); + assert_eq!(running[0].0, PathBuf::from("alive.json")); + + let _ = running[0].1.kill(); + let _ = running[0].1.wait(); + } } diff --git a/src/cli/main.rs b/src/cli/main.rs index 14644ce..a6f51cc 100644 --- a/src/cli/main.rs +++ b/src/cli/main.rs @@ -104,6 +104,8 @@ fn run_degree(subcommand: DegreeSubcommand, config: &Config, verbose: bool) { no_csv, no_report, include, + jobs, + school, } => { let options = commands::degree::AnalyzeOptions { calc_strategy: calc_strategy.map(|s| s.to_string()), @@ -117,6 +119,8 @@ fn run_degree(subcommand: DegreeSubcommand, config: &Config, verbose: bool) { no_report, verbose, include_courses: include, + jobs, + school, }; commands::degree::run_analyze(&files, &options, config); } @@ -134,6 +138,12 @@ fn run_degree(subcommand: DegreeSubcommand, config: &Config, verbose: bool) { verbose, ); } + DegreeSubcommand::Convert { files, out, pretty } => { + commands::degree::run_convert(&files, out.as_deref(), pretty, verbose); + } + DegreeSubcommand::Schema { out } => { + commands::degree::run_schema(out.as_deref()); + } } } diff --git a/src/core/degree/json_parser.rs b/src/core/degree/json_parser.rs new file mode 100644 index 0000000..2bc681f --- /dev/null +++ b/src/core/degree/json_parser.rs @@ -0,0 +1,387 @@ +//! JSON degree parser (unified format) with ai-landscape auto-conversion. +//! +//! The unified JSON is the serialized [`DegreeProgram`] (the same model YAML +//! produces) with prerequisites carried as the symmetric tagged structure +//! (`{"and"|"or": [...]}`, bare string = leaf). On load we also accept a raw +//! ai-landscape program file and convert it on the fly (see +//! [`crate::core::degree::landscape_convert`]). + +use std::path::Path; + +use serde::Serialize; +use serde_json::Value; + +use super::landscape_convert::convert_landscape_str; +use super::yaml_parser::{parse_degree_yaml, resolve_prerequisites, DegreeParseError}; +use crate::core::models::DegreeProgram; +use crate::core::prerequisite_parser::parse_to_ast; + +/// Parse a unified degree JSON string (auto-converting ai-landscape files). +/// +/// # Errors +/// Returns an error if the JSON is invalid or matches neither shape. +pub fn parse_degree_json(json: &str) -> Result { + let (program, warnings) = parse_degree_json_with_warnings(json)?; + for w in &warnings { + eprintln!("warning: {w}"); + } + Ok(program) +} + +/// Parse a unified degree JSON string, returning conversion warnings (if any). +/// +/// # Errors +/// Returns an error if the JSON is invalid or matches neither shape. +pub fn parse_degree_json_with_warnings( + json: &str, +) -> Result<(DegreeProgram, Vec), DegreeParseError> { + let value: Value = serde_json::from_str(json) + .map_err(|e| DegreeParseError::json_message(format!("Failed to parse JSON: {e}")))?; + + let (mut program, warnings) = if looks_like_landscape(&value) { + let result = convert_landscape_str(json).map_err(DegreeParseError::json_message)?; + (result.program, result.warnings) + } else { + let program: DegreeProgram = serde_json::from_value(value).map_err(|e| { + DegreeParseError::json_message(format!( + "unrecognized JSON ({e}). Expected a unified degree (top-level `degree`) or an \ + ai-landscape program (a `courses` category map). Cluster pipeline files nest \ + programs under `course_scraper..results` / `course_verifier...` — run \ + `degree convert` on those (it expands each program)." + )) + })?; + (program, Vec::new()) + }; + + resolve_prerequisites(&mut program); + Ok((program, warnings)) +} + +/// Parse a degree from a string of unknown format (YAML or JSON). +/// +/// Dispatches on the first non-whitespace byte: `{` or `[` selects the JSON +/// loader (which also auto-converts a raw ai-landscape program and returns +/// conversion warnings); anything else is treated as YAML. This is the +/// content-level analogue of the extension-based file loader the CLI uses, so +/// callers that only have a body (e.g. MCP `*_content` inputs) accept unified +/// JSON and ai-landscape JSON, not just YAML. +/// +/// # Errors +/// Returns an error if the content parses as neither unified/ai-landscape JSON +/// nor a degree YAML document. +pub fn parse_degree_auto(content: &str) -> Result<(DegreeProgram, Vec), DegreeParseError> { + let looks_json = content + .trim_start() + .as_bytes() + .first() + .is_some_and(|b| matches!(b, b'{' | b'[')); + if looks_json { + parse_degree_json_with_warnings(content) + } else { + parse_degree_yaml(content).map(|program| (program, Vec::new())) + } +} + +/// Heuristic: an ai-landscape file has a `courses` object whose values are +/// arrays (category -> list), whereas the unified format maps course keys to +/// course objects. +fn looks_like_landscape(value: &Value) -> bool { + value + .get("courses") + .and_then(Value::as_object) + .is_some_and(|courses| courses.values().next().is_some_and(Value::is_array)) +} + +/// Load a unified degree JSON file (auto-converting ai-landscape files). +/// +/// # Errors +/// Returns an error if the file cannot be read or parsed. +pub fn load_degree_from_json>(path: P) -> Result { + let path = path.as_ref(); + let contents = std::fs::read_to_string(path).map_err(|e| { + DegreeParseError::IoError(format!("Failed to read {}: {e}", path.display())) + })?; + let (program, warnings) = parse_degree_json_with_warnings(&contents)?; + for w in &warnings { + eprintln!("warning ({}): {w}", path.display()); + } + Ok(program) +} + +/// Serialize a degree program to unified JSON, emitting prerequisites in the +/// structured tagged form. Set `pretty` for human-readable output. +/// +/// # Errors +/// Returns an error if serialization fails. +pub fn serialize_degree_json( + program: &DegreeProgram, + pretty: bool, +) -> Result { + let value = to_unified_value(program)?; + unified_value_to_string(&value, pretty) + .map_err(|e| DegreeParseError::json_message(format!("Failed to serialize JSON: {e}"))) +} + +/// Serialize a unified-degree `Value` to a JSON string with `degree` first +/// (then `requirements`, `courses`, and any `conversion_warnings`). +/// +/// The file then opens to the program's identity. Nested objects keep +/// `serde_json`'s deterministic sorted key order; this only fixes the top-level +/// layout. +/// +/// # Errors +/// Returns the underlying `serde_json` error if serialization fails. +pub fn unified_value_to_string(value: &Value, pretty: bool) -> Result { + #[derive(Serialize)] + struct Ordered<'a> { + degree: &'a Value, + requirements: &'a Value, + courses: &'a Value, + #[serde(skip_serializing_if = "Option::is_none")] + conversion_warnings: Option<&'a Value>, + } + let null = Value::Null; + let ordered = Ordered { + degree: value.get("degree").unwrap_or(&null), + requirements: value.get("requirements").unwrap_or(&null), + courses: value.get("courses").unwrap_or(&null), + conversion_warnings: value.get("conversion_warnings"), + }; + if pretty { + serde_json::to_string_pretty(&ordered) + } else { + serde_json::to_string(&ordered) + } +} + +/// Build the unified-JSON `Value` for a program. +/// +/// Serializes the model, then rewrites each course's `prerequisites_raw` +/// boolean string into the structured `{"and"|"or": ...}` tagged form under the +/// `prerequisites` key. +/// +/// # Errors +/// Returns an error if model serialization fails. +pub fn to_unified_value(program: &DegreeProgram) -> Result { + let mut value = serde_json::to_value(program) + .map_err(|e| DegreeParseError::json_message(format!("Failed to serialize program: {e}")))?; + structurize_prereqs(&mut value); + Ok(value) +} + +/// Replace each course's `prerequisites_raw` string with the structured tagged +/// `prerequisites` object. Courses without a parseable expression are left +/// without a prerequisites field. +fn structurize_prereqs(value: &mut Value) { + let Some(courses) = value.get_mut("courses").and_then(Value::as_object_mut) else { + return; + }; + for course in courses.values_mut() { + let Some(obj) = course.as_object_mut() else { + continue; + }; + if let Some(raw) = obj.remove("prerequisites_raw") { + if let Some(expr) = raw.as_str().and_then(parse_to_ast) { + if let Ok(structured) = serde_json::to_value(&expr) { + obj.insert("prerequisites".to_string(), structured); + } + } + } + } +} + +/// Save a degree program to a unified JSON file (pretty-printed). +/// +/// # Errors +/// Returns an error if the file cannot be written or serialization fails. +pub fn save_degree_to_json>( + program: &DegreeProgram, + path: P, +) -> Result<(), DegreeParseError> { + let path = path.as_ref(); + let json = serialize_degree_json(program, true)?; + std::fs::write(path, json) + .map_err(|e| DegreeParseError::IoError(format!("Failed to write {}: {e}", path.display()))) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_unified_value_to_string_puts_degree_first() { + // Keys deliberately out of order; degree must lead, warnings trail. + let value = serde_json::json!({ + "courses": {"CS1": {"name": "Intro"}}, + "requirements": {"core": {"type": "all"}}, + "degree": {"name": "Test", "degree_type": "BS"}, + "conversion_warnings": ["w1"], + }); + let out = unified_value_to_string(&value, false).unwrap(); + let dp = out.find("\"degree\"").unwrap(); + let rp = out.find("\"requirements\"").unwrap(); + let cp = out.find("\"courses\"").unwrap(); + let wp = out.find("\"conversion_warnings\"").unwrap(); + assert!( + dp < rp && rp < cp && cp < wp, + "order should be degree, requirements, courses, warnings: {out}" + ); + } + + #[test] + fn test_unified_value_to_string_omits_absent_warnings() { + let value = serde_json::json!({ + "degree": {"name": "Test"}, + "requirements": {}, + "courses": {}, + }); + let out = unified_value_to_string(&value, false).unwrap(); + assert!( + !out.contains("conversion_warnings"), + "absent warnings must be omitted: {out}" + ); + } + + #[test] + fn test_parse_degree_auto_dispatches_by_shape() { + // YAML body → YAML loader, no conversion warnings. + let yaml = "degree:\n name: Y\n degree_type: BS\n system_type: semester\n\ + requirements:\n core:\n type: all\n courses: [CS101]\n\ + courses:\n CS101:\n prefix: CS\n number: '101'\n title: Intro\n credits: 3\n"; + let (p, w) = parse_degree_auto(yaml).unwrap(); + assert_eq!(p.degree.name, "Y"); + assert!(w.is_empty(), "YAML path emits no conversion warnings"); + + // Unified JSON body → JSON loader. + let json = r#"{"degree":{"name":"J","degree_type":"BS","system_type":"semester"}, + "requirements":{"core":{"type":"all","courses":["CS101"]}}, + "courses":{"CS101":{"name":"Intro","prefix":"CS","number":"101","credit_hours":3.0}}}"#; + let (p2, _) = parse_degree_auto(json).unwrap(); + assert_eq!(p2.degree.name, "J"); + + // Raw ai-landscape JSON (leading '{') → auto-convert, surfacing warnings + // (the lone course has no course_hours, so credits are defaulted). + let land = r#"{"university":"U","degree":"BS in CS","ai_program":null, + "courses":{"cs_course_core":[{"course_code":"CS 101","title":"Intro", + "picklist":[],"prerequisites":[],"corequisites":[],"strict_corequisites":[]}]}}"#; + let (p3, w3) = parse_degree_auto(land).unwrap(); + assert!(!p3.courses.is_empty()); + assert!( + w3.iter().any(|m| m.to_lowercase().contains("credit")), + "missing course_hours should warn: {w3:?}" + ); + } + + #[test] + fn test_parse_degree_auto_sniffs_past_leading_whitespace() { + // Leading newlines/spaces must not defeat the `{`/`[` JSON sniff. + let json = "\n\n {\"degree\":{\"name\":\"W\",\"degree_type\":\"BS\",\"system_type\":\"semester\"},\ + \"requirements\":{\"core\":{\"type\":\"all\",\"courses\":[\"CS101\"]}},\ + \"courses\":{\"CS101\":{\"name\":\"Intro\",\"prefix\":\"CS\",\"number\":\"101\",\"credit_hours\":3.0}}}"; + let (p, _) = parse_degree_auto(json).unwrap(); + assert_eq!(p.degree.name, "W"); + } + + #[test] + fn test_roundtrip_unified_json_structured_prereqs() { + let landscape = r#"{ + "university": "Test U", + "degree": "Bachelor's of Science Computer Science", + "ai_program": null, + "courses": { + "cs_course_core": [ + {"course_code":"CS 101","title":"Intro","course_hours":"4","prerequisites":[]}, + {"course_code":"CS 201","title":"DS","course_hours":"4","prerequisites":[["CS 101"]]} + ] + } + }"#; + + // Auto-convert from landscape shape. + let (program, _warn) = parse_degree_json_with_warnings(landscape).unwrap(); + assert!(program.courses.contains_key("CS201")); + + // Serialize to unified JSON: prereqs become the structured tagged form. + let unified = serialize_degree_json(&program, false).unwrap(); + assert!(unified.contains("\"prerequisites\"")); + assert!(!unified.contains("prerequisites_raw")); + + // Re-parse the unified JSON (now the non-landscape branch) and confirm + // the prerequisite survives the round-trip. + let reparsed = parse_degree_json(&unified).unwrap(); + assert_eq!( + reparsed.courses["CS201"].prerequisites_raw.as_deref(), + Some("CS101") + ); + } + + #[test] + fn test_parse_degree_json_invalid_json_errors() { + assert!(parse_degree_json_with_warnings("{ not json ").is_err()); + } + + #[test] + fn test_parse_degree_json_cluster_shape_errors_with_guidance() { + // A cluster pipeline file is neither landscape nor a unified degree, so + // the loader errors — and the message steers the user to `degree convert`. + let cluster = r#"{"course_verifier":{"CS BS":{"results":{ + "university":"U","degree":"BS","courses":{ + "cs_course_core":[{"course_code":"CS 101","title":"Intro","course_hours":"3","prerequisites":[]}]}}}}}"#; + let msg = parse_degree_json_with_warnings(cluster) + .unwrap_err() + .to_string(); + assert!( + msg.contains("course_scraper") || msg.contains("degree convert"), + "cluster files should be steered to `degree convert`; got: {msg}" + ); + } + + #[test] + fn test_parse_degree_json_neither_shape_errors() { + // Valid JSON, `courses` maps a key to an object (so not landscape), but + // the value isn't a valid Course -> unified branch must surface an error. + let bad = r#"{"courses":{"CS1":{"credit_hours":"not-a-number"}}}"#; + assert!(parse_degree_json_with_warnings(bad).is_err()); + } + + #[test] + fn test_load_and_save_degree_json_file_roundtrip() { + let dir = tempfile::tempdir().unwrap(); + let landscape = dir.path().join("prog.json"); + std::fs::write( + &landscape, + r#"{"university":"U","degree":"BS CS","ai_program":null, + "courses":{"cs_course_core":[ + {"course_code":"CS 201","title":"DS","course_hours":"4", + "prerequisites":[["CS 101"]]}]}}"#, + ) + .unwrap(); + + // Load auto-converts the landscape file. + let program = load_degree_from_json(&landscape).unwrap(); + assert!(program.courses.contains_key("CS201")); + + // Save to unified JSON and reload through the file path. + let unified = dir.path().join("prog.unified.json"); + save_degree_to_json(&program, &unified).unwrap(); + let reloaded = load_degree_from_json(&unified).unwrap(); + assert_eq!( + reloaded.courses["CS201"].prerequisites_raw.as_deref(), + Some("CS101") + ); + } + + #[test] + fn test_detects_landscape_vs_unified() { + let landscape: Value = + serde_json::from_str(r#"{"courses":{"cs_course_core":[]}}"#).unwrap(); + // empty arrays -> first value is array + assert!(looks_like_landscape( + &serde_json::from_str(r#"{"courses":{"core":[{"course_code":"CS1"}]}}"#).unwrap() + )); + // unified: course keys map to objects + assert!(!looks_like_landscape( + &serde_json::from_str(r#"{"courses":{"CS1":{"name":"x"}}}"#).unwrap() + )); + let _ = landscape; + } +} diff --git a/src/core/degree/landscape_convert.rs b/src/core/degree/landscape_convert.rs new file mode 100644 index 0000000..7938a29 --- /dev/null +++ b/src/core/degree/landscape_convert.rs @@ -0,0 +1,1017 @@ +//! Converter: ai-landscape program JSON -> unified [`DegreeProgram`]. +//! +//! ai-landscape-tools (the sibling web-crawling/validation project) emits one +//! JSON file per program with a fixed shape: +//! +//! ```jsonc +//! { +//! "university": "...", "school": null, "department": "...", +//! "degree": "Bachelor's of Science Computer Science", +//! "ai_program": "Minor" | "Major" | "Concentration" | "Certificate" | null, +//! "curriculum_link": "...", "avg_classes_per_term": "5", "verified": true, +//! "courses": { +//! "cs_course_core": [ , ... ], +//! "ai_program_required_courses": [ ... ], +//! "ai_program_unrestricted_electives": [ ... ], +//! "unrestricted_elective": [ ... ] +//! } +//! } +//! ``` +//! +//! Each course carries `prerequisites` as `Vec>` where the **outer +//! list is AND** and the **inner list is OR** (the opposite of our DNF), and an +//! inner `"X and Y"` string is itself an AND. We flip that into our +//! [`PrereqExpr`] (`All`/`Any`/`Course`) tree at the boundary. +//! +//! This converter is transitional: once ai-landscape emits the unified JSON +//! directly, it can be retired. + +use std::collections::HashMap; + +use serde::{Deserialize, Deserializer}; + +use crate::core::models::degree::{FromClause, Requirement, RequirementType}; +use crate::core::models::{Course, Degree, DegreeProgram}; +use crate::core::prerequisite_parser::PrereqExpr; + +/// Deserialize a string field, mapping an explicit JSON `null` (which some +/// source files use for a missing title/code) to an empty string. `#[serde(default)]` +/// alone only covers absent keys, not an explicit `null`. +fn string_or_null<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + Ok(Option::::deserialize(deserializer)?.unwrap_or_default()) +} + +/// Default credit hours assumed when a source course is missing `course_hours`. +const DEFAULT_CREDIT_HOURS: f32 = 3.0; + +// ai-landscape cluster pipeline-stage keys. +const STAGE_VERIFIER: &str = "course_verifier"; +const STAGE_SCRAPER: &str = "course_scraper"; +const STAGE_RESULTS: &str = "results"; + +// ai-landscape category names (their fixed vocabulary). +const CAT_CS_CORE: &str = "cs_course_core"; +const CAT_AI_REQUIRED: &str = "ai_program_required_courses"; +const CAT_AI_ELECTIVE: &str = "ai_program_unrestricted_electives"; +const CAT_UNRESTRICTED: &str = "unrestricted_elective"; + +// Unified classification tag vocabulary (program-agnostic). +const TAG_AI: &str = "ai"; +const TAG_CORE: &str = "core"; +const TAG_ELECTIVE: &str = "elective"; +const TAG_REQUIRED: &str = "required"; +const TAG_OTHER: &str = "other"; + +// Unified `Degree`/`Requirement` category values. +const DEGREE_CAT_MAJOR: &str = "major"; +const DEGREE_CAT_ELECTIVE: &str = "elective"; + +// --------------------------------------------------------------------------- +// Transitional input structs (ai-landscape shape) +// --------------------------------------------------------------------------- + +/// Top-level ai-landscape program record. +#[derive(Debug, Clone, serde::Deserialize)] +pub struct LandscapeProgram { + /// Institution name. + #[serde(default)] + pub university: String, + /// School/college within the institution, if recorded. + #[serde(default)] + pub school: Option, + /// Department, if recorded. + #[serde(default)] + pub department: Option, + /// Free-text degree title (e.g. "Bachelor's of Science Computer Science"). + #[serde(default)] + pub degree: String, + /// AI program flavor (`Major`/`Minor`/`Concentration`/…) or null. + #[serde(default)] + pub ai_program: Option, + /// Catalog/curriculum source URL. + #[serde(default)] + pub curriculum_link: Option, + /// Category name -> list of courses. Category names are not fixed in code + /// so unknown categories degrade gracefully. + #[serde(default)] + pub courses: HashMap>, +} + +/// A single course inside an ai-landscape category list. +#[derive(Debug, Clone, serde::Deserialize)] +pub struct LandscapeCourse { + /// Source course code (e.g. "CSCE 3201"). + #[serde(default, deserialize_with = "string_or_null")] + pub course_code: String, + /// Course title. + #[serde(default, deserialize_with = "string_or_null")] + pub title: String, + /// Credit hours (numeric string, occasionally missing). + #[serde(default)] + pub course_hours: Option, + /// Catalog URL for the course. + #[serde(default)] + pub course_url: Option, + /// Picklist tags (`"Name [N]"` = choose N from group `Name`). + #[serde(default)] + pub picklist: Vec, + /// Prerequisites: outer list = AND, inner list = OR (their convention). + #[serde(default)] + pub prerequisites: Vec>, + /// Corequisites in the same AND-of-OR shape. + #[serde(default)] + pub corequisites: Vec>, + /// Strict corequisites (must be taken together) in the same shape. + #[serde(default)] + pub strict_corequisites: Vec>, +} + +/// `course_hours` arrives as a numeric string ("3") but tolerate raw numbers. +#[derive(Debug, Clone, serde::Deserialize)] +#[serde(untagged)] +pub enum Hours { + /// Numeric string form, e.g. `"3"`. + Str(String), + /// Raw JSON number form. + Num(f64), +} + +impl Hours { + fn as_credits(&self) -> Option { + match self { + #[allow(clippy::cast_possible_truncation)] + Self::Num(n) => Some(*n as f32), + Self::Str(s) => { + let t = s.trim(); + if t.is_empty() { + None + } else { + t.parse::().ok() + } + } + } + } +} + +/// Result of a conversion: the unified program plus any data-quality warnings. +#[derive(Debug, Clone)] +pub struct ConversionResult { + /// The converted unified program. + pub program: DegreeProgram, + /// Data-quality warnings raised during conversion. + pub warnings: Vec, +} + +// --------------------------------------------------------------------------- +// Public entry points +// --------------------------------------------------------------------------- + +/// Convert an ai-landscape JSON string into a unified [`DegreeProgram`]. +/// +/// # Errors +/// Returns an error string if the JSON does not match the ai-landscape shape. +pub fn convert_landscape_str(json: &str) -> Result { + let program: LandscapeProgram = serde_json::from_str(json) + .map_err(|e| format!("Failed to parse ai-landscape JSON: {e}"))?; + Ok(convert_landscape(&program)) +} + +/// Extract convertible programs from an ai-landscape *cluster* pipeline file. +/// +/// Each program's data lives at `course_verifier..results` or +/// `course_scraper..results`; the verifier stage is preferred, falling +/// back to the scraper. Programs whose results are absent, unparseable, or have +/// no courses are skipped. Returns `None` if the value is not a cluster file +/// (no `course_*` stage). +#[must_use] +pub fn extract_cluster_programs( + value: &serde_json::Value, +) -> Option> { + let verifier = value + .get(STAGE_VERIFIER) + .and_then(serde_json::Value::as_object); + let scraper = value + .get(STAGE_SCRAPER) + .and_then(serde_json::Value::as_object); + if verifier.is_none() && scraper.is_none() { + return None; + } + + // Deterministic, de-duplicated program order across both stages. + let mut names: Vec<&String> = verifier + .into_iter() + .chain(scraper) + .flat_map(serde_json::Map::keys) + .collect(); + names.sort(); + names.dedup(); + + let mut programs = Vec::new(); + for name in names { + if let Some(prog) = stage_program(verifier, name).or_else(|| stage_program(scraper, name)) { + programs.push((name.clone(), prog)); + } + } + Some(programs) +} + +/// Deserialize `..results` into a [`LandscapeProgram`], returning +/// `None` unless it parses and has at least one non-empty course category. +fn stage_program( + stage: Option<&serde_json::Map>, + name: &str, +) -> Option { + let results = stage?.get(name)?.get(STAGE_RESULTS)?; + let program: LandscapeProgram = serde_json::from_value(results.clone()).ok()?; + program + .courses + .values() + .any(|list| !list.is_empty()) + .then_some(program) +} + +/// Canonical category ordering so requirement IDs / output are deterministic. +const KNOWN_CATEGORY_ORDER: [&str; 4] = [ + CAT_CS_CORE, + CAT_AI_REQUIRED, + CAT_AI_ELECTIVE, + CAT_UNRESTRICTED, +]; + +/// Convert a parsed [`LandscapeProgram`] into a unified [`DegreeProgram`]. +#[must_use] +pub fn convert_landscape(src: &LandscapeProgram) -> ConversionResult { + let mut warnings = Vec::new(); + let mut courses: HashMap = HashMap::new(); + + // picklist name -> (count, member course keys, is_ai) + let mut picklists: HashMap = HashMap::new(); + // category -> non-picklisted course keys (for the "all" requirement) + let mut category_required: Vec<(String, Vec)> = Vec::new(); + + for category in ordered_categories(src) { + let meta = category_meta(&category); + let Some(list) = src.courses.get(&category) else { + continue; + }; + let mut plain_keys: Vec = Vec::new(); + for lc in list { + classify_course( + lc, + &meta, + &mut courses, + &mut picklists, + &mut plain_keys, + &mut warnings, + ); + } + if !plain_keys.is_empty() { + category_required.push((category.clone(), plain_keys)); + } + } + + let requirements = assemble_requirements(category_required, picklists); + let degree = build_degree(src); + + ConversionResult { + program: DegreeProgram { + degree, + requirements, + courses, + }, + warnings, + } +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +struct PicklistAcc { + count: u32, + keys: Vec, + is_ai: bool, +} + +/// Process one source course within a category: create/merge its unified course +/// entry, then record it as a plain required course or as picklist member(s). +fn classify_course( + lc: &LandscapeCourse, + meta: &CategoryMeta, + courses: &mut HashMap, + picklists: &mut HashMap, + plain_keys: &mut Vec, + warnings: &mut Vec, +) { + let key = normalize_key(&lc.course_code); + if key.is_empty() { + return; + } + build_or_merge_course(courses, &key, lc, meta.course_ai, warnings); + + if lc.picklist.is_empty() { + if !plain_keys.contains(&key) { + plain_keys.push(key); + } + return; + } + for entry in &lc.picklist { + record_picklist(entry, &key, meta.course_ai, picklists, warnings); + } +} + +/// Record one `"Name [N]"` picklist membership for `key`, accumulating the +/// choose-N count and AI flag for that group. +fn record_picklist( + entry: &str, + key: &str, + is_ai: bool, + picklists: &mut HashMap, + warnings: &mut Vec, +) { + let Some((name, count)) = parse_picklist(entry) else { + warnings.push(format!("Unparseable picklist tag `{entry}` on {key}")); + return; + }; + let acc = picklists.entry(name).or_insert_with(|| PicklistAcc { + count, + keys: Vec::new(), + is_ai: false, + }); + acc.count = acc.count.max(count); + acc.is_ai |= is_ai; + if !acc.keys.iter().any(|k| k == key) { + acc.keys.push(key.to_string()); + } +} + +/// Assemble requirements: one `all` requirement per category (for its +/// non-picklisted courses) and one `select` requirement per picklist group. +fn assemble_requirements( + category_required: Vec<(String, Vec)>, + picklists: HashMap, +) -> HashMap { + let mut requirements: HashMap = HashMap::new(); + + for (category, keys) in category_required { + let meta = category_meta(&category); + requirements.insert( + category.clone(), + Requirement { + name: Some(meta.display.to_string()), + req_type: RequirementType::All, + category: Some(meta.degree_category.to_string()), + courses: Some(keys), + from: None, + count: None, + credits: None, + credit_range: None, + constraints: None, + options: None, + tags: Some(meta.req_tags.iter().map(|s| (*s).to_string()).collect()), + }, + ); + } + + for (name, acc) in picklists { + let mut tags = vec![TAG_ELECTIVE.to_string()]; + if acc.is_ai { + tags.insert(0, TAG_AI.to_string()); + } + requirements.insert( + format!("picklist_{}", slug(&name)), + Requirement { + name: Some(name), + req_type: RequirementType::Select, + category: Some(DEGREE_CAT_ELECTIVE.to_string()), + courses: None, + from: Some(FromClause { + courses: Some(acc.keys), + pattern: None, + include: None, + exclude: None, + groups: None, + groups_required: None, + per_group: None, + }), + count: Some(acc.count), + credits: None, + credit_range: None, + constraints: None, + options: None, + tags: Some(tags), + }, + ); + } + + requirements +} + +struct CategoryMeta { + req_tags: &'static [&'static str], + course_ai: bool, + degree_category: &'static str, + display: &'static str, +} + +fn category_meta(category: &str) -> CategoryMeta { + match category { + CAT_CS_CORE => CategoryMeta { + req_tags: &[TAG_CORE], + course_ai: false, + degree_category: DEGREE_CAT_MAJOR, + display: "Core Courses", + }, + CAT_AI_REQUIRED => CategoryMeta { + req_tags: &[TAG_AI, TAG_REQUIRED], + course_ai: true, + degree_category: DEGREE_CAT_MAJOR, + display: "AI Required Courses", + }, + CAT_AI_ELECTIVE => CategoryMeta { + req_tags: &[TAG_AI, TAG_ELECTIVE], + course_ai: true, + degree_category: DEGREE_CAT_ELECTIVE, + display: "AI Electives", + }, + CAT_UNRESTRICTED => CategoryMeta { + req_tags: &[TAG_ELECTIVE], + course_ai: false, + degree_category: DEGREE_CAT_ELECTIVE, + display: "Unrestricted Electives", + }, + _ => CategoryMeta { + req_tags: &[TAG_OTHER], + course_ai: false, + degree_category: DEGREE_CAT_MAJOR, + display: "Other Courses", + }, + } +} + +/// Known categories first (in canonical order), then any extras the file +/// happens to contain, sorted for determinism. +fn ordered_categories(src: &LandscapeProgram) -> Vec { + let mut out: Vec = KNOWN_CATEGORY_ORDER + .iter() + .filter(|c| src.courses.contains_key(**c)) + .map(|c| (*c).to_string()) + .collect(); + let mut extras: Vec = src + .courses + .keys() + .filter(|k| !KNOWN_CATEGORY_ORDER.contains(&k.as_str())) + .cloned() + .collect(); + extras.sort(); + out.extend(extras); + out +} + +fn build_or_merge_course( + courses: &mut HashMap, + key: &str, + lc: &LandscapeCourse, + is_ai: bool, + warnings: &mut Vec, +) { + if let Some(existing) = courses.get_mut(key) { + // Cross-category duplicate: just union the ai tag. + if is_ai { + existing.add_tag(TAG_AI); + } + return; + } + + let (prefix, number) = split_code(&lc.course_code); + let credits = lc.course_hours.as_ref().and_then(Hours::as_credits); + let credit_hours = credits.unwrap_or_else(|| { + warnings.push(format!( + "Course {key} missing course_hours; assuming {DEFAULT_CREDIT_HOURS} credits" + )); + DEFAULT_CREDIT_HOURS + }); + + let prereq_expr = convert_groups_and_of_or(&lc.prerequisites); + let prerequisites_raw = prereq_expr.map(|e| e.to_expression_string()); + + let mut course = Course::new(lc.title.clone(), prefix, number, credit_hours); + course.prerequisites_raw = prerequisites_raw; + course.corequisites = flatten_coreqs(&lc.corequisites, key, "corequisite", warnings); + course.strict_corequisites = + flatten_coreqs(&lc.strict_corequisites, key, "strict corequisite", warnings); + if is_ai { + course.add_tag(TAG_AI); + } + courses.insert(key.to_string(), course); +} + +/// Their corequisites are `Vec>` (AND-of-OR) but our model stores a +/// flat required list. Take the first alternative of each OR group; warn when an +/// alternative is dropped. +fn flatten_coreqs( + groups: &[Vec], + key: &str, + label: &str, + warnings: &mut Vec, +) -> Vec { + let mut out = Vec::new(); + for group in groups { + if group.len() > 1 { + warnings.push(format!( + "{key}: collapsed a {label} choice ({} alternatives) to the first option", + group.len() + )); + } + if let Some(first) = group.first() { + for code in split_on_and(first) { + let k = normalize_key(&code); + if !k.is_empty() && !out.contains(&k) { + out.push(k); + } + } + } + } + out +} + +/// Convert ai-landscape `Vec>` (outer AND, inner OR, inner `"X and Y"` +/// = nested AND) into our [`PrereqExpr`]. Returns `None` for empty input. +#[must_use] +pub fn convert_groups_and_of_or(groups: &[Vec]) -> Option { + let parts: Vec = groups.iter().filter_map(|g| or_group_to_expr(g)).collect(); + collapse(parts, true) +} + +fn or_group_to_expr(group: &[String]) -> Option { + let alts: Vec = group + .iter() + .filter_map(|item| { + let leaves: Vec = split_on_and(item) + .into_iter() + .filter_map(|code| { + let k = normalize_key(&code); + if k.is_empty() { + None + } else { + Some(PrereqExpr::Course(k)) + } + }) + .collect(); + collapse(leaves, true) + }) + .collect(); + collapse(alts, false) +} + +/// Collapse a child list into `All` (`and = true`) or `Any` (`and = false`), +/// flattening the single-child and empty cases. +fn collapse(mut parts: Vec, and: bool) -> Option { + match parts.len() { + 0 => None, + 1 => parts.pop(), + _ => Some(if and { + PrereqExpr::All(parts) + } else { + PrereqExpr::Any(parts) + }), + } +} + +/// Split a course-code string on the word "and" (case-insensitive, whitespace +/// delimited), e.g. `"MAC 2312 and MAD 2104"` -> `["MAC 2312", "MAD 2104"]`. +fn split_on_and(s: &str) -> Vec { + let mut out = Vec::new(); + let mut rest = s.trim(); + loop { + let lower = rest.to_lowercase(); + if let Some(pos) = lower.find(" and ") { + out.push(rest[..pos].trim().to_string()); + rest = rest[pos + 5..].trim_start(); + } else { + let t = rest.trim(); + if !t.is_empty() { + out.push(t.to_string()); + } + break; + } + } + out +} + +/// Normalize a course code to a graph key: split into prefix+number, drop +/// parser-hostile characters (`()`, grade `[..]`, whitespace) so the key is safe +/// inside a boolean prerequisite string. +fn normalize_key(code: &str) -> String { + let (prefix, number) = split_code(code); + format!("{prefix}{number}") +} + +/// Split a raw course code into `(prefix, number)`, stripping `()`, `[..]` and +/// collapsing whitespace. e.g. `"CSCE 3201"` -> `("CSCE","3201")`, +/// `"M 362K"` -> `("M","362K")`, `"COT3100"` -> `("COT","3100")`. +#[allow(clippy::option_if_let_else)] +fn split_code(code: &str) -> (String, String) { + // Strip grade brackets and parentheses (e.g. "CS 429(H)" -> "CS 429H"). + let mut cleaned = String::with_capacity(code.len()); + let mut in_bracket = false; + for c in code.chars() { + match c { + '[' => in_bracket = true, + ']' => in_bracket = false, + '(' | ')' => {} + _ if !in_bracket => cleaned.push(c), + _ => {} + } + } + let cleaned = cleaned.trim(); + + if let Some(idx) = cleaned.find(char::is_whitespace) { + let prefix = cleaned[..idx].trim().to_string(); + let number: String = cleaned[idx..].split_whitespace().collect(); + (prefix, number) + } else if let Some(idx) = cleaned.find(|c: char| c.is_ascii_digit()) { + (cleaned[..idx].to_string(), cleaned[idx..].to_string()) + } else { + (cleaned.to_string(), String::new()) + } +} + +/// Parse a picklist tag `"Name [N]"` (spacing is inconsistent, e.g. +/// `"Electives[4]"`) into `(name, count)`. +fn parse_picklist(entry: &str) -> Option<(String, u32)> { + let open = entry.find('[')?; + let close = entry[open..].find(']')? + open; + let name = entry[..open].trim(); + let count: u32 = entry[open + 1..close].trim().parse().ok()?; + if name.is_empty() { + None + } else { + Some((name.to_string(), count)) + } +} + +fn build_degree(src: &LandscapeProgram) -> Degree { + let mut degree = Degree::new( + if src.degree.is_empty() { + "Computer Science".to_string() + } else { + src.degree.clone() + }, + infer_degree_type(&src.degree), + None, + "semester".to_string(), + ); + if !src.university.is_empty() { + degree.institution = Some(src.university.clone()); + } + degree.source_url.clone_from(&src.curriculum_link); + degree.id = Some(slug(&format!( + "{}-{}-{}", + src.university, + src.degree, + src.ai_program.as_deref().unwrap_or("program") + ))); + + // Program-level tags generalize the ai_program enum. + if let Some(flavor) = ai_flavor(src.ai_program.as_deref()) { + let mut tags = vec!["ai".to_string()]; + tags.push(format!("ai-{}", slug(flavor))); + degree.tags = Some(tags); + } + degree +} + +/// Returns the `ai_program` flavor when it represents a real AI program. +fn ai_flavor(ai_program: Option<&str>) -> Option<&str> { + match ai_program.map(str::trim) { + Some(s) + if !s.is_empty() + && !s.eq_ignore_ascii_case("null") + && !s.eq_ignore_ascii_case("none") => + { + Some(s) + } + _ => None, + } +} + +fn infer_degree_type(degree: &str) -> String { + let d = degree.to_lowercase(); + if d.contains("bachelor") && (d.contains("art") || d.contains(" ba")) { + "BA".to_string() + } else if d.contains("master") { + "MS".to_string() + } else { + "BS".to_string() + } +} + +/// kebab-case slug from arbitrary text. +fn slug(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut prev_dash = false; + for c in s.trim().chars() { + if c.is_ascii_alphanumeric() { + out.push(c.to_ascii_lowercase()); + prev_dash = false; + } else if !prev_dash && !out.is_empty() { + out.push('-'); + prev_dash = true; + } + } + out.trim_matches('-').to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_extract_cluster_programs_falls_back_to_scraper_only() { + // Program lives only under course_scraper: verifier branch is None, so + // the scraper branch supplies it. + let value: serde_json::Value = serde_json::from_str( + r#"{ + "course_scraper": { + "DS BS": {"results": {"university":"U","degree":"BS DS","courses":{ + "cs_course_core":[{"course_code":"DS 200","title":"Data","course_hours":"3","prerequisites":[]}]}}} + }, + "course_verifier": {} + }"#, + ) + .unwrap(); + let progs = extract_cluster_programs(&value).unwrap(); + assert_eq!(progs.len(), 1); + assert_eq!(progs[0].0, "DS BS"); + assert!(convert_landscape(&progs[0].1) + .program + .courses + .contains_key("DS200")); + } + + #[test] + fn test_extract_cluster_programs_skips_missing_unparseable_and_empty() { + // NoResults: no `results` key. BadResults: `results` not a program object. + // EmptyCats: parses but no courses. Good: survives. + let value: serde_json::Value = serde_json::from_str( + r#"{ + "course_verifier": { + "NoResults": {"status": "ok"}, + "BadResults": {"results": 42}, + "EmptyCats": {"results": {"university":"U","degree":"X","courses":{"cs_course_core":[]}}}, + "Good": {"results": {"university":"U","degree":"BS","courses":{ + "cs_course_core":[{"course_code":"CS 101","title":"Intro","course_hours":"3","prerequisites":[]}]}}} + } + }"#, + ) + .unwrap(); + let progs = extract_cluster_programs(&value).unwrap(); + assert_eq!(progs.len(), 1, "only `Good` has non-empty courses"); + assert_eq!(progs[0].0, "Good"); + } + + #[test] + fn test_extract_cluster_programs_stage_not_object_is_none() { + // `course_verifier` present but not an object (and no scraper) -> non-cluster. + let value: serde_json::Value = + serde_json::from_str(r#"{"course_verifier": "running"}"#).unwrap(); + assert!(extract_cluster_programs(&value).is_none()); + } + + #[test] + fn test_extract_cluster_programs_prefers_verifier_and_skips_empty() { + let value: serde_json::Value = serde_json::from_str( + r#"{ + "university": "Test U", + "course_scraper": { + "CS BS": {"results": {"university":"Test U","degree":"BS CS","courses":{ + "cs_course_core":[{"course_code":"CS 101","title":"Intro","course_hours":"4","prerequisites":[]}]}}}, + "Empty Prog": {"results": {"university":"Test U","degree":"X","courses":{"cs_course_core":[]}}} + }, + "course_verifier": { + "CS BS": {"results": {"university":"Test U","degree":"BS CS","courses":{ + "cs_course_core":[ + {"course_code":"CS 101","title":"Intro V","course_hours":"4","prerequisites":[]}, + {"course_code":"CS 201","title":"DS","course_hours":"4","prerequisites":[["CS 101"]]}]}}} + } + }"#, + ) + .unwrap(); + + let progs = extract_cluster_programs(&value).unwrap(); + // "CS BS" resolves from the verifier (2 courses); "Empty Prog" is skipped. + assert_eq!(progs.len(), 1); + assert_eq!(progs[0].0, "CS BS"); + let conv = convert_landscape(&progs[0].1); + assert!(conv.program.courses.contains_key("CS201")); + } + + #[test] + fn test_extract_cluster_programs_none_for_non_cluster() { + let value: serde_json::Value = + serde_json::from_str(r#"{"courses":{"cs_course_core":[]}}"#).unwrap(); + assert!(extract_cluster_programs(&value).is_none()); + } + + #[test] + fn test_convert_tolerates_null_title_and_code() { + // Real source files sometimes carry `"title": null` / `"course_code": null`. + let json = r#"{ + "university": "U", "degree": "BS CS", + "courses": { "cs_course_core": [ + {"course_code":"CS 101","title":null,"course_hours":null,"prerequisites":[]}, + {"course_code":null,"title":"Ghost","prerequisites":[]} + ]} + }"#; + let result = convert_landscape_str(json).expect("null fields must not fail conversion"); + // Null title -> empty name; null code -> dropped (empty key). + assert!(result.program.courses.contains_key("CS101")); + assert_eq!(result.program.courses["CS101"].name, ""); + assert_eq!(result.program.courses.len(), 1); + } + + #[test] + fn test_split_on_and_variants() { + // Case-insensitive " and ", trims, drops empties; no-and returns whole. + assert_eq!( + split_on_and("MAC 2312 and MAD 2104"), + vec!["MAC 2312", "MAD 2104"] + ); + assert_eq!(split_on_and("A aNd B AND C"), vec!["A", "B", "C"]); + assert_eq!(split_on_and("CS 101"), vec!["CS 101"]); + assert_eq!(split_on_and(" CS 101 "), vec!["CS 101"]); + assert!(split_on_and("").is_empty()); + assert!(split_on_and(" ").is_empty()); + // "and" inside a token (no surrounding spaces) is not a split point. + assert_eq!(split_on_and("ANDREW 100"), vec!["ANDREW 100"]); + } + + #[test] + fn test_flatten_coreqs_first_alternative_split_and_dedup() { + let mut warnings = Vec::new(); + // Group 1: OR with 2 alternatives -> keep first, warn. + // Group 2: single "X and Y" -> split into two keys. + let groups = vec![ + vec!["CS 101".to_string(), "CS 102".to_string()], + vec!["MA 100 and MA 200".to_string()], + ]; + let out = flatten_coreqs(&groups, "CS500", "corequisite", &mut warnings); + assert_eq!(out, vec!["CS101", "MA100", "MA200"]); + assert!(warnings + .iter() + .any(|w| w.contains("CS500") && w.contains("2 alternatives"))); + + // Empty input -> empty, no warning. Duplicates collapse. + let mut w2 = Vec::new(); + assert!(flatten_coreqs(&[], "X", "corequisite", &mut w2).is_empty()); + assert!(w2.is_empty()); + let dup = vec![vec!["CS 101".to_string()], vec!["CS 101".to_string()]]; + assert_eq!( + flatten_coreqs(&dup, "X", "corequisite", &mut w2), + vec!["CS101"] + ); + } + + #[test] + fn test_ai_flavor_recognizes_real_and_rejects_sentinels() { + assert_eq!(ai_flavor(Some("Minor")), Some("Minor")); + assert_eq!(ai_flavor(Some(" Concentration ")), Some("Concentration")); + assert_eq!(ai_flavor(None), None); + assert_eq!(ai_flavor(Some("")), None); + assert_eq!(ai_flavor(Some(" ")), None); + assert_eq!(ai_flavor(Some("null")), None); + assert_eq!(ai_flavor(Some("NONE")), None); + } + + #[test] + fn test_hours_as_credits_str_num_and_garbage() { + assert_eq!(Hours::Num(3.0).as_credits(), Some(3.0)); + assert_eq!(Hours::Str("4".to_string()).as_credits(), Some(4.0)); + assert_eq!(Hours::Str(" 3.5 ".to_string()).as_credits(), Some(3.5)); + assert_eq!(Hours::Str(String::new()).as_credits(), None); + assert_eq!(Hours::Str(" ".to_string()).as_credits(), None); + assert_eq!(Hours::Str("three".to_string()).as_credits(), None); + } + + #[test] + fn test_slug_and_infer_degree_type() { + assert_eq!(slug("AI Minor!"), "ai-minor"); + assert_eq!(slug(" --x-- "), "x"); + assert_eq!(slug(""), ""); + + assert_eq!(infer_degree_type("Bachelor of Arts in CS"), "BA"); + assert_eq!(infer_degree_type("Master of Science"), "MS"); + assert_eq!( + infer_degree_type("Bachelor's of Science Computer Science"), + "BS" + ); + assert_eq!(infer_degree_type(""), "BS"); + } + + #[test] + fn test_prereq_flip_and_of_or() { + // their: outer AND, inner OR -> (CS429H|CS429) & (M362K|SDS321) + let groups = vec![ + vec!["CS 429(H)".to_string(), "CS 429".to_string()], + vec!["M 362K".to_string(), "SDS 321".to_string()], + ]; + let expr = convert_groups_and_of_or(&groups).unwrap(); + // Top level must be AND (All) + assert!(matches!(expr, PrereqExpr::All(_))); + let json = serde_json::to_string(&expr).unwrap(); + assert_eq!( + json, + r#"{"and":[{"or":["CS429H","CS429"]},{"or":["M362K","SDS321"]}]}"# + ); + } + + #[test] + fn test_inner_and_inside_or() { + // [["MAC 2312 and MAD 2104", "MAC 2313"]] -> (MAC2312 & MAD2104) | MAC2313 + let groups = vec![vec![ + "MAC 2312 and MAD 2104".to_string(), + "MAC 2313".to_string(), + ]]; + let expr = convert_groups_and_of_or(&groups).unwrap(); + let json = serde_json::to_string(&expr).unwrap(); + assert_eq!(json, r#"{"or":[{"and":["MAC2312","MAD2104"]},"MAC2313"]}"#); + } + + #[test] + fn test_single_group_single_course() { + let groups = vec![vec!["MA 26100".to_string()]]; + let expr = convert_groups_and_of_or(&groups).unwrap(); + assert_eq!(expr, PrereqExpr::Course("MA26100".to_string())); + } + + #[test] + fn test_split_code_variants() { + assert_eq!(split_code("CSCE 3201"), ("CSCE".into(), "3201".into())); + assert_eq!(split_code("M 362K"), ("M".into(), "362K".into())); + assert_eq!(split_code("COT3100"), ("COT".into(), "3100".into())); + assert_eq!(split_code("CS 429(H)"), ("CS".into(), "429H".into())); + } + + #[test] + fn test_parse_picklist() { + assert_eq!( + parse_picklist("Foundations [1]"), + Some(("Foundations".into(), 1)) + ); + assert_eq!( + parse_picklist("Electives[4]"), + Some(("Electives".into(), 4)) + ); + assert_eq!(parse_picklist("no brackets"), None); + } + + #[test] + fn test_full_conversion_dedup_and_credits() { + let json = r#"{ + "university": "Test U", + "degree": "Bachelor's of Science Computer Science", + "ai_program": "Minor", + "courses": { + "cs_course_core": [ + {"course_code":"CS 101","title":"Intro","course_hours":"4","prerequisites":[]}, + {"course_code":"CS 201","title":"DS","prerequisites":[["CS 101"]]} + ], + "unrestricted_elective": [ + {"course_code":"CS 101","title":"Intro","course_hours":"4","prerequisites":[]} + ], + "ai_program_required_courses": [ + {"course_code":"CS 440","title":"AI","course_hours":"3","picklist":["AI Pick [1]"],"prerequisites":[["CS 201"]]} + ] + } + }"#; + let result = convert_landscape_str(json).unwrap(); + let p = &result.program; + + // CS101 appears in two categories but collapses to one course entry. + assert!(p.courses.contains_key("CS101")); + assert_eq!(p.courses.len(), 3); + + // CS201 missing course_hours -> defaulted + warning. + assert!((p.courses["CS201"].credit_hours - DEFAULT_CREDIT_HOURS).abs() < f32::EPSILON); + assert!(result.warnings.iter().any(|w| w.contains("CS201"))); + + // AI course tagged + picklist became a select requirement. + assert_eq!( + p.courses["CS440"].tags.as_deref(), + Some(&["ai".to_string()][..]) + ); + assert!(p + .requirements + .values() + .any(|r| r.req_type == RequirementType::Select)); + + // Program tagged as AI minor. + let tags = p.degree.tags.as_ref().unwrap(); + assert!(tags.contains(&"ai".to_string())); + assert!(tags.contains(&"ai-minor".to_string())); + } +} diff --git a/src/core/degree/mod.rs b/src/core/degree/mod.rs index 7b4ed2c..649b296 100644 --- a/src/core/degree/mod.rs +++ b/src/core/degree/mod.rs @@ -25,6 +25,8 @@ pub mod audit; pub mod course_reference; pub mod gen_ed_tracker; +pub mod json_parser; +pub mod landscape_convert; pub mod plan_generator; pub mod plan_selector; pub mod plan_validation; @@ -54,6 +56,16 @@ pub use yaml_parser::{ DegreeParseError, }; +// JSON input/output (unified format) and ai-landscape conversion +pub use json_parser::{ + load_degree_from_json, parse_degree_auto, parse_degree_json, parse_degree_json_with_warnings, + save_degree_to_json, serialize_degree_json, to_unified_value, unified_value_to_string, +}; +pub use landscape_convert::{ + convert_landscape, convert_landscape_str, extract_cluster_programs, ConversionResult, + LandscapeProgram, +}; + // Re-export course reference types pub use course_reference::CourseReference; diff --git a/src/core/degree/plan_generator.rs b/src/core/degree/plan_generator.rs index 86d9eab..888b69c 100644 --- a/src/core/degree/plan_generator.rs +++ b/src/core/degree/plan_generator.rs @@ -30,8 +30,15 @@ use crate::core::models::degree::Requirement; use fastrand::Rng; use std::collections::{HashMap, HashSet}; -/// Categories that should be enumerated for plan generation -const ENUMERABLE_CATEGORIES: [&str; 1] = ["major"]; +/// Categories whose requirements are enumerated as plan-variation choice points. +/// +/// `elective` is included so a "choose k of N" elective pool actually produces +/// plan variety; without it every converted program collapsed to a single plan +/// (`variations_run = 1`). Oversized pools stay memory-safe because the resolver +/// caps how many combinations it materializes (see `MAX_MATERIALIZED_COMBINATIONS`). +/// `gen_ed`/`supporting` are deliberately excluded — they keep their flat, +/// credit-reducing treatment. +const ENUMERABLE_CATEGORIES: [&str; 2] = ["major", "elective"]; // ============================================================================ // Helper Functions @@ -763,10 +770,12 @@ impl<'a> PlanGenerator<'a> { if self.major_requirements.is_empty() { return 1; // Single plan with no major choices } + // Saturating product: a capped elective pool can still push the nominal + // plan space past usize::MAX, and overflow would panic in debug builds. self.major_requirements .iter() .map(|r| r.choice_count.max(1)) - .product() + .fold(1usize, usize::saturating_mul) } /// Get statistics about the plan space @@ -1337,6 +1346,7 @@ mod tests { Requirement { name: Some("Core".to_string()), req_type: RequirementType::All, + tags: None, category: Some("major".to_string()), courses: Some(vec!["CS1000".to_string(), "CS2000".to_string()]), from: None, @@ -1354,6 +1364,7 @@ mod tests { Requirement { name: Some("Elective".to_string()), req_type: RequirementType::Select, + tags: None, category: Some("major".to_string()), courses: None, from: Some(FromClause { @@ -1392,6 +1403,59 @@ mod tests { assert_eq!(generator.estimate_plan_count(), 3); } + #[test] + fn test_elective_category_is_enumerated() { + // Bug B: a `category: "elective"` select must now contribute to the plan + // space. Previously only "major" was enumerated, collapsing converted + // programs to a single plan. + let courses = sample_courses(); + + // Baseline (core fixed + a major 1-of-3 select) = 3 plans. + let base = PlanGenerator::new( + &sample_requirements(), + &courses, + PlanGeneratorConfig::default(), + ); + assert_eq!(base.estimate_plan_count(), 3); + + // Add an elective-category select (choose 1 of 3): 3 -> 9. + let mut reqs = sample_requirements(); + reqs.insert( + "free_choice".to_string(), + Requirement { + name: Some("Free Choice".to_string()), + req_type: RequirementType::Select, + tags: None, + category: Some("elective".to_string()), + courses: None, + from: Some(FromClause { + courses: Some(vec![ + "CS1000".to_string(), + "CS2000".to_string(), + "CS3000".to_string(), + ]), + pattern: None, + include: None, + exclude: None, + groups: None, + groups_required: None, + per_group: None, + }), + count: Some(1), + credits: None, + credit_range: None, + constraints: None, + options: None, + }, + ); + let generator = PlanGenerator::new(&reqs, &courses, PlanGeneratorConfig::default()); + assert_eq!( + generator.estimate_plan_count(), + 9, + "an elective-category select(1 of 3) must multiply the plan space 3 -> 9" + ); + } + #[test] fn test_plan_generator_generates_all() { let courses = sample_courses(); @@ -1486,6 +1550,7 @@ mod tests { Requirement { name: Some("Gen Ed Math".to_string()), req_type: RequirementType::Select, + tags: None, category: Some("gen_ed".to_string()), courses: None, from: Some(FromClause { diff --git a/src/core/degree/requirement_resolver.rs b/src/core/degree/requirement_resolver.rs index bbd055c..e17ada6 100644 --- a/src/core/degree/requirement_resolver.rs +++ b/src/core/degree/requirement_resolver.rs @@ -5,8 +5,18 @@ use crate::core::models::course::Course; use crate::core::models::degree::{FromClause, Requirement, RequirementType}; +use fastrand::Rng; use std::collections::{HashMap, HashSet}; +/// Upper bound on the number of `select` combinations materialized for a single +/// requirement. A pool whose full `C(n, k)` exceeds this (e.g. "choose 15 of 42" +/// ≈ 10¹¹) is down-sampled to this many distinct combinations instead of being +/// fully enumerated. Plan-level sampling (capped by `max_plans`) then explores +/// across requirements, so losing exhaustiveness on one oversized pool is +/// acceptable — and it prevents the multi-gigabyte allocation that previously +/// OOM-killed full-catalog runs. +const MAX_MATERIALIZED_COMBINATIONS: usize = 2_000; + /// Represents the possible choices for a single requirement #[derive(Debug, Clone)] pub struct ResolvedRequirement { @@ -293,8 +303,9 @@ impl<'a> RequirementResolver<'a> { .with_exclude_used(exclude_used); } - // Generate all combinations of size `count` - let combinations = self.generate_combinations(&pool, count); + // Generate combinations of size `count`, bounding the materialized set so + // an oversized pool can't exhaust memory (see MAX_MATERIALIZED_COMBINATIONS). + let combinations = self.bounded_combinations(&pool, count); // For exclude_used requirements, also track the pool and count for dynamic selection let mut resolved = @@ -732,6 +743,66 @@ impl<'a> RequirementResolver<'a> { false } + /// Generate size-`k` combinations from `pool`, fully when `C(n, k)` is small + /// and as a bounded random sample when it would exceed + /// [`MAX_MATERIALIZED_COMBINATIONS`]. + #[allow(clippy::unused_self)] // Keep as method for API consistency + fn bounded_combinations(&self, pool: &[String], k: usize) -> Vec> { + if Self::combinations_exceed(pool.len(), k, MAX_MATERIALIZED_COMBINATIONS) { + Self::sample_combinations(pool, k, MAX_MATERIALIZED_COMBINATIONS) + } else { + self.generate_combinations(pool, k) + } + } + + /// Whether `C(n, k)` strictly exceeds `cap`, computed without overflowing: + /// the running value never grows past `cap * n` before the early return. + fn combinations_exceed(n: usize, k: usize, cap: usize) -> bool { + if k > n { + return false; + } + let k = k.min(n - k); // C(n, k) == C(n, n - k); use the smaller factor count + let cap = cap as u128; + let mut result: u128 = 1; + for i in 0..k as u128 { + result = result * (n as u128 - i) / (i + 1); + if result > cap { + return true; + } + } + false + } + + /// Draw up to `cap` distinct size-`k` combinations from `pool`. Deterministic + /// (fixed seed derived from the pool shape) so analysis output is reproducible. + fn sample_combinations(pool: &[String], k: usize, cap: usize) -> Vec> { + let n = pool.len(); + if k == 0 || k > n { + return vec![Vec::new()]; + } + let mut rng = Rng::with_seed(0x5EED_0000 ^ ((n as u64) << 16) ^ k as u64); + let mut seen: HashSet> = HashSet::with_capacity(cap); + let mut out: Vec> = Vec::with_capacity(cap); + // Bound attempts so we can't spin forever if cap approaches C(n, k). + let max_attempts = cap.saturating_mul(20).max(cap + 1); + let mut attempts = 0; + while out.len() < cap && attempts < max_attempts { + attempts += 1; + let mut idx: Vec = Vec::with_capacity(k); + while idx.len() < k { + let c = rng.usize(0..n); + if !idx.contains(&c) { + idx.push(c); + } + } + idx.sort_unstable(); + if seen.insert(idx.clone()) { + out.push(idx.iter().map(|&i| pool[i].clone()).collect()); + } + } + out + } + /// Generate all combinations of size k from a pool #[allow(clippy::unused_self)] // Keep as method for API consistency fn generate_combinations(&self, pool: &[String], k: usize) -> Vec> { @@ -903,6 +974,7 @@ mod tests { let req = Requirement { name: Some("Core".to_string()), req_type: RequirementType::All, + tags: None, category: Some("major".to_string()), courses: Some(vec!["CS1000".to_string(), "CS2000".to_string()]), from: None, @@ -929,6 +1001,7 @@ mod tests { let req = Requirement { name: Some("Electives".to_string()), req_type: RequirementType::Select, + tags: None, category: Some("major".to_string()), courses: None, from: Some(FromClause { @@ -974,6 +1047,91 @@ mod tests { assert!(combos.contains(&vec!["B".to_string(), "C".to_string()])); } + #[test] + fn test_combinations_exceed_matches_binomial() { + // Exact boundary cases around the cap. + assert!(!RequirementResolver::combinations_exceed(5, 2, 10)); // C(5,2)=10, not >10 + assert!(RequirementResolver::combinations_exceed(5, 2, 9)); // 10 > 9 + assert!(!RequirementResolver::combinations_exceed( + 42, + 15, + usize::MAX + )); // huge but not > MAX-ish cap + // The pathological pool that used to OOM must be flagged as oversized. + assert!(RequirementResolver::combinations_exceed(42, 15, 2_000)); + assert!(RequirementResolver::combinations_exceed(16_000, 6, 2_000)); + // k > n is zero combinations, never exceeds. + assert!(!RequirementResolver::combinations_exceed(3, 5, 1)); + } + + #[test] + fn test_sample_combinations_bounded_distinct_and_valid() { + let pool: Vec = (0..42).map(|i| format!("C{i}")).collect(); + let cap = 2_000; + let sample = RequirementResolver::sample_combinations(&pool, 15, cap); + + assert!(sample.len() <= cap, "must not exceed the cap"); + assert!( + sample.len() > 1, + "should produce many distinct combinations" + ); + // Every combination has the right size and only real pool courses. + for combo in &sample { + assert_eq!(combo.len(), 15); + assert!(combo.iter().all(|c| pool.contains(c))); + let mut uniq = combo.clone(); + uniq.sort(); + uniq.dedup(); + assert_eq!(uniq.len(), 15, "courses within a combination are distinct"); + } + // Distinct combinations (no duplicates in the sampled set). + let mut keys: Vec> = sample.clone(); + keys.sort(); + keys.dedup(); + assert_eq!(keys.len(), sample.len(), "combinations are distinct"); + } + + #[test] + fn test_sample_combinations_is_deterministic() { + let pool: Vec = (0..30).map(|i| format!("C{i}")).collect(); + let a = RequirementResolver::sample_combinations(&pool, 10, 500); + let b = RequirementResolver::sample_combinations(&pool, 10, 500); + assert_eq!(a, b, "fixed seed must yield reproducible output"); + } + + #[test] + fn test_bounded_combinations_caps_huge_pool_but_enumerates_small() { + let courses = sample_courses(); + let resolver = RequirementResolver::new(&courses); + + // Small pool: fully enumerated (C(5,2)=10). + let small: Vec = (0..5).map(|i| format!("S{i}")).collect(); + assert_eq!(resolver.bounded_combinations(&small, 2).len(), 10); + + // Huge pool: capped at MAX_MATERIALIZED_COMBINATIONS instead of C(42,15). + let big: Vec = (0..42).map(|i| format!("B{i}")).collect(); + assert_eq!( + resolver.bounded_combinations(&big, 15).len(), + MAX_MATERIALIZED_COMBINATIONS + ); + } + + #[test] + fn test_bounded_combinations_k_zero_and_k_equals_n() { + let courses = sample_courses(); + let resolver = RequirementResolver::new(&courses); + let pool: Vec = (0..5).map(|i| format!("C{i}")).collect(); + + // k == 0 -> the single empty combination. + let zero = resolver.bounded_combinations(&pool, 0); + assert_eq!(zero, vec![Vec::::new()]); + + // k == n -> exactly one combination containing all courses. + let all = resolver.bounded_combinations(&pool, 5); + assert_eq!(all.len(), 1); + assert_eq!(all[0].len(), 5); + } + #[test] fn test_expand_course_list_bundles() { let courses = sample_courses(); diff --git a/src/core/degree/trim.rs b/src/core/degree/trim.rs index 9c47503..e2670be 100644 --- a/src/core/degree/trim.rs +++ b/src/core/degree/trim.rs @@ -971,6 +971,7 @@ mod tests { Requirement { name: None, req_type: RequirementType::All, + tags: None, category: None, courses: Some(courses.into_iter().map(str::to_string).collect()), from: None, @@ -1103,6 +1104,7 @@ mod tests { let select_req = Requirement { name: Some("pick 2".to_string()), req_type: RequirementType::Select, + tags: None, category: None, courses: None, from: Some(empty_from_with_courses(vec![ @@ -1135,6 +1137,7 @@ mod tests { let req = Requirement { name: None, req_type: RequirementType::All, + tags: None, category: None, courses: Some(vec!["{BIO100, BIO200}".to_string()]), from: None, @@ -1160,6 +1163,7 @@ mod tests { let req = Requirement { name: None, req_type: RequirementType::All, + tags: None, category: None, courses: Some(vec!["{CS530, CS535}".to_string()]), from: None, @@ -1187,6 +1191,7 @@ mod tests { let req = Requirement { name: None, req_type: RequirementType::All, + tags: None, category: None, courses: Some(vec!["{BIO100, BIO200}".to_string()]), from: None, @@ -1216,6 +1221,7 @@ mod tests { let req = Requirement { name: None, req_type: RequirementType::All, + tags: None, category: None, courses: Some(vec!["{MATH215, MATH241}".to_string()]), from: None, @@ -1264,6 +1270,7 @@ mod tests { let req = Requirement { name: None, req_type: RequirementType::All, + tags: None, category: None, courses: Some(vec!["{MATH215, MATH241}".to_string()]), from: None, @@ -1302,6 +1309,7 @@ mod tests { let req = Requirement { name: None, req_type: RequirementType::Select, + tags: None, category: None, courses: None, from: Some(FromClause { @@ -1406,6 +1414,7 @@ mod tests { let req = Requirement { name: None, req_type: RequirementType::All, + tags: None, category: None, courses: Some(vec!["{MATH215, MATH241}".to_string()]), from: None, @@ -1447,6 +1456,7 @@ mod tests { let req = Requirement { name: None, req_type: RequirementType::Select, + tags: None, category: None, courses: None, from: Some(FromClause { @@ -1502,6 +1512,7 @@ mod tests { let req = Requirement { name: None, req_type: RequirementType::Select, + tags: None, category: None, courses: None, from: Some(FromClause { @@ -1551,6 +1562,7 @@ mod tests { let req = Requirement { name: None, req_type: RequirementType::Select, + tags: None, category: None, courses: None, from: Some(FromClause { @@ -1591,6 +1603,7 @@ mod tests { let nested = Requirement { name: None, req_type: RequirementType::Select, + tags: None, category: None, courses: None, from: Some(FromClause { @@ -1615,6 +1628,7 @@ mod tests { let outer = Requirement { name: None, req_type: RequirementType::OneOf, + tags: None, category: None, courses: None, from: None, @@ -1641,6 +1655,7 @@ mod tests { let req = Requirement { name: None, req_type: RequirementType::All, + tags: None, category: None, courses: Some(vec![ "CS100".to_string(), @@ -1724,6 +1739,7 @@ mod tests { let select_req = Requirement { name: None, req_type: RequirementType::Select, + tags: None, category: None, courses: None, from: Some(empty_from_with_courses(vec![ diff --git a/src/core/degree/yaml_parser.rs b/src/core/degree/yaml_parser.rs index d7fd507..da101f2 100644 --- a/src/core/degree/yaml_parser.rs +++ b/src/core/degree/yaml_parser.rs @@ -25,6 +25,9 @@ pub enum DegreeParseError { /// 1-indexed source column, when known. column: Option, }, + + /// JSON parse/serialize error (unified JSON or ai-landscape input). + JsonError(String), } impl DegreeParseError { @@ -38,6 +41,12 @@ impl DegreeParseError { column: None, } } + + /// Construct a `JsonError` from a message. + #[must_use] + pub fn json_message(message: impl Into) -> Self { + Self::JsonError(message.into()) + } } impl std::fmt::Display for DegreeParseError { @@ -45,6 +54,7 @@ impl std::fmt::Display for DegreeParseError { match self { Self::IoError(msg) => write!(f, "IO Error: {msg}"), Self::YamlError { message, .. } => write!(f, "YAML Parse Error: {message}"), + Self::JsonError(msg) => write!(f, "JSON Parse Error: {msg}"), } } } @@ -104,8 +114,11 @@ pub fn parse_degree_yaml(yaml_content: &str) -> Result panic!("expected YamlError, got IoError({msg})"), + DegreeParseError::JsonError(msg) => panic!("expected YamlError, got JsonError({msg})"), } } diff --git a/src/core/models/course.rs b/src/core/models/course.rs index d3b4261..bbabe94 100644 --- a/src/core/models/course.rs +++ b/src/core/models/course.rs @@ -1,5 +1,6 @@ //! Course model +use crate::core::prerequisite_parser::PrereqExpr; use serde::{Deserialize, Deserializer, Serialize}; /// Variable credit range for courses with flexible credits @@ -85,14 +86,22 @@ where #[derive(Deserialize)] #[serde(untagged)] enum PrereqRaw { + /// A boolean-expression string (YAML form), e.g. "(CS101 | CS102) & CS103", + /// or a bare single-course leaf, e.g. "CS101". String(String), + /// Legacy resolved prerequisite list (ignored; not the raw form). Vec(Vec), + /// Unified-JSON structured form: a `{"and"|"or": [...]}` tagged object. + Expr(PrereqExpr), Null, } match PrereqRaw::deserialize(deserializer)? { PrereqRaw::String(s) => Ok(Some(s)), PrereqRaw::Vec(_v) => Ok(None), // Vec is resolved prerequisites, not raw + // Collapse the structured tree back into the canonical boolean string so + // the rest of the pipeline (trim, course graph, resolver) is unchanged. + PrereqRaw::Expr(expr) => Ok(Some(expr.to_expression_string())), PrereqRaw::Null => Ok(None), } } @@ -177,6 +186,11 @@ pub struct Course { /// Variable credit range (alternative to fixed `credit_hours`) #[serde(skip_serializing_if = "Option::is_none")] pub credit_range: Option, + + /// Classification tags (e.g., `["ai"]`). Generalizes the ai-landscape + /// course categories so the unified format is program-agnostic. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tags: Option>, } impl Course { @@ -209,6 +223,7 @@ impl Course { repeatable: None, max_repeat_credits: None, credit_range: None, + tags: None, } } @@ -246,6 +261,7 @@ impl Course { repeatable: None, max_repeat_credits: None, credit_range: None, + tags: None, } } @@ -319,6 +335,15 @@ impl Course { pub fn set_canonical_name(&mut self, name: String) { self.canonical_name = Some(name); } + + /// Add a classification tag (e.g. `"ai"`), creating the tag list if needed + /// and skipping duplicates. + pub fn add_tag(&mut self, tag: &str) { + let tags = self.tags.get_or_insert_with(Vec::new); + if !tags.iter().any(|t| t == tag) { + tags.push(tag.to_string()); + } + } } #[cfg(test)] @@ -399,6 +424,51 @@ mod tests { assert_eq!(course.corequisites[0], "PHYS1152"); } + #[test] + fn test_deserialize_structured_prereq_object_into_raw_string() { + // Unified-JSON structured tagged prereqs collapse to the canonical + // boolean string (the OR child of the AND is parenthesized). + let json = r#"{ + "name": "Data Structures", "prefix": "CS", "number": "201", + "credit_hours": 4.0, + "prerequisites": {"and": ["CS101", {"or": ["CS102", "CS103"]}]} + }"#; + let course: Course = serde_json::from_str(json).unwrap(); + assert_eq!( + course.prerequisites_raw.as_deref(), + Some("CS101 & (CS102 | CS103)") + ); + } + + #[test] + fn test_deserialize_bare_string_prereq_passthrough() { + let json = r#"{"name":"X","prefix":"CS","number":"2","credit_hours":3.0, + "prerequisites":"CS101 | CS102"}"#; + let course: Course = serde_json::from_str(json).unwrap(); + assert_eq!(course.prerequisites_raw.as_deref(), Some("CS101 | CS102")); + } + + #[test] + fn test_deserialize_null_prereq_is_none() { + let json = r#"{"name":"X","prefix":"CS","number":"1","credit_hours":3.0, + "prerequisites":null}"#; + let course: Course = serde_json::from_str(json).unwrap(); + assert!(course.prerequisites_raw.is_none()); + } + + #[test] + fn test_add_tag_dedups_and_creates() { + let mut course = Course::new("X".into(), "CS".into(), "1".into(), 3.0); + assert!(course.tags.is_none()); + course.add_tag("ai"); + course.add_tag("ai"); // duplicate ignored + course.add_tag("core"); + assert_eq!( + course.tags.as_deref(), + Some(&["ai".to_string(), "core".to_string()][..]) + ); + } + #[test] fn test_canonical_name() { let mut course = Course::new( diff --git a/src/core/models/degree.rs b/src/core/models/degree.rs index 01d84d7..9a628d6 100644 --- a/src/core/models/degree.rs +++ b/src/core/models/degree.rs @@ -81,6 +81,11 @@ pub struct Degree { /// Whether courses can satisfy multiple requirements #[serde(skip_serializing_if = "Option::is_none")] pub allow_double_counting: Option, + + /// Program-level classification tags (e.g., `["ai"]`, `["ai-concentration"]`). + /// Generalizes the ai-landscape `ai_program` enum so any program type fits. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tags: Option>, } /// A requirement in a degree program @@ -125,6 +130,12 @@ pub struct Requirement { /// Mutually exclusive paths (for type: `one_of`) #[serde(skip_serializing_if = "Option::is_none")] pub options: Option>, + + /// Classification tags for this requirement group (e.g., `["core"]`, + /// `["ai","required"]`, `["elective"]`). Generalizes the ai-landscape + /// fixed category names. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tags: Option>, } /// Types of requirements @@ -276,6 +287,7 @@ impl Degree { grade_minimum_note: None, major_subjects: None, allow_double_counting: None, + tags: None, } } @@ -311,6 +323,7 @@ impl Degree { grade_minimum_note: None, major_subjects: None, allow_double_counting: Some(allow_double_counting), + tags: None, } } diff --git a/src/core/prerequisite_parser.rs b/src/core/prerequisite_parser.rs index a9e18e4..c94d37d 100644 --- a/src/core/prerequisite_parser.rs +++ b/src/core/prerequisite_parser.rs @@ -21,6 +21,7 @@ //! - `(CS101 & CS102) | CS103` - Both CS101 and CS102, OR just CS103 //! - `CS101[B] & CS102[C]` - With grade requirements (stripped) +use serde::{Deserialize, Deserializer, Serialize, Serializer}; use std::collections::HashSet; /// AND/OR tree form of a prerequisite expression. @@ -93,6 +94,89 @@ impl PrereqExpr { } } +// Unified-JSON serialization for prerequisites (symmetric tagged form): +// - a leaf course -> a bare JSON string ("CS101") +// - All (AND) -> {"and": [ ]} +// - Any (OR) -> {"or": [ ]} +// This nests to any depth and round-trips losslessly with the AST. The node +// type encodes the operator, so the structure is unambiguous for consumers. +impl Serialize for PrereqExpr { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + use serde::ser::SerializeMap; + match self { + Self::Course(c) => serializer.serialize_str(c), + Self::All(xs) => { + let mut map = serializer.serialize_map(Some(1))?; + map.serialize_entry("and", xs)?; + map.end() + } + Self::Any(xs) => { + let mut map = serializer.serialize_map(Some(1))?; + map.serialize_entry("or", xs)?; + map.end() + } + } + } +} + +impl<'de> Deserialize<'de> for PrereqExpr { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct PrereqExprVisitor; + + impl<'de> serde::de::Visitor<'de> for PrereqExprVisitor { + type Value = PrereqExpr; + + fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.write_str("a course string or an object with a single `and`/`or` key") + } + + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + Ok(PrereqExpr::Course(v.to_string())) + } + + fn visit_string(self, v: String) -> Result + where + E: serde::de::Error, + { + Ok(PrereqExpr::Course(v)) + } + + fn visit_map(self, mut map: A) -> Result + where + A: serde::de::MapAccess<'de>, + { + let key: String = map + .next_key()? + .ok_or_else(|| serde::de::Error::custom("expected `and` or `or` key"))?; + let children: Vec = map.next_value()?; + if map.next_key::()?.is_some() { + return Err(serde::de::Error::custom( + "prerequisite object must have exactly one key (`and` or `or`)", + )); + } + match key.as_str() { + "and" => Ok(PrereqExpr::All(children)), + "or" => Ok(PrereqExpr::Any(children)), + other => Err(serde::de::Error::custom(format!( + "unknown prerequisite operator `{other}` (expected `and` or `or`)" + ))), + } + } + } + + deserializer.deserialize_any(PrereqExprVisitor) + } +} + /// Parse a prerequisite expression into a structural AND/OR tree. /// /// Returns `None` for empty input. Operator precedence matches @@ -462,6 +546,69 @@ mod tests { assert_eq!(result, vec![vec!["CS101".to_string()]]); } + #[test] + fn test_prereq_expr_json_tagged_form() { + // Leaf -> bare string + let leaf = PrereqExpr::Course("CS101".into()); + assert_eq!(serde_json::to_string(&leaf).unwrap(), "\"CS101\""); + + // (X & Y) | Z -> {"or":[{"and":["X","Y"]},"Z"]} + let expr = parse_to_ast("(X & Y) | Z").unwrap(); + let json = serde_json::to_string(&expr).unwrap(); + assert_eq!(json, r#"{"or":[{"and":["X","Y"]},"Z"]}"#); + + // Round-trips losslessly + let back: PrereqExpr = serde_json::from_str(&json).unwrap(); + assert_eq!(back, expr); + } + + #[test] + fn test_prereq_expr_json_rejects_bad_operator() { + let err = serde_json::from_str::(r#"{"xor":["A","B"]}"#); + assert!(err.is_err()); + } + + #[test] + fn test_prereq_expr_deserialize_bare_string_leaf() { + let leaf: PrereqExpr = serde_json::from_str("\"CS101\"").unwrap(); + assert_eq!(leaf, PrereqExpr::Course("CS101".to_string())); + } + + #[test] + fn test_prereq_expr_deserialize_empty_and_or() { + let and: PrereqExpr = serde_json::from_str(r#"{"and":[]}"#).unwrap(); + assert_eq!(and, PrereqExpr::All(vec![])); + let or: PrereqExpr = serde_json::from_str(r#"{"or":[]}"#).unwrap(); + assert_eq!(or, PrereqExpr::Any(vec![])); + } + + #[test] + fn test_prereq_expr_deserialize_rejects_two_key_object() { + let err = serde_json::from_str::(r#"{"and":["A"],"or":["B"]}"#); + assert!(err.is_err(), "object with two keys must be rejected"); + } + + #[test] + fn test_prereq_expr_deserialize_nested_depth_roundtrips() { + // (A & (B | C)) | D — recursion through both node types. + let json = r#"{"or":[{"and":["A",{"or":["B","C"]}]},"D"]}"#; + let expr: PrereqExpr = serde_json::from_str(json).unwrap(); + assert_eq!( + expr, + PrereqExpr::Any(vec![ + PrereqExpr::All(vec![ + PrereqExpr::Course("A".into()), + PrereqExpr::Any(vec![ + PrereqExpr::Course("B".into()), + PrereqExpr::Course("C".into()), + ]), + ]), + PrereqExpr::Course("D".into()), + ]) + ); + assert_eq!(serde_json::to_string(&expr).unwrap(), json); + } + #[test] fn test_parse_to_dnf_and() { let result = parse_to_dnf("CS101 & CS102"); diff --git a/src/core/report/mod.rs b/src/core/report/mod.rs index 7e68255..6fd62c8 100644 --- a/src/core/report/mod.rs +++ b/src/core/report/mod.rs @@ -8,6 +8,7 @@ pub mod degree_report; pub mod formats; pub mod plan_export; pub mod term_scheduler; +pub mod unified_report; pub mod visualization; use crate::core::metrics::CurriculumMetrics; diff --git a/src/core/report/plan_export.rs b/src/core/report/plan_export.rs index 8121956..9224e19 100644 --- a/src/core/report/plan_export.rs +++ b/src/core/report/plan_export.rs @@ -432,7 +432,9 @@ pub fn scored_plan_to_model( /// Sanitize a string for use as a filename /// /// Replaces characters that are invalid in filenames with underscores. -fn sanitize_filename(s: &str) -> String { +/// Replace filesystem-hostile characters in `s` with `_` so it can be used as a +/// filename component. Shared across report exporters. +pub(crate) fn sanitize_filename(s: &str) -> String { s.chars() .map(|c| match c { '/' | '\\' | ':' | '*' | '?' | '"' | '<' | '>' | '|' | ' ' => '_', @@ -508,7 +510,7 @@ pub struct DegreeSummary { } /// Summary of a single selected plan for JSONL export -#[derive(Debug, Clone)] +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct PlanSummary { /// Plan category name pub category: String, @@ -761,6 +763,26 @@ pub fn export_index_csv( Ok(output_path) } +/// (Re)create `index.csv` in `output_dir` containing only the header row. +/// +/// Used by parallel batch runners to write the header once up front so that +/// concurrent [`export_index_csv`] calls (which only append a row when the file +/// already exists) don't each race to write their own header. For typical row +/// sizes, an `O_APPEND` write stays under `PIPE_BUF` and so lands atomically on +/// POSIX; a pathologically long row could in principle interleave. +/// +/// # Errors +/// Returns an error if the file cannot be created or written. +pub fn write_index_csv_header(output_dir: &Path) -> Result> { + let output_path = output_dir.join("index.csv"); + if let Some(parent) = output_path.parent() { + fs::create_dir_all(parent)?; + } + let mut file = File::create(&output_path)?; + writeln!(file, "{INDEX_CSV_HEADER}")?; + Ok(output_path) +} + /// Format a single row for the index CSV fn format_index_csv_row( school: &School, @@ -1121,4 +1143,45 @@ mod tests { "Header should appear exactly once" ); } + + #[test] + fn test_write_index_csv_header_writes_only_header() { + let tmp = TempDir::new().unwrap(); + let path = write_index_csv_header(tmp.path()).unwrap(); + + assert_eq!(path, tmp.path().join("index.csv")); + let contents = fs::read_to_string(&path).unwrap(); + assert_eq!(contents.trim_end(), INDEX_CSV_HEADER); + assert_eq!(contents.lines().count(), 1, "header only, no data rows"); + } + + #[test] + fn test_write_index_csv_header_creates_missing_parent_dirs() { + let tmp = TempDir::new().unwrap(); + let nested = tmp.path().join("a").join("b"); + let path = write_index_csv_header(&nested).unwrap(); + assert!(path.exists(), "header file created under a new directory"); + } + + #[test] + fn test_write_index_csv_header_then_append_gives_one_header() { + // The parallel-batch contract: pre-write the header, then workers only + // append rows (export_index_csv must not add a second header). + let tmp = TempDir::new().unwrap(); + write_index_csv_header(tmp.path()).unwrap(); + + let school = create_test_school(); + let degree = create_test_degree(); + let aggregator = create_test_aggregator(); + let selected = create_test_selected(); + export_index_csv(&school, °ree, &aggregator, &selected, tmp.path()).unwrap(); + + let contents = fs::read_to_string(tmp.path().join("index.csv")).unwrap(); + assert_eq!(contents.lines().count(), 2, "header + one appended row"); + assert_eq!( + contents.matches("degree_id,degree_name").count(), + 1, + "pre-written header must not be duplicated by the append" + ); + } } diff --git a/src/core/report/unified_report.rs b/src/core/report/unified_report.rs new file mode 100644 index 0000000..2c19a5f --- /dev/null +++ b/src/core/report/unified_report.rs @@ -0,0 +1,389 @@ +//! Unified metrics-rich degree report (JSON). +//! +//! Produces a single self-contained JSON document carrying **the whole degree +//! structure plus the numbers**: the unified program (degree metadata + tags, +//! requirements, courses with structured prerequisites), per-course metric +//! statistics, degree-level metric statistics, and the sampling metadata +//! (`variations_run`, `sample_type`). This is what the ai-landscape +//! whole-degree visualization (and a future DB load) consume. + +use std::error::Error; +use std::path::{Path, PathBuf}; + +use serde::Serialize; +use serde_json::{json, Value}; + +use crate::core::degree::json_parser::to_unified_value; +use crate::core::models::DegreeProgram; +use crate::core::report::plan_export::sanitize_filename; +use crate::core::statistics::aggregator::{MetricStats, MetricsAggregator}; +use crate::core::statistics::DescriptiveStats; + +/// One term of a selected plan's schedule (mirrors the MCP `analyze_degree` +/// term shape so consumers can reuse the same rendering). +#[derive(Serialize)] +struct TermSchedule { + /// 1-based term number. + term: usize, + /// Course ids scheduled in this term. + courses: Vec, + /// Total credits this term. + credits: f32, +} + +/// A selected sample plan with its course schedule, surfaced in the report so a +/// consumer can see exactly which courses each exemplar (shortest/longest/…) +/// contains — especially the shortest path. +#[derive(Serialize)] +struct SelectedPlanReport { + /// Plan category ("shortest", "longest", …). + category: String, + /// Terms required to complete this plan. + terms_required: usize, + /// Total structural complexity. + total_complexity: usize, + /// Longest delay factor. + longest_delay: usize, + /// Whether the plan is calc-ready. + is_calc_ready: bool, + /// Total credits across the plan. + credits: f32, + /// Number of courses in the plan. + course_count: usize, + /// Longest prerequisite (delay) chain — the critical path. + critical_path: Vec, + /// Term-by-term schedule (empty terms omitted). + schedule: Vec, +} + +/// Build the unified report `Value` for a single analyzed degree. +/// +/// `sample_type` is the human-readable sampling strategy (e.g. "shuffled"). +/// +/// # Errors +/// Returns an error if the program cannot be serialized. +pub fn build_degree_report( + program: &DegreeProgram, + aggregator: &MetricsAggregator, + selected: &crate::core::degree::SelectedPlans, + sample_type: &str, +) -> Result> { + // Start from the unified program (degree, requirements, courses w/ prereqs). + let mut value = to_unified_value(program).map_err(|e| Box::::from(e.to_string()))?; + + // Attach per-course metric statistics, keyed by course id. + if let Some(courses) = value.get_mut("courses").and_then(Value::as_object_mut) { + for (key, course_value) in courses.iter_mut() { + if let (Some(stats), Some(obj)) = + (aggregator.course_stats(key), course_value.as_object_mut()) + { + obj.insert("metrics".to_string(), serde_json::to_value(&stats)?); + } + } + } + + // Degree-level metrics + sampling metadata. + let degree_stats = aggregator.degree_stats(); + let selected_plans: Vec = selected + .iter() + .map(|(category, plan)| SelectedPlanReport { + category: category.display_name().to_string(), + terms_required: plan.score.terms_required, + total_complexity: plan.score.total_complexity, + longest_delay: plan.score.longest_delay, + is_calc_ready: plan.score.is_calc_ready, + credits: plan.variant.total_credits, + course_count: plan.variant.courses.len(), + critical_path: plan.score.longest_delay_chain.clone(), + schedule: plan + .schedule + .terms + .iter() + .filter(|t| !t.courses.is_empty()) + .map(|t| TermSchedule { + term: t.number, + courses: t.courses.clone(), + credits: t.total_credits, + }) + .collect(), + }) + .collect(); + + if let Some(obj) = value.as_object_mut() { + obj.insert( + "analysis".to_string(), + json!({ + "variations_run": aggregator.plan_count(), + "sample_type": sample_type, + "metrics": { + "complexity": degree_stats.total_complexity, + "delay": degree_stats.longest_delay, + "credits": degree_stats.total_credits, + } + }), + ); + obj.insert( + "selected_plans".to_string(), + serde_json::to_value(&selected_plans)?, + ); + } + + Ok(value) +} + +/// Write the unified report JSON to `/_report.json`. +/// +/// # Errors +/// Returns an error if the report cannot be built or written. +pub fn export_degree_report_json( + program: &DegreeProgram, + aggregator: &MetricsAggregator, + selected: &crate::core::degree::SelectedPlans, + sample_type: &str, + out_dir: &Path, +) -> Result> { + let value = build_degree_report(program, aggregator, selected, sample_type)?; + std::fs::create_dir_all(out_dir)?; + let path = out_dir.join(format!( + "{}_report.json", + sanitize_filename(&program.degree.degree_id()) + )); + std::fs::write(&path, report_value_to_pretty(&value)?)?; + Ok(path) +} + +/// Serialize a built report `Value` to pretty JSON with `degree` first, then the +/// analysis summary, requirements, and selected plans, with the large `courses` +/// block last. Nested objects keep `serde_json`'s deterministic sorted key order. +fn report_value_to_pretty(value: &Value) -> Result { + #[derive(Serialize)] + struct Ordered<'a> { + degree: &'a Value, + analysis: &'a Value, + requirements: &'a Value, + selected_plans: &'a Value, + courses: &'a Value, + } + let null = Value::Null; + let ordered = Ordered { + degree: value.get("degree").unwrap_or(&null), + analysis: value.get("analysis").unwrap_or(&null), + requirements: value.get("requirements").unwrap_or(&null), + selected_plans: value.get("selected_plans").unwrap_or(&null), + courses: value.get("courses").unwrap_or(&null), + }; + serde_json::to_string_pretty(&ordered) +} + +/// Degree-level rollup for one program, used to build a school-level report. +#[derive(Debug, Clone)] +pub struct ProgramRollup { + /// Degree identifier. + pub id: String, + /// Degree/program name. + pub name: String, + /// Program-level tags (e.g. `["ai"]`). + pub tags: Option>, + /// Number of plan variations analyzed. + pub variations_run: usize, + /// Degree-level total-complexity statistics. + pub complexity: MetricStats, + /// Degree-level longest-delay statistics. + pub delay: MetricStats, + /// Degree-level total-credits statistics. + pub credits: MetricStats, +} + +impl ProgramRollup { + /// Build a rollup from an analyzed program + aggregator. + #[must_use] + pub fn from_analysis(program: &DegreeProgram, aggregator: &MetricsAggregator) -> Self { + let stats = aggregator.degree_stats(); + Self { + id: program.degree.degree_id(), + name: program.degree.name.clone(), + tags: program.degree.tags.clone(), + variations_run: aggregator.plan_count(), + complexity: stats.total_complexity, + delay: stats.longest_delay, + credits: stats.total_credits, + } + } +} + +/// Write a school-level report aggregating the degree-level metrics of every +/// program in the school. School metrics summarize each program's median +/// values across the school. +/// +/// # Errors +/// Returns an error if the file cannot be written. +pub fn export_school_report_json( + school: &str, + programs: &[ProgramRollup], + out_dir: &Path, +) -> Result> { + let program_values: Vec = programs + .iter() + .map(|p| { + json!({ + "id": p.id, + "name": p.name, + "tags": p.tags, + "variations_run": p.variations_run, + "metrics": { + "complexity": p.complexity, + "delay": p.delay, + "credits": p.credits, + } + }) + }) + .collect(); + + let complexity_medians: Vec = programs.iter().map(|p| p.complexity.median).collect(); + let delay_medians: Vec = programs.iter().map(|p| p.delay.median).collect(); + let credit_medians: Vec = programs.iter().map(|p| p.credits.median).collect(); + + let report = json!({ + "school": school, + "program_count": programs.len(), + "school_metrics": { + "complexity": aggregate(&complexity_medians), + "delay": aggregate(&delay_medians), + "credits": aggregate(&credit_medians), + }, + "programs": program_values, + }); + + std::fs::create_dir_all(out_dir)?; + let path = out_dir.join(format!("{}_school_report.json", sanitize_filename(school))); + std::fs::write(&path, serde_json::to_string_pretty(&report)?)?; + Ok(path) +} + +/// Summary statistics over a set of values (the per-program medians). +/// +/// Returns `Null` for an empty set; otherwise the full quartile / mean / +/// standard-deviation block from [`DescriptiveStats`] (shared, correct +/// percentile median). +fn aggregate(values: &[f64]) -> Value { + if values.is_empty() { + return Value::Null; + } + let s = DescriptiveStats::from_values(values); + json!({ + "count": s.count, + "mean": s.mean, + "median": s.median, + "std_dev": s.std_dev, + "min": s.min, + "max": s.max, + "q1": s.q1, + "q3": s.q3, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::metrics::CourseMetrics; + use crate::core::models::Degree; + use std::collections::HashMap; + + fn f(v: &Value, key: &str) -> f64 { + v.get(key).and_then(Value::as_f64).unwrap() + } + + #[test] + fn test_aggregate_empty_is_null() { + assert_eq!(aggregate(&[]), Value::Null); + } + + #[test] + fn test_aggregate_forwards_descriptive_stats() { + // Population std_dev (/n): values 2,4,4,4,5,5,7,9 -> mean 5, std_dev 2. + let v = aggregate(&[2.0, 4.0, 4.0, 4.0, 5.0, 5.0, 7.0, 9.0]); + assert_eq!(v.get("count").and_then(Value::as_u64), Some(8)); + assert!((f(&v, "mean") - 5.0).abs() < 1e-9); + assert!((f(&v, "std_dev") - 2.0).abs() < 1e-9); + assert!((f(&v, "min") - 2.0).abs() < 1e-9); + assert!((f(&v, "max") - 9.0).abs() < 1e-9); + // Quartiles present (the bug-fix that the old hand-rolled version lacked). + assert!(v.get("q1").is_some() && v.get("q3").is_some()); + } + + fn aggregator_with_two_plans() -> MetricsAggregator { + let mut agg = MetricsAggregator::default(); + for c in [10usize, 20] { + let mut cm = HashMap::new(); + cm.insert( + "CS1000".to_string(), + CourseMetrics { + complexity: c, + centrality: c / 2, + delay: c / 5, + blocking: c / 3, + }, + ); + agg.add_plan(&cm, 4.0); + } + agg + } + + fn sample_program() -> DegreeProgram { + let mut degree = Degree::new( + "Computer Science".to_string(), + "BS".to_string(), + None, + "semester".to_string(), + ); + degree.id = Some("bs-cs".to_string()); + degree.tags = Some(vec!["ai".to_string()]); + DegreeProgram { + degree, + requirements: HashMap::new(), + courses: HashMap::new(), + } + } + + #[test] + fn test_program_rollup_from_analysis_copies_identity_and_stats() { + let rollup = ProgramRollup::from_analysis(&sample_program(), &aggregator_with_two_plans()); + assert_eq!(rollup.id, "bs-cs"); + assert_eq!(rollup.name, "Computer Science"); + assert_eq!(rollup.tags.as_deref(), Some(&["ai".to_string()][..])); + assert_eq!(rollup.variations_run, 2); + // per-plan total complexity = 10 then 20 -> mean 15. + assert!((rollup.complexity.mean - 15.0).abs() < 1e-9); + } + + #[test] + fn test_export_school_report_json_writes_sanitized_file_with_metrics() { + let dir = tempfile::tempdir().unwrap(); + let rollups = vec![ProgramRollup::from_analysis( + &sample_program(), + &aggregator_with_two_plans(), + )]; + let path = export_school_report_json("Khoury College", &rollups, dir.path()).unwrap(); + + assert_eq!( + path.file_name().unwrap().to_str().unwrap(), + "Khoury_College_school_report.json" + ); + let v: Value = serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap(); + assert_eq!(v["school"], "Khoury College"); + assert_eq!(v["program_count"], 1); + assert!(v["school_metrics"]["complexity"].is_object()); + assert_eq!(v["programs"].as_array().unwrap().len(), 1); + } + + #[test] + fn test_export_school_report_json_empty_programs_null_metrics() { + let dir = tempfile::tempdir().unwrap(); + let path = export_school_report_json("Empty School", &[], dir.path()).unwrap(); + let v: Value = serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap(); + assert_eq!(v["program_count"], 0); + assert!(v["school_metrics"]["complexity"].is_null()); + assert!(v["school_metrics"]["delay"].is_null()); + assert!(v["school_metrics"]["credits"].is_null()); + } +} diff --git a/src/core/statistics/aggregator.rs b/src/core/statistics/aggregator.rs index 4e0d8cb..fcda4b6 100644 --- a/src/core/statistics/aggregator.rs +++ b/src/core/statistics/aggregator.rs @@ -102,7 +102,7 @@ impl Default for CourseAggregator { } /// Aggregated statistics for a single course across all plans -#[derive(Debug, Clone)] +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct AggregatedCourseStats { /// Course identifier pub course_id: String, @@ -119,7 +119,7 @@ pub struct AggregatedCourseStats { } /// Statistics for a single metric -#[derive(Debug, Clone)] +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct MetricStats { /// Minimum value pub min: f64, @@ -200,7 +200,7 @@ impl MetricStats { } /// Degree-level aggregated statistics -#[derive(Debug, Clone)] +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct AggregatedDegreeStats { /// Total plans processed pub plan_count: usize, diff --git a/src/core/statistics/streaming.rs b/src/core/statistics/streaming.rs index c816091..3bb210f 100644 --- a/src/core/statistics/streaming.rs +++ b/src/core/statistics/streaming.rs @@ -56,7 +56,7 @@ impl WelfordAccumulator { let delta = value - self.mean; self.mean += delta / self.count as f64; let delta2 = value - self.mean; - self.m2 += delta * delta2; + self.m2 = delta.mul_add(delta2, self.m2); } /// Get the current count diff --git a/src/mcp/server.rs b/src/mcp/server.rs index 0d9a6ac..b698b76 100644 --- a/src/mcp/server.rs +++ b/src/mcp/server.rs @@ -6,12 +6,12 @@ use std::sync::Arc; use crate::core::config::DatabaseConfig; use crate::mcp::tools::{ - analyze, audit, cache, course_detail, match_courses, pipeline, plan_graph, report, samples, - schema, shared, trim, validate, visualize, AnalyzeDegreeRequest, AuditDegreeRequest, - CacheYamlRequest, DegreePipelineRequest, FindCoursesMatchingRequest, - GenerateDegreeReportRequest, GetCourseDetailRequest, GetCurriculumVisualizationRequest, - GetSchemaRequest, ListSampleDegreesRequest, RenderPlanGraphRequest, TrimDegreeRequest, - ValidateDegreeRequest, + analyze, audit, cache, convert, course_detail, json_schema, match_courses, pipeline, + plan_graph, report, samples, schema, shared, trim, validate, visualize, AnalyzeDegreeRequest, + AuditDegreeRequest, CacheYamlRequest, ConvertDegreeRequest, DegreePipelineRequest, + FindCoursesMatchingRequest, GenerateDegreeReportRequest, GetCourseDetailRequest, + GetCurriculumVisualizationRequest, GetDegreeJsonSchemaRequest, GetSchemaRequest, + ListSampleDegreesRequest, RenderPlanGraphRequest, TrimDegreeRequest, ValidateDegreeRequest, }; use rmcp::{ handler::server::{router::tool::ToolRouter, wrapper::Parameters}, @@ -89,6 +89,32 @@ impl NuAnalyticsMcpServer { schema::execute(req.section.as_deref()) } + /// Return the unified-degree JSON Schema (machine-validatable) + #[tool( + description = "Return the machine-validatable JSON Schema (2020-12) for the unified degree format — the structure convert_degree produces and that analyze_degree / validate_degree / trim_degree accept. Use it to validate a unified degree document or to understand the format, including wildcard `from` pools (e.g. pattern \"CS:2500+\" or \"*:*\"). Distinct from get_degree_schema, which returns the human-readable YAML reference." + )] + #[allow(clippy::unused_self)] + fn get_degree_json_schema( + &self, + Parameters(_req): Parameters, + ) -> String { + json_schema::execute() + } + + /// Convert an ai-landscape program JSON to the unified degree JSON + #[tool( + description = "Convert an ai-landscape program JSON into the unified NuAnalytics degree JSON (the same format analyze_degree / validate_degree accept). Provide exactly ONE source: json_content (inline) or json_path (file on the server). For a multi-program cluster pipeline file, omit `program` to get the program inventory, or set `program` to convert one of them. Returns the unified JSON plus any conversion_warnings and a cache: handle you can pass as `degree_id` to validate_degree / analyze_degree without re-pasting." + )] + #[allow(clippy::unused_self)] + fn convert_degree(&self, Parameters(req): Parameters) -> String { + convert::execute_json( + req.json_content, + req.json_path, + req.program.as_deref(), + req.pretty.unwrap_or(true), + ) + } + /// Validate a degree program YAML #[tool( description = "Validate a degree program YAML. Returns detailed validation results including errors, warnings, and suggestions. Provide exactly ONE YAML source: yaml_content (inline string), yaml_path (file on the server's filesystem), or degree_id (DB lookup). Pass allow_unmatched_patterns=true to surface external gen-ed pool patterns (e.g. \"*:100+\") as warnings instead of errors when courses aren't enumerated locally." diff --git a/src/mcp/tools/analyze.rs b/src/mcp/tools/analyze.rs index e3353b7..7cd6268 100644 --- a/src/mcp/tools/analyze.rs +++ b/src/mcp/tools/analyze.rs @@ -9,7 +9,7 @@ //! without duplicating ~50 lines of orchestration. use crate::core::degree::{ - parse_degree_yaml, DegreeParseError, PlanGenerationStats, PlanGenerator, PlanGeneratorConfig, + parse_degree_auto, DegreeParseError, PlanGenerationStats, PlanGenerator, PlanGeneratorConfig, PlanSelector, PlanSelectorConfig, PlanVariant, SamplingStrategy, SelectedPlans, }; use crate::core::metrics::compute_all_metrics; @@ -505,7 +505,10 @@ pub(crate) fn build_artifacts( .clamp(MIN_ANALYSIS_TIMEOUT_SECS, MAX_ANALYSIS_TIMEOUT_SECS); let deadline = Some(Instant::now() + Duration::from_secs(timeout_secs)); - let program = parse_degree_yaml(yaml_content).map_err(|e| format_parse_error(&e))?; + // Accept YAML or unified/ai-landscape JSON (auto-detected). Conversion + // warnings are surfaced by validate_degree / convert_degree, not here. + let (program, _conversion_warnings) = + parse_degree_auto(yaml_content).map_err(|e| format_parse_error(&e))?; let school = build_school(&program); let mut graph_result = CourseGraph::from_degree_program(&program); @@ -1014,10 +1017,7 @@ pub fn execute_json( // ============================================================================ fn format_parse_error(e: &DegreeParseError) -> String { - match e { - DegreeParseError::IoError(msg) => format!("File error: {msg}"), - DegreeParseError::YamlError { message, .. } => format!("YAML syntax error: {message}"), - } + crate::mcp::tools::shared::format_degree_parse_error(e) } pub(super) const fn metric_stats_json(s: &MetricStats) -> MetricStatsJson { diff --git a/src/mcp/tools/audit.rs b/src/mcp/tools/audit.rs index 29111ce..bf180d1 100644 --- a/src/mcp/tools/audit.rs +++ b/src/mcp/tools/audit.rs @@ -8,7 +8,7 @@ use crate::core::degree::audit::{ detect_lowest_course_level, find_deep_chains, find_missing_intermediate_prereqs, find_upper_level_without_prereqs, MissingIntermediateFinding, }; -use crate::core::degree::{parse_degree_yaml, DegreeParseError}; +use crate::core::degree::{parse_degree_auto, DegreeParseError}; use crate::core::models::CourseGraph; use crate::core::validate_degree_program; use crate::core::DegreeProgram; @@ -198,9 +198,9 @@ pub fn execute( ) -> AuditResponse { let threshold = chain_threshold.unwrap_or(DEFAULT_CHAIN_THRESHOLD); - // Try to parse the YAML - let program = match parse_degree_yaml(yaml_content) { - Ok(p) => p, + // Parse the degree (YAML or unified/ai-landscape JSON, auto-detected). + let program = match parse_degree_auto(yaml_content) { + Ok((p, _warnings)) => p, Err(e) => { return AuditResponse { passed: false, @@ -360,10 +360,7 @@ pub fn execute_json( // ============================================================================ fn format_parse_error(e: &DegreeParseError) -> String { - match e { - DegreeParseError::IoError(msg) => format!("File error: {msg}"), - DegreeParseError::YamlError { message, .. } => format!("YAML syntax error: {message}"), - } + crate::mcp::tools::shared::format_degree_parse_error(e) } /// Tag a missing-prereq finding as internal (subject in `major_subjects`), diff --git a/src/mcp/tools/convert.rs b/src/mcp/tools/convert.rs new file mode 100644 index 0000000..7680b24 --- /dev/null +++ b/src/mcp/tools/convert.rs @@ -0,0 +1,446 @@ +//! ai-landscape → unified degree JSON conversion tool. +//! +//! Provides the `convert_degree` MCP tool: turn an ai-landscape program JSON +//! into the unified `NuAnalytics` degree JSON (the same format the YAML degrees +//! produce), surfacing conversion warnings. A cluster pipeline file (many +//! programs) is inventoried; pass `program` to convert one out of it. + +use rmcp::schemars; +use serde::{Deserialize, Serialize}; + +use crate::core::degree::{ + convert_landscape, extract_cluster_programs, parse_degree_json_with_warnings, to_unified_value, + unified_value_to_string, +}; +use crate::mcp::cache::YAML_CACHE; +use crate::mcp::tools::shared::{ + deserialize_opt_bool, read_yaml_file, to_json_pretty, ToolFollowup, TOOL_ANALYZE_DEGREE, + TOOL_VALIDATE_DEGREE, +}; + +/// Request parameters for the `convert_degree` tool. +#[derive(Debug, Deserialize, schemars::JsonSchema)] +pub struct ConvertDegreeRequest { + /// Inline ai-landscape program JSON (or an already-unified degree JSON, + /// which is normalized through). Provide exactly one of this or `json_path`. + #[schemars(description = "Inline ai-landscape (or unified) program JSON")] + pub json_content: Option, + /// Path to an ai-landscape JSON file on the MCP server's filesystem. + #[schemars(description = "Path to an ai-landscape JSON file on the server")] + pub json_path: Option, + /// When the input is a multi-program *cluster* pipeline file, the program + /// name to convert. Omit to get the cluster's program inventory instead. + #[schemars(description = "Program name to convert out of a cluster file")] + pub program: Option, + /// Pretty-print the unified JSON (default true). + #[serde(default, deserialize_with = "deserialize_opt_bool")] + #[schemars(description = "Pretty-print the unified JSON output (default true)")] + pub pretty: Option, +} + +/// One program entry in a cluster file's inventory. +#[derive(Debug, Serialize)] +pub struct ClusterProgramInfo { + /// Program name (the cluster key; pass back as `program`). + pub name: String, + /// Institution the program belongs to. + pub university: String, + /// Degree title as scraped. + pub degree: String, +} + +/// Response from `convert_degree`. +#[derive(Debug, Serialize)] +pub struct ConvertResponse { + /// Whether conversion (or inventory) succeeded. + pub success: bool, + /// Error message when `success` is false. + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + /// Shape of the input: `"single"` program or `"cluster"` pipeline file. + pub kind: &'static str, + /// Number of programs found in the input (1 for a single program). + pub program_count: usize, + /// The converted unified degree JSON (absent for a cluster inventory). + #[serde(skip_serializing_if = "Option::is_none")] + pub unified_json: Option, + /// Data-quality warnings raised during conversion (defaulted credits, etc.). + #[serde(skip_serializing_if = "Vec::is_empty")] + pub conversion_warnings: Vec, + /// Inventory of programs (populated for a cluster file with no `program`). + #[serde(skip_serializing_if = "Vec::is_empty")] + pub programs: Vec, + /// Cache handle for the converted JSON; pass as `degree_id` to chain into + /// `validate_degree` / `analyze_degree` without re-pasting. + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_id: Option, + /// Free-text guidance (e.g. how to convert a cluster program). + #[serde(skip_serializing_if = "Option::is_none")] + pub note: Option, + /// Suggested next MCP calls. + pub tool_followups: Vec, +} + +impl ConvertResponse { + /// Build a failed response carrying `msg` as the error. + fn error(msg: impl std::fmt::Display) -> Self { + Self { + success: false, + error: Some(msg.to_string()), + kind: "single", + program_count: 0, + unified_json: None, + conversion_warnings: Vec::new(), + programs: Vec::new(), + cache_id: None, + note: None, + tool_followups: Vec::new(), + } + } +} + +/// Build a converted-degree response from a unified JSON string + warnings. +fn converted( + kind: &'static str, + program_count: usize, + unified: String, + warnings: Vec, +) -> ConvertResponse { + // Cache the unified body so callers can chain by `degree_id` (the degree + // tools auto-detect JSON via parse_degree_auto). `.ok()`: a poisoned cache + // must not fail the conversion — we just omit the chaining handle. + let cache_id = YAML_CACHE + .lock() + .ok() + .map(|mut cache| cache.insert(unified.clone())); + let followups = cache_id.as_ref().map_or_else(Vec::new, |id| { + vec![ + ToolFollowup { + tool: TOOL_VALIDATE_DEGREE, + reason: "Validate the converted degree before analyzing.".to_string(), + suggested_args: serde_json::json!({ "degree_id": id }), + }, + ToolFollowup { + tool: TOOL_ANALYZE_DEGREE, + reason: "Analyze the converted degree (plans + metrics).".to_string(), + suggested_args: serde_json::json!({ "degree_id": id }), + }, + ] + }); + ConvertResponse { + success: true, + error: None, + kind, + program_count, + unified_json: Some(unified), + conversion_warnings: warnings, + programs: Vec::new(), + cache_id, + note: None, + tool_followups: followups, + } +} + +/// Serialize a degree program to the degree-first unified JSON string, mapping +/// any serializer failure to a `DegreeParseError`. Shared by the single-program +/// and cluster paths. +fn to_unified_json( + program: &crate::core::DegreeProgram, + pretty: bool, +) -> Result { + let value = to_unified_value(program)?; + unified_value_to_string(&value, pretty).map_err(|e| { + crate::core::degree::DegreeParseError::json_message(format!( + "Failed to serialize unified JSON: {e}" + )) + }) +} + +/// Convert an ai-landscape (or unified) program JSON to the unified format. +/// +/// Resolves exactly one of `json_content` / `json_path`, then: for a cluster +/// pipeline file, returns the program inventory (or converts the one named by +/// `program`); for a single program, auto-converts (ai-landscape) or normalizes +/// (already-unified) to the unified JSON. +#[must_use] +pub fn execute( + json_content: Option, + json_path: Option, + program: Option<&str>, + pretty: bool, +) -> ConvertResponse { + let content = match (json_content, json_path) { + (Some(_), Some(_)) => { + return ConvertResponse::error( + "Provide exactly one of json_content or json_path (not both)", + ) + } + (None, None) => { + return ConvertResponse::error("Must provide exactly one of: json_content or json_path") + } + (Some(c), None) => c, + (None, Some(p)) => match read_yaml_file(&p) { + Ok(c) => c, + Err(e) => return ConvertResponse::error(e), + }, + }; + + let value: serde_json::Value = match serde_json::from_str(&content) { + Ok(v) => v, + Err(e) => return ConvertResponse::error(format!("Invalid JSON: {e}")), + }; + + // Cluster pipeline file → inventory or single-program extraction. + if let Some(programs) = extract_cluster_programs(&value) { + return convert_cluster(&programs, program, pretty); + } + + // Single program (raw ai-landscape or already-unified). + match parse_degree_json_with_warnings(&content) { + Ok((prog, warnings)) => match to_unified_json(&prog, pretty) { + Ok(unified) => converted("single", 1, unified, warnings), + Err(e) => ConvertResponse::error(e), + }, + Err(e) => ConvertResponse::error(e), + } +} + +/// Handle a cluster pipeline file's already-extracted programs. +/// +/// With no `program` selector, returns a bounded inventory of every program +/// (name + institution + degree) instead of dumping them all. With a selector, +/// converts that one program (exact match, then case-insensitive fallback) and +/// errors with the available names if none match. +fn convert_cluster( + programs: &[(String, crate::core::degree::LandscapeProgram)], + program: Option<&str>, + pretty: bool, +) -> ConvertResponse { + let Some(sel) = program else { + // No selector → return the inventory. + let inventory: Vec = programs + .iter() + .map(|(name, prog)| ClusterProgramInfo { + name: name.clone(), + university: prog.university.clone(), + degree: prog.degree.clone(), + }) + .collect(); + return ConvertResponse { + success: true, + error: None, + kind: "cluster", + program_count: inventory.len(), + unified_json: None, + conversion_warnings: Vec::new(), + programs: inventory, + cache_id: None, + note: Some( + "Multi-program cluster file. Re-call with `program` set to one of the listed \ + names to convert it, or use the CLI `degree convert` to expand all programs to \ + files." + .to_string(), + ), + tool_followups: Vec::new(), + }; + }; + + let Some((_, prog)) = programs.iter().find(|(name, _)| name == sel).or_else(|| { + programs + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case(sel)) + }) else { + let available: Vec<&str> = programs.iter().map(|(n, _)| n.as_str()).collect(); + return ConvertResponse::error(format!( + "Program {sel:?} not found in cluster. Available: {}", + available.join(", ") + )); + }; + + let result = convert_landscape(prog); + match to_unified_json(&result.program, pretty) { + Ok(unified) => converted("cluster", programs.len(), unified, result.warnings), + Err(e) => ConvertResponse::error(e), + } +} + +/// Execute `convert_degree` and serialize the response to a JSON string. +#[must_use] +pub fn execute_json( + json_content: Option, + json_path: Option, + program: Option<&str>, + pretty: bool, +) -> String { + to_json_pretty(&execute(json_content, json_path, program, pretty)) +} + +#[cfg(test)] +mod tests { + use super::*; + + const AI_LANDSCAPE: &str = r#"{ + "university": "Test University", + "degree": "BS in Computer Science", + "ai_program": null, + "courses": { + "cs_course_core": [ + {"course_code": "CS 101", "title": "Intro", "course_hours": "3", + "picklist": [], "prerequisites": [], "corequisites": [], "strict_corequisites": []} + ] + } + }"#; + + fn first_arg(content: &str) -> ConvertResponse { + execute(Some(content.to_string()), None, None, true) + } + + #[test] + fn convert_single_ai_landscape_returns_unified_json() { + let r = first_arg(AI_LANDSCAPE); + assert!(r.success, "expected success, got {:?}", r.error); + assert_eq!(r.kind, "single"); + assert_eq!(r.program_count, 1); + let unified = r.unified_json.expect("unified json present"); + // Degree-first, parses as a unified degree. + let v: serde_json::Value = serde_json::from_str(&unified).unwrap(); + assert!( + v.get("degree").is_some(), + "unified output has a degree block" + ); + assert!(v.get("courses").is_some()); + assert!( + r.cache_id.is_some(), + "converted body is cached for chaining" + ); + } + + #[test] + fn convert_already_unified_passes_through() { + let unified_in = r#"{"degree":{"name":"X","degree_type":"BS","system_type":"semester"}, + "requirements":{"core":{"type":"all","courses":["CS101"]}}, + "courses":{"CS101":{"name":"Intro","prefix":"CS","number":"101","credit_hours":3.0}}}"#; + let r = first_arg(unified_in); + assert!( + r.success, + "unified input should normalize through: {:?}", + r.error + ); + assert!(r.unified_json.unwrap().contains("\"degree\"")); + } + + #[test] + fn convert_malformed_json_reports_error() { + let r = first_arg("{ not valid json"); + assert!(!r.success); + assert!(r.error.unwrap().contains("Invalid JSON")); + } + + #[test] + fn convert_requires_exactly_one_source() { + let both = execute( + Some("{}".to_string()), + Some("/x.json".to_string()), + None, + true, + ); + assert!(!both.success); + let neither = execute(None, None, None, true); + assert!(!neither.success); + } + + #[test] + fn convert_cluster_without_program_returns_inventory() { + let cluster = r#"{ + "course_verifier": { + "Computer Science BS": { "results": { + "university": "Test U", "degree": "BS CS", + "courses": {"cs_course_core": [ + {"course_code":"CS 101","title":"Intro","course_hours":"3", + "picklist":[],"prerequisites":[],"corequisites":[],"strict_corequisites":[]} + ]} + }} + } + }"#; + let r = execute(Some(cluster.to_string()), None, None, true); + assert!(r.success); + assert_eq!(r.kind, "cluster"); + assert!(!r.programs.is_empty(), "inventory should list the program"); + assert!(r.unified_json.is_none(), "no single program selected"); + assert!(r.note.is_some()); + } + + #[test] + fn convert_cluster_with_program_converts_it() { + let cluster = r#"{ + "course_verifier": { + "Computer Science BS": { "results": { + "university": "Test U", "degree": "BS CS", + "courses": {"cs_course_core": [ + {"course_code":"CS 101","title":"Intro","course_hours":"3", + "picklist":[],"prerequisites":[],"corequisites":[],"strict_corequisites":[]} + ]} + }} + } + }"#; + let r = execute( + Some(cluster.to_string()), + None, + Some("Computer Science BS"), + true, + ); + assert!(r.success, "expected conversion, got {:?}", r.error); + assert_eq!(r.kind, "cluster"); + assert!(r.unified_json.unwrap().contains("\"degree\"")); + } + + #[test] + fn execute_json_emits_parseable_json() { + let s = execute_json(Some(AI_LANDSCAPE.to_string()), None, None, true); + let v: serde_json::Value = serde_json::from_str(&s).unwrap(); + assert_eq!(v["success"], serde_json::json!(true)); + } + + #[test] + fn convert_json_path_nonexistent_file_surfaces_read_error() { + let r = execute( + None, + Some("/nonexistent/nuanalytics/degree.json".to_string()), + None, + true, + ); + assert!(!r.success); + assert!( + r.error.unwrap().contains("Failed to read"), + "missing-file read error should be surfaced" + ); + } + + #[test] + fn convert_cluster_program_not_found_lists_available() { + let cluster = r#"{ + "course_verifier": { + "Computer Science BS": { "results": { + "university": "Test U", "degree": "BS CS", + "courses": {"cs_course_core": [ + {"course_code":"CS 101","title":"Intro","course_hours":"3", + "picklist":[],"prerequisites":[],"corequisites":[],"strict_corequisites":[]} + ]} + }} + } + }"#; + let r = execute( + Some(cluster.to_string()), + None, + Some("No Such Program"), + true, + ); + assert!(!r.success); + let err = r.error.unwrap(); + assert!(err.contains("not found")); + assert!( + err.contains("Computer Science BS"), + "available names listed: {err}" + ); + } +} diff --git a/src/mcp/tools/course_detail.rs b/src/mcp/tools/course_detail.rs index 87f4bbf..a175ebc 100644 --- a/src/mcp/tools/course_detail.rs +++ b/src/mcp/tools/course_detail.rs @@ -11,7 +11,7 @@ //! true (default). use crate::core::degree::audit::extract_course_level; -use crate::core::degree::{parse_degree_yaml, DegreeParseError}; +use crate::core::degree::{parse_degree_auto, DegreeParseError}; use crate::core::models::CourseGraph; use crate::core::DegreeProgram; use crate::mcp::tools::analyze::{metric_stats_json, AnalysisArtifacts, MetricStatsJson}; @@ -153,8 +153,8 @@ pub fn execute( include_analysis: bool, max_plans: Option, ) -> CourseDetailResponse { - let program = match parse_degree_yaml(yaml_content) { - Ok(p) => p, + let program = match parse_degree_auto(yaml_content) { + Ok((p, _warnings)) => p, Err(e) => return error_response(course_id, format_parse_error(&e)), }; @@ -383,10 +383,7 @@ fn error_response(course_id: &str, error: impl Into) -> CourseDetailResp } fn format_parse_error(e: &DegreeParseError) -> String { - match e { - DegreeParseError::IoError(msg) => format!("File error: {msg}"), - DegreeParseError::YamlError { message, .. } => format!("YAML syntax error: {message}"), - } + crate::mcp::tools::shared::format_degree_parse_error(e) } // ============================================================================ diff --git a/src/mcp/tools/json_schema.rs b/src/mcp/tools/json_schema.rs new file mode 100644 index 0000000..5b256ec --- /dev/null +++ b/src/mcp/tools/json_schema.rs @@ -0,0 +1,68 @@ +//! Unified-degree JSON Schema tool. +//! +//! Provides the `get_degree_json_schema` MCP tool, which returns the +//! machine-validatable JSON Schema for the unified degree format — the same +//! checked-in `degree.schema.json` asset the CLI's `degree schema` command +//! emits. Use it to validate a converted/authored unified degree, or to +//! understand the format (including wildcard `from` pools) programmatically. +//! +//! This is distinct from `get_degree_schema`, which serves the human-readable +//! YAML reference rather than a machine schema. + +use rmcp::schemars; +use serde::Deserialize; + +/// The unified-degree JSON Schema, embedded at compile time so it ships with +/// the binary (single source of truth shared with the CLI). +const UNIFIED_SCHEMA: &str = include_str!("../../assets/degree.schema.json"); + +/// Request parameters for the `get_degree_json_schema` tool (none). +#[derive(Debug, Deserialize, schemars::JsonSchema)] +pub struct GetDegreeJsonSchemaRequest {} + +/// Return the unified-degree JSON Schema (JSON Schema 2020-12) as a string, +/// ready to hand to a validator. +#[must_use] +pub fn execute() -> String { + UNIFIED_SCHEMA.to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn schema_is_valid_json_with_unified_shape() { + let schema = execute(); + let v: serde_json::Value = serde_json::from_str(&schema).expect("schema is valid JSON"); + // The unified degree requires these three top-level keys. + let names: Vec<&str> = v + .get("required") + .and_then(serde_json::Value::as_array) + .expect("required array") + .iter() + .filter_map(serde_json::Value::as_str) + .collect(); + assert!(names.contains(&"degree")); + assert!(names.contains(&"requirements")); + assert!(names.contains(&"courses")); + } + + #[test] + fn schema_documents_wildcard_from_pools() { + // The fix that prompted this tool: `from` must document pattern/courses + // so wildcard pools are discoverable, not an untyped object. + let v: serde_json::Value = serde_json::from_str(&execute()).unwrap(); + let from_clause = v + .get("$defs") + .and_then(|d| d.get("fromClause")) + .expect("fromClause def present"); + let props = from_clause + .get("properties") + .and_then(serde_json::Value::as_object) + .expect("fromClause properties"); + assert!(props.contains_key("courses")); + assert!(props.contains_key("pattern")); + assert!(props.contains_key("include")); + } +} diff --git a/src/mcp/tools/match_courses.rs b/src/mcp/tools/match_courses.rs index 9612245..8124f88 100644 --- a/src/mcp/tools/match_courses.rs +++ b/src/mcp/tools/match_courses.rs @@ -186,10 +186,7 @@ fn error_response( } fn format_parse_error(e: &DegreeParseError) -> String { - match e { - DegreeParseError::IoError(msg) => format!("File error: {msg}"), - DegreeParseError::YamlError { message, .. } => format!("YAML syntax error: {message}"), - } + crate::mcp::tools::shared::format_degree_parse_error(e) } // ============================================================================ diff --git a/src/mcp/tools/mod.rs b/src/mcp/tools/mod.rs index 287ff5a..3235c67 100644 --- a/src/mcp/tools/mod.rs +++ b/src/mcp/tools/mod.rs @@ -5,7 +5,9 @@ pub mod analyze; pub mod audit; pub mod cache; +pub mod convert; pub mod course_detail; +pub mod json_schema; pub mod match_courses; pub mod pipeline; pub mod plan_graph; @@ -35,7 +37,9 @@ pub mod scaffold; pub use analyze::AnalyzeDegreeRequest; pub use audit::AuditDegreeRequest; pub use cache::CacheYamlRequest; +pub use convert::ConvertDegreeRequest; pub use course_detail::GetCourseDetailRequest; +pub use json_schema::GetDegreeJsonSchemaRequest; pub use match_courses::FindCoursesMatchingRequest; pub use pipeline::DegreePipelineRequest; pub use plan_graph::RenderPlanGraphRequest; diff --git a/src/mcp/tools/shared.rs b/src/mcp/tools/shared.rs index a7b71a5..de2a179 100644 --- a/src/mcp/tools/shared.rs +++ b/src/mcp/tools/shared.rs @@ -30,6 +30,7 @@ pub fn format_degree_parse_error(e: &DegreeParseError) -> String { (Some(l), Some(c)) => format!("YAML syntax error at line {l} column {c}: {message}"), _ => format!("YAML syntax error: {message}"), }, + DegreeParseError::JsonError(msg) => format!("JSON syntax error: {msg}"), } } diff --git a/src/mcp/tools/trim.rs b/src/mcp/tools/trim.rs index 6d53b23..3e6d2f5 100644 --- a/src/mcp/tools/trim.rs +++ b/src/mcp/tools/trim.rs @@ -12,7 +12,7 @@ use serde::{Deserialize, Serialize}; use std::collections::HashSet; use crate::core::degree::{ - parse_degree_yaml, save_degree_to_yaml, serialize_degree_yaml, trim_program, DegreeParseError, + parse_degree_auto, save_degree_to_yaml, serialize_degree_yaml, trim_program, DegreeParseError, TrimOptions, TrimReport, }; use crate::mcp::cache::YAML_CACHE; @@ -162,8 +162,8 @@ pub fn execute( output_path: Option<&str>, source_path: Option<&str>, ) -> TrimResponse { - let program = match parse_degree_yaml(yaml_content) { - Ok(p) => p, + let program = match parse_degree_auto(yaml_content) { + Ok((p, _warnings)) => p, Err(e) => return parse_error_response(&e, yaml_content), }; @@ -269,7 +269,7 @@ const fn empty_response() -> TrimResponse { fn parse_error_response(e: &DegreeParseError, yaml: &str) -> TrimResponse { let (line, column) = match e { DegreeParseError::YamlError { line, column, .. } => (*line, *column), - DegreeParseError::IoError(_) => (None, None), + DegreeParseError::IoError(_) | DegreeParseError::JsonError(_) => (None, None), }; let context = match (line, column) { (Some(l), Some(c)) => Some(format_yaml_context(yaml, l, c)), diff --git a/src/mcp/tools/validate.rs b/src/mcp/tools/validate.rs index dff313d..605b451 100644 --- a/src/mcp/tools/validate.rs +++ b/src/mcp/tools/validate.rs @@ -4,7 +4,7 @@ //! and returns structured feedback. use crate::core::degree::audit::{detect_lowest_course_level, find_upper_level_without_prereqs}; -use crate::core::degree::{parse_degree_yaml, DegreeParseError, RequirementResolver}; +use crate::core::degree::{parse_degree_auto, DegreeParseError, RequirementResolver}; use crate::core::models::degree::{FromClause, Requirement, RequirementType}; use crate::core::models::CourseGraph; use crate::core::{ @@ -136,6 +136,11 @@ pub struct ValidationResponse { pub errors: Vec, /// List of validation warnings pub warnings: Vec, + /// Warnings emitted while converting a non-unified input (e.g. an + /// ai-landscape JSON body): defaulted credit hours, collapsed corequisite + /// choices, etc. Empty when the input was already YAML / unified JSON. + #[serde(skip_serializing_if = "Vec::is_empty")] + pub conversion_warnings: Vec, /// Context about what's defined in the degree pub context: Option, /// Pool resolution for every requirement that uses a `from` clause. @@ -188,13 +193,13 @@ pub fn execute( allow_unmatched_patterns: bool, include_hidden_prereq_warnings: bool, ) -> ValidationResponse { - // Try to parse the YAML - let program = match parse_degree_yaml(yaml_content) { - Ok(p) => p, + // Parse the degree (YAML or unified/ai-landscape JSON, auto-detected). + let (program, conversion_warnings) = match parse_degree_auto(yaml_content) { + Ok((p, w)) => (p, w), Err(e) => { let (line, column) = match &e { DegreeParseError::YamlError { line, column, .. } => (*line, *column), - DegreeParseError::IoError(_) => (None, None), + DegreeParseError::IoError(_) | DegreeParseError::JsonError(_) => (None, None), }; let context = match (line, column) { (Some(l), Some(c)) => Some(format_yaml_context(yaml_content, l, c)), @@ -202,6 +207,7 @@ pub fn execute( }; return ValidationResponse { is_valid: false, + conversion_warnings: vec![], parse_error: Some(format_degree_parse_error(&e)), parse_error_line: line, parse_error_column: column, @@ -268,6 +274,7 @@ pub fn execute( ValidationResponse { is_valid: result.is_valid, + conversion_warnings, parse_error: None, parse_error_line: None, parse_error_column: None, @@ -686,6 +693,41 @@ const fn requirement_type_label(req_type: &RequirementType) -> &'static str { mod tests { use super::*; + #[test] + fn validate_accepts_unified_json_content() { + // Proves the tool now accepts unified-JSON bodies, not just YAML. + let json = r#"{"degree":{"name":"J","degree_type":"BS","system_type":"semester"}, + "requirements":{"core":{"type":"all","courses":["CS101"]}}, + "courses":{"CS101":{"name":"Intro","prefix":"CS","number":"101","credit_hours":3.0}}}"#; + let r = execute(json, false, true); + assert!( + r.parse_error.is_none(), + "unified-JSON content must parse: {:?}", + r.parse_error + ); + assert!( + r.conversion_warnings.is_empty(), + "already-unified input needs no conversion" + ); + } + + #[test] + fn validate_accepts_ai_landscape_json_and_surfaces_warnings() { + let land = r#"{"university":"U","degree":"BS in CS","ai_program":null, + "courses":{"cs_course_core":[{"course_code":"CS 101","title":"Intro", + "picklist":[],"prerequisites":[],"corequisites":[],"strict_corequisites":[]}]}}"#; + let r = execute(land, false, true); + assert!( + r.parse_error.is_none(), + "raw ai-landscape JSON must auto-convert on validate: {:?}", + r.parse_error + ); + assert!( + !r.conversion_warnings.is_empty(), + "defaulted credit hours should surface as a conversion warning" + ); + } + const VALID_YAML: &str = r#" degree: id: test-degree diff --git a/tests/rs/cli_degree.rs b/tests/rs/cli_degree.rs index be816ec..d88a5ad 100644 --- a/tests/rs/cli_degree.rs +++ b/tests/rs/cli_degree.rs @@ -674,8 +674,8 @@ fn test_degree_trim_skips_non_yaml_and_proceeds() { ); let stderr = String::from_utf8_lossy(&output.stderr); assert!( - stderr.contains("Skipping non-YAML file") && stderr.contains("Readme.md"), - "stderr should warn about the non-YAML input; got: {stderr}" + stderr.contains("Skipping non-degree") && stderr.contains("Readme.md"), + "stderr should warn about the non-degree input; got: {stderr}" ); assert!( temp_dir @@ -687,9 +687,9 @@ fn test_degree_trim_skips_non_yaml_and_proceeds() { } /// All-invalid input list: every file is filtered out, command fails with -/// the dedicated "no YAML files to process" error. +/// the dedicated "no degree files to process" error. #[test] -fn test_degree_trim_rejects_all_non_yaml_inputs() { +fn test_degree_trim_rejects_all_non_degree_inputs() { let output = Command::new("cargo") .args(["run", "--", "degree", "trim", "Readme.md", "Cargo.toml"]) .output() @@ -697,15 +697,130 @@ fn test_degree_trim_rejects_all_non_yaml_inputs() { assert!( !output.status.success(), - "trim must fail when no YAML inputs survive filtering" + "trim must fail when no degree inputs survive filtering" ); let stderr = String::from_utf8_lossy(&output.stderr); assert!( - stderr.contains("No YAML files to process after filtering"), + stderr.contains("No degree files to process after filtering"), "stderr should explain the empty-after-filter state; got: {stderr}" ); } +/// Trim accepts a unified-JSON input and round-trips the format: the trimmed +/// file is written as JSON (not YAML), preserving the `.json` extension. +#[test] +fn test_degree_trim_accepts_json_and_writes_json() { + let dir = tempfile::tempdir().expect("tempdir"); + let input = dir.path().join("prog.unified.json"); + write_min_degree(&input, "json-trim"); + let out = dir.path().join("out"); + + let output = Command::new(env!("CARGO_BIN_EXE_nuanalytics")) + .args(["degree", "trim"]) + .arg(&input) + .arg("-o") + .arg(format!("{}/", out.display())) + .output() + .expect("run trim on json"); + assert!( + output.status.success(), + "trim should accept JSON input. stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + + let trimmed = out.join("prog.unified_trimmed.json"); + assert!( + trimmed.exists(), + "trimmed JSON should be written at {trimmed:?}" + ); + let body = std::fs::read_to_string(&trimmed).unwrap(); + assert!( + body.trim_start().starts_with('{'), + "output must be JSON, not YAML; got: {}", + &body[..body.len().min(40)] + ); + assert!( + body.contains("\"degree\""), + "trimmed JSON keeps the degree block" + ); +} + +/// A mixed batch (one YAML + one JSON) processes both, and each output keeps +/// its input format: YAML→`.yaml`, JSON→`.json`. +#[test] +fn test_degree_trim_batch_mixed_yaml_and_json() { + let dir = tempfile::tempdir().expect("tempdir"); + let json_input = dir.path().join("prog.unified.json"); + write_min_degree(&json_input, "mixed-json"); + let out = dir.path().join("trimmed"); + + let output = Command::new(env!("CARGO_BIN_EXE_nuanalytics")) + .args([ + "degree", + "trim", + "samples/degrees/neu-khoury-bscs-boston.yaml", + ]) + .arg(&json_input) + .arg("-o") + .arg(format!("{}/", out.display())) + .output() + .expect("run trim on mixed batch"); + assert!( + output.status.success(), + "trim must process a mixed YAML+JSON batch. stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + + let yaml_out = out.join("neu-khoury-bscs-boston_trimmed.yaml"); + let json_out = out.join("prog.unified_trimmed.json"); + assert!( + yaml_out.exists(), + "YAML input should yield .yaml at {yaml_out:?}" + ); + assert!( + json_out.exists(), + "JSON input should yield .json at {json_out:?}" + ); + assert!( + std::fs::read_to_string(&json_out) + .unwrap() + .trim_start() + .starts_with('{'), + "JSON output must be JSON, not YAML" + ); + assert!( + std::fs::read_to_string(&yaml_out) + .unwrap() + .contains("degree:"), + "YAML output must stay YAML" + ); +} + +/// The overwrite guard applies to JSON inputs too: `-o ` is refused. +#[test] +fn test_degree_trim_refuses_to_overwrite_json_input() { + let dir = tempfile::tempdir().expect("tempdir"); + let input = dir.path().join("prog.unified.json"); + write_min_degree(&input, "json-self"); + + let output = Command::new(env!("CARGO_BIN_EXE_nuanalytics")) + .args(["degree", "trim"]) + .arg(&input) + .arg("-o") + .arg(&input) + .output() + .expect("run trim with -o == input"); + assert!( + !output.status.success(), + "trim must refuse to overwrite a JSON input in place" + ); + assert!( + String::from_utf8_lossy(&output.stderr).contains("refusing to overwrite"), + "stderr should explain the refusal; got: {}", + String::from_utf8_lossy(&output.stderr) + ); +} + /// Trailing-slash `-o` should auto-create the destination directory and /// drop the trimmed file inside it. #[test] @@ -800,3 +915,295 @@ fn test_degree_trim_keep_all_preserves_extra_subject() { "--keep-all MATH should preserve more MATH references (got default={default_count}, keep-math={keep_math_count})" ); } + +// --------------------------------------------------------------------------- +// degree convert (ai-landscape -> unified JSON) +// --------------------------------------------------------------------------- + +/// `degree convert` turns a raw ai-landscape program file into unified JSON +/// with structured prerequisites, then that unified file re-parses cleanly +/// (round-trip), including the AND/OR flip and structured prereq form. +#[test] +fn test_degree_convert_landscape_round_trip() { + let dir = tempfile::tempdir().expect("tempdir"); + let input = dir.path().join("prog.json"); + // outer list = AND, inner list = OR -> (CS101 | CS102) & CS103 + std::fs::write( + &input, + r#"{ + "university": "Test U", + "degree": "Bachelor of Science Computer Science", + "ai_program": "Minor", + "courses": { + "cs_course_core": [ + {"course_code":"CS 103","title":"DS","course_hours":"4", + "prerequisites":[["CS 101","CS 102"],["CS 103"]]} + ] + } + }"#, + ) + .unwrap(); + let out = dir.path().join("prog.unified.json"); + + let status = Command::new(env!("CARGO_BIN_EXE_nuanalytics")) + .args(["degree", "convert"]) + .arg(&input) + .arg("-o") + .arg(&out) + .arg("--pretty") + .output() + .expect("run convert"); + assert!(status.status.success(), "convert should succeed"); + + let text = std::fs::read_to_string(&out).expect("unified output written"); + // Structured tagged prereqs, not the raw string field. + assert!(text.contains("\"prerequisites\"")); + assert!(!text.contains("prerequisites_raw")); + assert!(text.contains("\"and\"") && text.contains("\"or\"")); + // Program tagged as AI. + assert!(text.contains("\"ai\"")); + + // The emitted unified JSON re-parses through the normal loader. + let program = nu_analytics::core::degree::load_degree_from_json(&out) + .expect("unified JSON should re-parse"); + assert!(program.courses.contains_key("CS103")); + // Flip preserved: top-level AND with an inner OR group. + assert_eq!( + program.courses["CS103"].prerequisites_raw.as_deref(), + Some("(CS101 | CS102) & CS103") + ); +} + +/// `degree convert` on a *cluster* pipeline file expands to one unified JSON +/// per program, named `__.unified.json` under -o. +/// Verifier wins over scraper; empty-course programs are skipped. +#[test] +fn test_degree_convert_cluster_expands_to_multiple_files() { + let dir = tempfile::tempdir().expect("tempdir"); + // File stem ("bigu") becomes the half of each output name. + let input = dir.path().join("bigu.json"); + std::fs::write( + &input, + r#"{ + "course_verifier": { + "CS BS": {"results": {"university":"Big U","degree":"BS CS","ai_program":"Major","courses":{ + "cs_course_core":[{"course_code":"CS 101","title":"Intro","course_hours":"4","prerequisites":[]}]}}}, + "AI Minor": {"results": {"university":"Big U","degree":"AI Minor","ai_program":"Minor","courses":{ + "ai_program_required_courses":[{"course_code":"AI 200","title":"AI","course_hours":"3","prerequisites":[["CS 101"]]}]}}}, + "Empty": {"results": {"university":"Big U","degree":"X","courses":{"cs_course_core":[]}}} + } + }"#, + ) + .unwrap(); + let out_dir = dir.path().join("converted"); + + let output = Command::new(env!("CARGO_BIN_EXE_nuanalytics")) + .args(["degree", "convert"]) + .arg(&input) + .arg("-o") + .arg(&out_dir) + .output() + .expect("run convert cluster"); + assert!( + output.status.success(), + "cluster convert should succeed. stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + + // safe_filename maps spaces to '_', so "CS BS" -> "CS_BS". + let cs = out_dir.join("bigu__CS_BS.unified.json"); + let ai = out_dir.join("bigu__AI_Minor.unified.json"); + assert!(cs.exists(), "expected {}", cs.display()); + assert!(ai.exists(), "expected {}", ai.display()); + assert!( + !out_dir.join("bigu__Empty.unified.json").exists(), + "empty-course program must be skipped" + ); + assert!( + String::from_utf8_lossy(&output.stdout).contains("2 program(s)"), + "should report 2 programs" + ); + + // Each emitted file re-parses through the normal loader. + let prog = nu_analytics::core::degree::load_degree_from_json(&ai) + .expect("cluster output should re-parse"); + assert_eq!( + prog.courses["AI200"].prerequisites_raw.as_deref(), + Some("CS101") + ); +} + +/// Valid-but-unrecognized JSON is skipped (not a hard failure) in a convert batch. +#[test] +fn test_degree_convert_skips_non_degree_json() { + let dir = tempfile::tempdir().expect("tempdir"); + let input = dir.path().join("checkpoint.json"); + std::fs::write(&input, r#"{"checkpoint": 7, "status": "running"}"#).unwrap(); + let output = Command::new(env!("CARGO_BIN_EXE_nuanalytics")) + .args(["degree", "convert"]) + .arg(&input) + .output() + .expect("run convert"); + assert!( + output.status.success(), + "skipping a sidecar file must not fail the batch" + ); + assert!( + String::from_utf8_lossy(&output.stdout) + .contains("skipped (not a degree/program/cluster file)"), + "should report the skip" + ); +} + +/// `degree schema` prints the unified-degree JSON Schema (valid JSON) to stdout. +#[test] +fn test_degree_schema_emits_valid_json() { + let output = Command::new(env!("CARGO_BIN_EXE_nuanalytics")) + .args(["degree", "schema"]) + .output() + .expect("run schema"); + assert!(output.status.success(), "schema should succeed"); + let stdout = String::from_utf8_lossy(&output.stdout); + let parsed: serde_json::Value = + serde_json::from_str(stdout.trim()).expect("emitted schema must be valid JSON"); + assert_eq!(parsed["title"], "NuAnalytics Unified Degree Program"); +} + +/// Write a minimal analyzable unified degree JSON with a distinct id/name. +#[cfg(test)] +fn write_min_degree(path: &std::path::Path, id: &str) { + std::fs::write( + path, + format!( + r#"{{"degree":{{"name":"Test {id}","degree_type":"BS","system_type":"semester","id":"{id}"}}, + "requirements":{{"core":{{"type":"all","courses":["CS101","CS102"]}}}}, + "courses":{{ + "CS101":{{"name":"Intro","prefix":"CS","number":"101","credit_hours":3.0}}, + "CS102":{{"name":"DS","prefix":"CS","number":"102","credit_hours":3.0,"prerequisites":"CS101"}} + }}}}"# + ), + ) + .unwrap(); +} + +/// A multi-file `degree analyze` runs the default worker pool (one process per +/// file) and produces one report JSON per program. +#[test] +fn test_degree_analyze_parallel_pool_produces_reports() { + let dir = tempfile::tempdir().expect("tempdir"); + for id in ["alpha", "beta", "gamma"] { + write_min_degree(&dir.path().join(format!("{id}.unified.json")), id); + } + let out = dir.path().join("metrics"); + + let output = Command::new(env!("CARGO_BIN_EXE_nuanalytics")) + .args(["degree", "analyze"]) + .arg(dir.path().join("alpha.unified.json")) + .arg(dir.path().join("beta.unified.json")) + .arg(dir.path().join("gamma.unified.json")) + .args(["-j", "2", "--no-report", "--metrics-dir"]) + .arg(&out) + .output() + .expect("run analyze pool"); + assert!( + output.status.success(), + "pool run should succeed. stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + stdout.contains("worker process"), + "should use the pool: {stdout}" + ); + assert!( + stdout.contains("analyzed 3/3"), + "should report 3/3: {stdout}" + ); + + let reports = std::fs::read_dir(&out) + .unwrap() + .filter_map(Result::ok) + .filter(|e| e.file_name().to_string_lossy().ends_with("_report.json")) + .count(); + assert_eq!(reports, 3, "one report JSON per program"); + + // Single index.csv with exactly one header + three rows (no header race). + let index = std::fs::read_to_string(out.join("index.csv")).unwrap(); + let lines: Vec<&str> = index.lines().collect(); + assert_eq!(lines.len(), 4, "header + 3 rows; got {lines:?}"); + assert_eq!( + index.matches("degree_id,degree_name").count(), + 1, + "exactly one header row" + ); +} + +/// One failing file in the pool is isolated: it's logged and the run exits +/// non-zero, but the other programs still produce reports. +#[test] +fn test_degree_analyze_parallel_isolates_failure() { + let dir = tempfile::tempdir().expect("tempdir"); + write_min_degree(&dir.path().join("good.unified.json"), "good"); + std::fs::write(dir.path().join("bad.unified.json"), r#"{"not":"a degree"}"#).unwrap(); + let out = dir.path().join("metrics"); + + let output = Command::new(env!("CARGO_BIN_EXE_nuanalytics")) + .args(["degree", "analyze"]) + .arg(dir.path().join("good.unified.json")) + .arg(dir.path().join("bad.unified.json")) + .args(["-j", "2", "--no-report", "--metrics-dir"]) + .arg(&out) + .output() + .expect("run analyze pool"); + assert!( + !output.status.success(), + "a failing file makes the run exit non-zero" + ); + + let failures = std::fs::read_to_string(out.join("failures.log")).unwrap(); + assert!( + failures.contains("bad.unified.json"), + "bad file logged: {failures}" + ); + assert!( + out.join("good_report.json").exists(), + "the good program still produced its report" + ); +} + +/// `-j 1` forces the in-process path even for a multi-file batch: reports are +/// still produced, but no worker-pool banner is printed. +#[test] +fn test_degree_analyze_jobs_one_runs_in_process() { + let dir = tempfile::tempdir().expect("tempdir"); + for id in ["alpha", "beta"] { + write_min_degree(&dir.path().join(format!("{id}.unified.json")), id); + } + let out = dir.path().join("metrics"); + + let output = Command::new(env!("CARGO_BIN_EXE_nuanalytics")) + .args(["degree", "analyze"]) + .arg(dir.path().join("alpha.unified.json")) + .arg(dir.path().join("beta.unified.json")) + .args(["-j", "1", "--no-report", "--metrics-dir"]) + .arg(&out) + .output() + .expect("run analyze in-process"); + assert!( + output.status.success(), + "in-process run should succeed. stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + !stdout.contains("worker process"), + "`-j 1` must not spawn the worker pool: {stdout}" + ); + + let reports = std::fs::read_dir(&out) + .unwrap() + .filter_map(Result::ok) + .filter(|e| e.file_name().to_string_lossy().ends_with("_report.json")) + .count(); + assert_eq!(reports, 2, "one report JSON per program even in-process"); +}