diff --git a/.env.example b/.env.example index 510a601..5765188 100644 --- a/.env.example +++ b/.env.example @@ -1,18 +1,20 @@ -# Vapi GitOps — copy values into environment-specific files (never commit real keys). +# Vapi GitOps — copy values into org-specific files (never commit real keys). # -# This repo loads secrets from `.env.` when you run commands, e.g.: -# - `.env.dev` → `npm run push:dev`, `npm run pull:dev`, etc. -# - `.env.stg` → `npm run push:stg`, … -# - `.env.prod` → `npm run push:prod`, … +# This repo loads secrets from `.env.` when you run commands, e.g.: +# - `.env.my-org` → `npm run push -- my-org`, `npm run pull -- my-org`, etc. +# - `.env.my-org-prod` → `npm run push -- my-org-prod`, … # -# You can optionally add `.env..local` or `.env.local` for overrides +# You can optionally add `.env..local` or `.env.local` for overrides # (see `src/config.ts`). # -# Different Vapi organizations (dev vs prod workspaces) need different private API keys. -# Create a separate file per environment and paste the private key for that org only. +# Different Vapi organizations need different private API keys. +# Create a separate file per org slug and paste the private key for that org only. +# Run `npm run setup` for an interactive wizard that creates these files. # Required: Vapi private API key for the organization you are syncing to. VAPI_TOKEN=your-vapi-private-key-here -# Optional: defaults to https://api.vapi.ai if unset (use for local/API proxies only). +# Optional: API base URL — defaults to US (https://api.vapi.ai). +# Set to the EU endpoint for EU-region orgs. # VAPI_BASE_URL=https://api.vapi.ai +# VAPI_BASE_URL=https://api.eu.vapi.ai diff --git a/.gitignore b/.gitignore index ff0f903..9248f16 100644 --- a/.gitignore +++ b/.gitignore @@ -2,15 +2,9 @@ node_modules/ # Environment files (secrets - never commit these!) +# Covers dev/stg/prod and any org slug (e.g. .env.roofr-production) .env -.env.dev -.env.staging -.env.stg -.env.prod -.env.local -.env.*.local - -# Keep the example file +.env.* !.env.example # IDE @@ -24,5 +18,7 @@ Thumbs.db # Logs *.log +tmp/ + # Local agent state .claude/ diff --git a/.vapi-state.dev.json b/.vapi-state.dev.json deleted file mode 100644 index 685d367..0000000 --- a/.vapi-state.dev.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "credentials": {}, - "assistants": {}, - "structuredOutputs": {}, - "tools": {}, - "squads": {}, - "personalities": {}, - "scenarios": {}, - "simulations": {}, - "simulationSuites": {} -} diff --git a/.vapi-state.prod.json b/.vapi-state.prod.json deleted file mode 100644 index 685d367..0000000 --- a/.vapi-state.prod.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "credentials": {}, - "assistants": {}, - "structuredOutputs": {}, - "tools": {}, - "squads": {}, - "personalities": {}, - "scenarios": {}, - "simulations": {}, - "simulationSuites": {} -} diff --git a/AGENTS.md b/AGENTS.md index e720ca5..dcf62ba 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,9 +6,9 @@ This project manages **Vapi voice agent configurations** as code. All resources **Prompt quality:** Whenever you create a new assistant or change an existing assistant’s system prompt, read **`docs/Vapi Prompt Optimization Guide.md`** first. It goes deeper on structure, voice constraints, tool usage, and evaluation than the summary in this file. -**Environment-scoped resources:** Resources live in `resources//` (e.g. `resources/dev/`, `resources/prod/`). Each environment directory is isolated — `push:dev` only touches `resources/dev/`, `push:prod` only touches `resources/prod/`. See **`docs/environment-scoped-resources.md`** for the full promotion workflow and rationale. +**Org-scoped resources:** Resources live in `resources//` (e.g. `resources/my-org/`, `resources/my-org-prod/`). Each org directory is isolated — `npm run push -- my-org` only touches `resources/my-org/`. Run `npm run setup` to create a new org. -**Template-safe first run:** In a fresh clone, prefer `npm run pull:dev:bootstrap` (or the matching env) to refresh `.vapi-state..json` and credential mappings without materializing the target org's resources into `resources//`. `push:` will auto-run the same bootstrap sync when it detects empty or stale state for the resources being applied. +**Template-safe first run:** In a fresh clone, prefer `npm run pull -- --bootstrap` to refresh `.vapi-state..json` and credential mappings without materializing the target org's resources into `resources//`. `npm run push -- ` will auto-run the same bootstrap sync when it detects empty or stale state for the resources being applied. **Excluding resources from sync (`.vapi-ignore`):** To prevent specific resources from being pulled at all (e.g. assistants owned by another team or legacy resources you don't want to manage), create `resources//.vapi-ignore` with gitignore-style patterns. See `resources//.vapi-ignore.example` for syntax and examples. Ignored resources are silently skipped on every pull and never tracked in state — distinct from "locally deleted" which keeps an entry in state. @@ -38,20 +38,20 @@ This project manages **Vapi voice agent configurations** as code. All resources | I want to... | What to do | | ----------------------------------- | ----------------------------------------------------------------------------- | -| Edit an assistant's system prompt | Edit the markdown body in `resources//assistants/.md` | -| Change assistant settings | Edit the YAML frontmatter in the same `.md` file | -| Add a new tool | Create `resources//tools/.yml` | -| Add a new assistant | Create `resources//assistants/.md` | -| Create a multi-agent squad | Create `resources//squads/.yml` | -| Add post-call analysis | Create `resources//structuredOutputs/.yml` | -| Write test simulations | Create files under `resources//simulations/` | -| Promote resources across envs | Copy files from `resources/dev/` to `resources/stg/` or `resources/prod/` | -| Test webhook event delivery locally | Run `npm run mock:webhook` and tunnel with ngrok | -| Push changes to Vapi | `npm run push:dev` or `npm run push:prod` | -| Pull latest from Vapi | `npm run pull:dev`, `npm run pull:dev:force`, or `npm run pull:dev:bootstrap` | -| Pull one known remote resource | `npm run pull:dev -- assistants --id ` | -| Push only one file | `npm run push:dev resources/dev/assistants/my-agent.md` | -| Test a call | `npm run call:dev -- -a ` | +| Edit an assistant's system prompt | Edit the markdown body in `resources//assistants/.md` | +| Change assistant settings | Edit the YAML frontmatter in the same `.md` file | +| Add a new tool | Create `resources//tools/.yml` | +| Add a new assistant | Create `resources//assistants/.md` | +| Create a multi-agent squad | Create `resources//squads/.yml` | +| Add post-call analysis | Create `resources//structuredOutputs/.yml` | +| Write test simulations | Create files under `resources//simulations/` | +| Promote resources across orgs | Copy files between `resources//` and `resources//` | +| Test webhook event delivery locally | Run `npm run mock:webhook` and tunnel with ngrok | +| Push changes to Vapi | `npm run push -- ` | +| Pull latest from Vapi | `npm run pull -- `, `--force`, or `--bootstrap` | +| Pull one known remote resource | `npm run pull -- --type assistants --id ` | +| Push only one file | `npm run push -- resources//assistants/my-agent.md` | +| Test a call | `npm run call -- -a ` | --- @@ -60,7 +60,6 @@ This project manages **Vapi voice agent configurations** as code. All resources ``` docs/ ├── Vapi Prompt Optimization Guide.md # In-depth prompt authoring -├── environment-scoped-resources.md # Environment isolation & promotion workflow ├── changelog.md # Template for tracking per-customer config changes └── learnings/ # Gotchas, recipes, and troubleshooting ├── README.md # Task-routed index — start here @@ -80,15 +79,14 @@ docs/ └── voicemail-detection.md # Voicemail vs human classification resources/ -├── dev/ # Dev environment resources (push:dev reads here) +├── / # Org-scoped resources (npm run push -- reads here) │ ├── assistants/ │ ├── tools/ │ ├── squads/ │ ├── structuredOutputs/ +│ ├── evals/ │ └── simulations/ -├── stg/ # Staging environment resources (push:stg reads here) -│ └── (same structure) -└── prod/ # Production environment resources (push:prod reads here) +└── / # Another org (each is isolated) └── (same structure) scripts/ @@ -103,7 +101,7 @@ scripts/ Assistants are voice agents that handle phone calls. They are defined as **Markdown files with YAML frontmatter**. -**File:** `resources//assistants/.md` +**File:** `resources//assistants/.md` ```markdown --- @@ -315,7 +313,7 @@ artifactPlan: Tools are functions the assistant can call during a conversation. -**File:** `resources//tools/.yml` +**File:** `resources//tools/.yml` #### Function Tool (calls a webhook) @@ -414,7 +412,7 @@ function: Structured outputs extract data from call transcripts after the call ends. They run LLM analysis on the conversation. -**File:** `resources//structuredOutputs/.yml` +**File:** `resources//structuredOutputs/.yml` #### Boolean Output (yes/no evaluation) @@ -496,12 +494,12 @@ schema: Squads define multi-agent systems where assistants can hand off to each other. -**File:** `resources//squads/.yml` +**File:** `resources//squads/.yml` ```yaml name: My Squad members: - - assistantId: intake-agent-a1b2c3d4 # References resources//assistants/.md + - assistantId: intake-agent-a1b2c3d4 # References resources//assistants/.md assistantOverrides: # Override assistant settings within this squad metadata: position: # Visual position in dashboard editor @@ -647,10 +645,10 @@ Resources reference each other by **filename without extension**: | From | Field | References | Example | | ------------- | ------------------------------------ | ----------------------- | ----------------------------------------- | -| Assistant | `model.toolIds[]` | Tool files | `- end-call-tool` | -| Assistant | `artifactPlan.structuredOutputIds[]` | Structured Output files | `- customer-data` | -| Squad | `members[].assistantId` | Assistant files | `assistantId: intake-agent-a1b2c3d4` | -| Squad handoff | `destinations[].assistantName` | Assistant `name` field | `assistantName: Booking Assistant` | +| Assistant | `model.toolIds[]` | Tool files | `- end-call-tool` | +| Assistant | `artifactPlan.structuredOutputIds[]` | Structured Output files | `- customer-data` | +| Squad | `members[].assistantId` | Assistant files | `assistantId: intake-agent-a1b2c3d4` | +| Squad handoff | `destinations[].assistantName` | Assistant `name` field | `assistantName: Booking Assistant` | | Simulation | `personalityId` | Personality files | `personalityId: skeptical-sam-a0000001` | | Simulation | `scenarioId` | Scenario files | `scenarioId: happy-path-booking-a0000002` | | Suite | `simulationIds[]` | Simulation test files | `- booking-test-1-a0000001` | @@ -739,26 +737,35 @@ Concrete example conversations showing expected behavior. ## Available Commands ```bash +# Setup +npm run setup # Interactive wizard: API key, org slug, resource selection + # Sync -npm run pull:dev # Pull from Vapi (preserve local changes) -npm run pull:dev:force # Pull from Vapi (overwrite everything) -npm run pull:dev:bootstrap # Refresh state without writing remote resources locally -npm run pull:dev -- squads --id # Pull one known remote resource by UUID -# `--id` requires exactly one resource type; it will error if omitted or combined with multiple types -npm run push:dev # Push all local changes to Vapi -npm run push:dev assistants # Push only assistants -npm run push:dev resources/dev/assistants/my-agent.md # Push single file +npm run pull -- # Pull from Vapi (preserve local changes) +npm run pull -- --force # Pull from Vapi (overwrite everything) +npm run pull -- --bootstrap # Refresh state without writing remote resources locally +npm run pull -- --type squads --id # Pull one known remote resource by UUID +npm run push -- # Push all local changes to Vapi +npm run push -- assistants # Push only assistants +npm run push -- resources//assistants/my-agent.md # Push single file +npm run apply -- # Pull then push (full sync) # Testing -npm run call:dev -- -a # Call an assistant via WebSocket -npm run call:dev -- -s # Call a squad via WebSocket -npm run mock:webhook # Run local webhook receiver for server message testing +npm run call -- -a # Call an assistant via WebSocket +npm run call -- -s # Call a squad via WebSocket +npm run eval -- -s # Run evals against a squad +npm run eval -- -a # Run evals against an assistant +npm run mock:webhook # Run local webhook receiver for server message testing + +# Maintenance +npm run cleanup -- # Dry-run: show orphaned remote resources +npm run cleanup -- --force # Delete orphaned remote resources # Build -npm run build # Type-check +npm run build # Type-check ``` -Replace `dev` with `prod` for production environment. +All commands accept an org slug (e.g. `my-org`). Running without arguments launches interactive mode. --- diff --git a/README.md b/README.md index e9884a2..16d2e6b 100644 --- a/README.md +++ b/README.md @@ -14,14 +14,6 @@ Manage Vapi resources via Git using YAML/Markdown as the source-of-truth. | **Reproducibility** | "It worked on my assistant!" | Declarative, version-controlled | | **Disaster Recovery** | Hope you have backups | Re-apply from git | -### Key Benefits - -- **Audit Trail** — Every change is a commit with author, timestamp, and reason -- **Code Review** — Catch misconfigurations before they hit production -- **Environment Parity** — Dev, staging, and prod stay in sync -- **No Drift** — Pull merges platform changes; push makes git the truth -- **Automation Ready** — Plug into CI/CD pipelines - ### Supported Resources | Resource | Status | Format | @@ -34,23 +26,7 @@ Manage Vapi resources via Git using YAML/Markdown as the source-of-truth. | **Scenarios** | ✅ | `.yml` | | **Simulations** | ✅ | `.yml` | | **Simulation Suites** | ✅ | `.yml` | - ---- - -## How to Use This Repo - -1. **Bootstrap state first** using `pull:*:bootstrap` when you need fresh platform mappings without downloading the org's resources into your working tree. -2. **Edit declarative resources** in `resources//` (`.md` assistants, `.yml` tools/squads/etc.). -3. **Push selectively while iterating** (resource type or file path), then run a full push before release. -4. **Promote by environment** (`dev` -> `stg` -> `prod`) by copying files between `resources/dev/`, `resources/stg/`, and `resources/prod/`. - -Use: - -- `pull` when Vapi might have changed -- `push` for explicit deploys -- `apply` (`pull -> merge -> push`) when you want one command for sync + deploy - -For template-based repos, `push` now auto-runs a bootstrap state sync when local state is missing credential mappings or contains stale IDs for the resources you're applying. +| **Evals** | ✅ | `.yml` | --- @@ -67,283 +43,247 @@ For template-based repos, `push` now auto-runs a bootstrap state sync when local npm install ``` -### Setup Environment +### Interactive Setup -```bash -# Copy example values, then set real keys -cp .env.example .env.dev -cp .env.example .env.stg -cp .env.example .env.prod +The easiest way to get started is the interactive setup wizard: -# Add the correct VAPI_TOKEN for each org/environment -# Note: this repo uses .env.stg (not .env.staging) +```bash +npm run setup ``` -### Commands +This will: -| Command | Description | -| ------------------------------- | -------------------------------------------------------------------------- | -| `npm run build` | Type-check the codebase | -| `npm run pull:dev` | Pull platform state, preserve local changes | -| `npm run pull:stg` | Pull staging state, preserve local changes | -| `npm run pull:dev:force` | Pull platform state, overwrite everything | -| `npm run pull:stg:force` | Pull staging state, overwrite everything | -| `npm run pull:prod` | Pull from prod, preserve local changes | -| `npm run pull:prod:force` | Pull from prod, overwrite everything | -| `npm run pull:dev:bootstrap` | Refresh dev state/credentials without writing remote resources locally | -| `npm run pull:stg:bootstrap` | Refresh staging state/credentials without writing remote resources locally | -| `npm run pull:prod:bootstrap` | Refresh prod state/credentials without writing remote resources locally | -| `npm run push:dev` | Push local files to Vapi (dev) | -| `npm run push:stg` | Push local files to Vapi (staging) | -| `npm run push:prod` | Push local files to Vapi (prod) | -| `npm run apply:dev` | Pull → Merge → Push in one shot (dev) | -| `npm run apply:stg` | Pull → Merge → Push in one shot (staging) | -| `npm run apply:prod` | Pull → Merge → Push in one shot (prod) | -| `npm run push:dev assistants` | Push only assistants (dev) | -| `npm run push:dev tools` | Push only tools (dev) | -| `npm run call:dev -- -a ` | Start a WebSocket call to an assistant (dev) | -| `npm run call:dev -- -s ` | Start a WebSocket call to a squad (dev) | -| `npm run mock:webhook` | Run local webhook receiver for Vapi server messages | - -### Basic Workflow +1. Prompt for your Vapi API key (with region auto-detection) +2. Ask for an org/folder name (e.g. `my-org`, `production`) +3. Let you choose which resources to download (all or pick individually) +4. Detect dependencies and offer to download them too +5. Create `.env.` and `resources//` for you -```bash -# First time in a template clone: refresh only state and credentials -npm run pull:dev:bootstrap +You can run setup multiple times to add more orgs. -# Add or edit only the resources you actually want under resources/dev/ +### Commands -# Push your changes (full sync) -npm run push:dev -``` +Every command works in two modes: -#### Bootstrap State Sync (Template-Safe First Run) +- **Interactive** — run without arguments, get prompted for org and resources +- **Direct** — pass an org slug and flags for scripting / CI -Use bootstrap pull when you need the latest platform IDs and credential mappings but do not want the repo filled with assistants, tools, and other resources from the target Vapi org: +| Command | Interactive | Direct | Description | +| --- | --- | --- | --- | +| `npm run setup` | ✅ | — | First-time org setup wizard | +| `npm run pull` | ✅ | `npm run pull -- [flags]` | Pull remote resources locally | +| `npm run push` | ✅ | `npm run push -- [flags]` | Push local resources to Vapi | +| `npm run apply` | ✅ | `npm run apply -- [--force]` | Pull → Merge → Push in one shot | +| `npm run call` | ✅ | `npm run call -- -a ` | Start a WebSocket call | +| `npm run cleanup` | ✅ | `npm run cleanup -- [--force]` | Delete orphaned remote resources | +| `npm run eval` | — | `npm run eval -- -s ` | Run evals against an assistant/squad | +| `npm run mock:webhook` | — | — | Local webhook receiver for testing | +| `npm run build` | — | — | Type-check the codebase | -```bash -npm run pull:dev:bootstrap -``` +### Interactive Mode -This mode: +When you run a command without arguments, you get a fully interactive experience: -- Pulls credentials into `.vapi-state..json` -- Refreshes remote resource ID mappings in the state file -- Leaves `resources//` untouched so your working tree stays focused on the resources you actually intend to manage +```bash +npm run push +# → Select org (if multiple configured) +# → All resources / Let me pick… +# → Searchable multi-select with git status indicators +# → Confirm and execute -If you skip this step, `push` will automatically run the same bootstrap sync when it detects empty or stale state for the resources being applied. +npm run pull +# → Select org +# → All resources / Let me pick… +# → Shows which resources are already local (✔) +# → Confirm and execute +``` -#### Pulling A Single Resource By UUID +Navigation: +- **Type** to search/filter resources +- **Space** to toggle selection +- **Ctrl+A** to select/deselect all visible +- **Enter** to confirm +- **Esc** to go back to the previous step -If you know the remote Vapi UUID for a specific resource, you can pull just that resource by combining exactly one resource type with `--id`: +### Direct Mode + +Pass an org slug as the first argument to skip interactive prompts: ```bash -# Materialize one squad locally -npm run pull:dev -- squads --id +# Pull everything for an org +npm run pull -- my-org -# Refresh state only for one assistant -npm run pull:dev:bootstrap -- assistants --id -``` +# Force pull (overwrite local changes) +npm run pull -- my-org --force -Notes: +# Push only assistants +npm run push -- my-org assistants -- `--id` currently supports remote Vapi UUIDs only -- `--id` must be paired with exactly one resource type such as `assistants`, `squads`, or `tools` -- Single-resource pull updates only the targeted resource mappings and preserves the rest of the state file +# Push a single file +npm run push -- my-org resources/my-org/assistants/my-agent.md -This will error if you do not provide exactly one resource type: +# Pull with bootstrap (state only, no files written) +npm run pull -- my-org --bootstrap -```bash -# Invalid: no resource type -npm run pull:dev -- --id +# Pull a single resource by UUID +npm run pull -- my-org --type assistants --id -# Invalid: more than one resource type -npm run pull:dev -- assistants squads --id -``` - -Promotion example: +# Call an assistant +npm run call -- my-org -a my-assistant -```bash -# After validating in dev, copy to staging and push -cp resources/dev/squads/your-squad.yml resources/stg/squads/ -npm run push:stg +# Call a squad +npm run call -- my-org -s my-squad -# Promote to prod when ready -cp resources/stg/squads/your-squad.yml resources/prod/squads/ -npm run push:prod +# Run evals +npm run eval -- my-org -s my-squad +npm run eval -- my-org -a my-assistant --filter booking ``` -#### Pulling Without Losing Local Work +--- -By default, `pull` preserves any files you've locally modified or deleted: +## Organization-Based Structure -```bash -# Edit an assistant locally... +Resources are scoped by organization (not fixed `dev`/`stg`/`prod` names). Each org gets: -npm run pull:dev -# ⏭️ my-assistant (locally changed, skipping) -# ✨ new-tool -> resources/dev/tools/new-tool.yml -# Your edits are preserved, new platform resources are downloaded -``` +- `.env.` — API token and base URL +- `.vapi-state..json` — resource ID ↔ UUID mappings +- `resources//` — all resource files -#### Force Pull (Platform as Source of Truth) +``` +vapi-gitops/ +├── .env.my-org # API token for my-org +├── .env.production # API token for production +├── .vapi-state.my-org.json # State file for my-org +├── .vapi-state.production.json # State file for production +├── resources/ +│ ├── my-org/ # Dev/test org resources +│ │ ├── assistants/ +│ │ ├── tools/ +│ │ ├── squads/ +│ │ ├── structuredOutputs/ +│ │ ├── evals/ +│ │ └── simulations/ +│ └── production/ # Production org resources +│ └── (same structure) +``` -When you want the platform version of everything, overwriting all local files: +### Promoting Resources Across Orgs ```bash -npm run pull:dev:force -# ⚡ Force mode: overwriting all local files with platform state +# Copy a squad from dev to production +cp resources/my-org/squads/voice-squad.yml resources/production/squads/ +cp resources/my-org/assistants/intake-agent.md resources/production/assistants/ + +# Push to production (missing dependencies auto-resolve) +npm run push -- production ``` -#### Reviewing Platform Changes +--- -```bash -# Pull platform state (your local changes are preserved) -npm run pull:dev +## How to Use This Repo -# See what changed on the platform vs your last commit -git diff +1. **Run `npm run setup`** to configure your first org +2. **Edit resources** in `resources//` (`.md` assistants, `.yml` tools/squads/etc.) +3. **Push changes** with `npm run push` (interactive) or `npm run push -- ` +4. **Pull updates** with `npm run pull` when the platform may have changed -# Accept platform changes for a specific file -git checkout -- resources/dev/tools/some-tool.yml -``` +Use: -### Selective Push (Partial Sync) +- `pull` when Vapi might have changed +- `push` for explicit deploys +- `apply` (`pull -> merge -> push`) for sync + deploy in one command -Push only specific resources instead of syncing everything: +### Bootstrap State Sync -#### By Resource Type +Use bootstrap pull when you need the latest platform IDs and credential mappings without downloading all remote resources: ```bash -npm run push:dev assistants -npm run push:dev tools -npm run push:dev squads -npm run push:dev structuredOutputs -npm run push:dev personalities -npm run push:dev scenarios -npm run push:dev simulations -npm run push:dev simulationSuites +npm run pull -- my-org --bootstrap ``` -#### By Specific File(s) +This refreshes `.vapi-state..json` and credential mappings while leaving `resources//` untouched. If you skip this step, `push` will automatically run it when it detects empty or stale state. -```bash -# Push a single file -npm run push:dev resources/dev/assistants/my-assistant.md +### Pulling a Single Resource By UUID -# Push multiple files -npm run push:dev resources/dev/assistants/booking.md resources/dev/tools/my-tool.yml +```bash +npm run pull -- my-org --type squads --id ``` -#### Combined +`--id` must be paired with exactly one resource type. + +### Pulling Without Losing Local Work + +By default, `pull` preserves any files you've locally modified or deleted: ```bash -# Push specific file within a type -npm run push:dev assistants resources/dev/assistants/booking.md +npm run pull -- my-org +# ⏭️ my-assistant (locally changed, skipping) +# ✨ new-tool -> resources/my-org/tools/new-tool.yml ``` -**Note:** Partial pushes skip deletion checks. Run full `npm run push:dev` to sync deletions. +Use `--force` to overwrite everything with the platform version. -#### Auto-Dependency Resolution +### Selective Push -Partial push is ideal for promoting specific squads or assistants to staging/prod without pushing everything. The engine automatically detects and creates missing dependencies: +Push only specific resources instead of everything: ```bash -# Push a single squad to staging — tools, structured outputs, and -# assistants are created automatically if they don't exist yet -npm run push:stg resources/stg/squads/everblue-voice-squad-20374c37.yml +# By resource type +npm run push -- my-org assistants +npm run push -- my-org tools + +# By specific file +npm run push -- my-org resources/my-org/assistants/my-assistant.md -# Push assistants to prod — missing tools and structured outputs -# are auto-applied first so references resolve correctly -npm run push:prod assistants +# Multiple files +npm run push -- my-org resources/my-org/assistants/a.md resources/my-org/tools/b.yml ``` -The dependency chain resolves recursively: +### Auto-Dependency Resolution + +When pushing a single squad or assistant, missing dependencies (tools, structured outputs, etc.) are automatically created first: ``` Squad push └─ missing assistants? → auto-create them first └─ missing tools / structured outputs? → auto-create those first - └─ then create the assistant └─ all references resolved → create the squad ✓ - -Assistant push - └─ missing tools / structured outputs? → auto-create them first - └─ all references resolved → create the assistant ✓ ``` -If a dependency already exists on the platform (UUID in the state file) but its nested dependencies don't, those are still auto-created and the parent resource is updated to reference them. +### Running Evals -This means you can work on everything in dev, then selectively push a single squad or assistant to staging or prod — no need for a full `push` that touches every resource. - -### Webhook Local Testing - -Use the local mock receiver when validating Vapi `serverMessages` delivery. +Evals run mock conversations against an assistant or squad and check assertions. ```bash -# 1) Run local receiver -npm run mock:webhook +# Run all evals against a squad (transient — loaded from local files) +npm run eval -- my-org -s my-squad -# 2) Expose localhost (example) -ngrok http 8787 -``` +# Run a specific eval by name filter +npm run eval -- my-org -a my-assistant --filter booking -Then set your assistant `server.url` to the ngrok HTTPS URL and include event types like: +# Use stored assistant/squad IDs from state (already pushed) +npm run eval -- my-org -s my-squad --stored -- `speech-update` -- `status-update` -- `end-of-call-report` +# Load assistant from a specific file path +npm run eval -- my-org -a resources/my-org/assistants/qa-tester.md -The mock server exposes: +# Provide variable overrides +npm run eval -- my-org -s my-squad -v eval-variables.json +``` -- `POST /webhook` (or `POST /`) -- `GET /health` -- `GET /events` +Evals must be pushed first (`npm run push -- my-org evals`). Eval definitions live in `resources//evals/*.yml`. ---- +### Webhook Local Testing -## Project Structure +```bash +# 1) Run local receiver +npm run mock:webhook +# 2) Expose localhost +ngrok http 8787 ``` -vapi-gitops/ -├── docs/ -│ ├── Vapi Prompt Optimization Guide.md # Prompt authoring reference -│ ├── environment-scoped-resources.md # Env isolation & promotion workflow -│ └── changelog.md # Template for per-customer change tracking -├── src/ -│ ├── pull.ts # Pull platform state (with git stash/pop merge) -│ ├── push.ts # Push local state to platform -│ ├── apply.ts # Orchestrator: pull → merge → push -│ ├── call.ts # WebSocket call script -│ ├── types.ts # TypeScript interfaces -│ ├── config.ts # Environment & configuration -│ ├── api.ts # Vapi HTTP client -│ ├── state.ts # State file management -│ ├── resources.ts # Resource loading (YAML, MD, TS) -│ ├── resolver.ts # Reference resolution -│ ├── credentials.ts # Credential resolution (name ↔ UUID) -│ └── delete.ts # Deletion & orphan checks -├── resources/ -│ ├── dev/ # Dev environment resources (push:dev reads here) -│ │ ├── assistants/ -│ │ ├── tools/ -│ │ ├── squads/ -│ │ ├── structuredOutputs/ -│ │ └── simulations/ -│ ├── stg/ # Staging resources (push:stg reads here) -│ │ └── (same structure) -│ └── prod/ # Production resources (push:prod reads here) -│ └── (same structure) -├── scripts/ -│ └── mock-vapi-webhook-server.ts # Local server message receiver -├── .env.example # Example env var file -├── .env.dev # Dev environment secrets (gitignored) -├── .env.stg # Staging environment secrets (gitignored) -├── .env.prod # Prod environment secrets (gitignored) -├── .vapi-state.dev.json # Dev state file -├── .vapi-state.stg.json # Staging state file -└── .vapi-state.prod.json # Prod state file -``` + +Set your assistant's `server.url` to the ngrok HTTPS URL. --- @@ -351,7 +291,7 @@ vapi-gitops/ ### Assistants with System Prompts (`.md`) -Assistants with system prompts use **Markdown with YAML frontmatter**. The system prompt is written as readable Markdown below the config: +Markdown with YAML frontmatter — the system prompt is readable Markdown below the config: ```markdown --- @@ -360,7 +300,7 @@ voice: provider: 11labs voiceId: abc123 model: - model: gpt-4o + model: gpt-4.1 provider: openai toolIds: - my-tool @@ -383,28 +323,6 @@ You are a helpful assistant for the business you represent. - Never make up information ``` -**Benefits:** - -- System prompts are readable Markdown (not escaped YAML strings) -- Proper syntax highlighting in editors -- Easy to write headers, lists, tables -- Configuration stays cleanly separated at the top - -### Assistants without System Prompts (`.yml`) - -Simple assistants without custom system prompts use plain YAML: - -```yaml -name: Simple Assistant -voice: - provider: vapi - voiceId: Elliot -model: - model: gpt-4o-mini - provider: openai -firstMessage: Hello! -``` - ### Tools (`.yml`) ```yaml @@ -455,6 +373,14 @@ members: - assistantId: specialist-agent ``` +### Evals (`.yml`) + +```yaml +name: Booking Happy Path +type: eval +# (eval config as per Vapi API) +``` + ### Simulations **Personality** (`simulations/personalities/`): @@ -472,7 +398,6 @@ name: Happy Path - New Customer description: New customer calling to schedule an appointment prompt: | You are a new customer calling to schedule your first appointment. - Be cooperative and provide all requested information. ``` **Simulation** (`simulations/tests/`): @@ -490,142 +415,6 @@ name: Booking Flow Tests simulationIds: - booking-test-case-1 - booking-test-case-2 - - booking-test-case-3 -``` - ---- - -## How-To Guides - -### How to Add a New Assistant - -**Option 1: With System Prompt (recommended)** - -Create `resources/dev/assistants/my-assistant.md`: - -```markdown ---- -name: My Assistant -voice: - provider: 11labs - voiceId: abc123 -model: - model: gpt-4o - provider: openai - toolIds: - - my-tool ---- - -# Your System Prompt Here - -Instructions for the assistant... -``` - -**Option 2: Without System Prompt** - -Create `resources/dev/assistants/my-assistant.yml`: - -```yaml -name: My Assistant -voice: - provider: vapi - voiceId: Elliot -model: - model: gpt-4o-mini - provider: openai -``` - -Then push: - -```bash -npm run push:dev -``` - -### How to Add a Tool - -Create `resources/dev/tools/my-tool.yml`: - -```yaml -type: function -function: - name: do_something - description: Does something useful - parameters: - type: object - properties: - input: - type: string - required: - - input -server: - url: https://my-api.com/endpoint -``` - -### How to Reference Resources - -Use the **filename without extension** as the resource ID: - -```yaml -# In an assistant -model: - toolIds: - - my-tool # → resources//tools/my-tool.yml - - utils/helper-tool # → resources//tools/utils/helper-tool.yml -artifactPlan: - structuredOutputIds: - - call-summary # → resources//structuredOutputs/call-summary.yml -``` - -```yaml -# In a squad -members: - - assistantId: intake-agent # → resources//assistants/intake-agent.md -``` - -```yaml -# In a simulation -personalityId: skeptical-sam # → resources//simulations/personalities/skeptical-sam.yml -scenarioId: happy-path # → resources//simulations/scenarios/happy-path.yml -``` - -### How to Delete a Resource - -1. **Remove references** to the resource from other files -2. **Delete the file**: `rm resources/dev/tools/my-tool.yml` -3. **Push**: `npm run push:dev` - -The engine will: - -- Detect the resource is in state but not in filesystem -- Check for orphan references (will error if still referenced) -- Delete from Vapi -- Remove from state file - -### How to Organize Resources into Folders - -Create subdirectories only when they help organize related resources by feature or workflow: - -``` -resources// -├── assistants/ -│ ├── shared/ -│ │ └── fallback.md -│ └── support/ -│ └── intake.md -├── tools/ -│ ├── shared/ -│ │ └── transfer-call.yml -│ └── support/ -│ └── lookup-customer.yml -``` - -Reference using full paths: - -```yaml -model: - toolIds: - - shared/transfer-call - - support/lookup-customer ``` --- @@ -634,8 +423,6 @@ model: ### Sync Workflow -Your local files are the source of truth. The engine respects that: - ``` pull (default) pull --force push ───────────── ───────────── ───────────── @@ -645,40 +432,21 @@ locally changed everything platform files ``` -**`pull`** downloads platform state. In default mode (git repo required), it detects locally modified or deleted files and skips them — your local work is preserved. New platform resources are still downloaded. Use `--force` to overwrite everything. +**`pull`** — downloads platform state. Detects locally modified files and skips them (your work is preserved). Use `--force` to overwrite everything. -**`push`** is the engine — reads local files and syncs them to the platform. Deleted files are removed from the platform. +**`push`** — reads local files and syncs them to the platform. Handles creates, updates, and deletions. -**`apply`** is the convenience wrapper — runs `pull` then `push` in sequence. - -> **Note:** The "skip locally changed files" feature requires a git repo with at least one commit. Without git, pull always overwrites (same as `--force`). +**`apply`** — runs `pull` then `push` in sequence. ### Processing Order -**Pull** (dependency order): - -1. Tools -2. Structured Outputs -3. Assistants -4. Squads -5. Personalities -6. Scenarios -7. Simulations -8. Simulation Suites - -**Push** (dependency order): - -1. Tools → 2. Structured Outputs → 3. Assistants → 4. Squads -2. Personalities → 6. Scenarios → 7. Simulations → 8. Simulation Suites - -**Delete** (reverse dependency order): +**Push** (dependency order): Tools → Structured Outputs → Assistants → Squads → Personalities → Scenarios → Simulations → Simulation Suites → Evals -1. Simulation Suites → 2. Simulations → 3. Scenarios → 4. Personalities -2. Squads → 6. Assistants → 7. Structured Outputs → 8. Tools +**Delete** (reverse dependency order): Evals → Simulation Suites → Simulations → ... → Tools ### Reference Resolution -The engine automatically resolves resource IDs to Vapi UUIDs: +Resource IDs (filenames without extension) are automatically resolved to Vapi UUIDs: ```yaml # You write: @@ -692,64 +460,83 @@ toolIds: ### Credential Management -Credentials (API keys, JWT secrets, etc.) are environment-specific and managed automatically through the state file. No secrets are stored in resource files or git. +Credentials are managed automatically through the state file. No secrets in resource files or git. -**How it works:** - -1. **Pull** fetches all credentials from `GET /credential` and stores `name-slug → UUID` in the state file -2. **Pull** replaces credential UUIDs with human-readable names in resource files -3. **Push** reverses the mapping — resolves credential names back to UUIDs before sending to the API +1. **Pull** fetches credentials from Vapi and stores `name → UUID` in the state file +2. Resource files use human-readable credential names +3. **Push** resolves names back to UUIDs before sending to the API ```yaml -# Resource file stores credential NAME (environment-agnostic) +# Resource file (environment-agnostic) server: - url: https://my-api.com/endpoint - credentialId: my-server-credential # ← human-readable name + credentialId: my-server-credential + +# State file (environment-specific) +# "my-server-credential": "2f6db611-ad08-4099-8bd8-74db37b0a07e" ``` +### State File + +Tracks resource ID ↔ Vapi UUID mappings per org: + ```json -// State file stores credential UUID (environment-specific) { - "credentials": { - "my-server-credential": "2f6db611-ad08-4099-8bd8-74db37b0a07e" - } + "credentials": { "my-cred": "uuid-0000" }, + "tools": { "my-tool": "uuid-1234" }, + "assistants": { "my-assistant": "uuid-5678" }, + "squads": { "my-squad": "uuid-abcd" }, + "evals": { "booking-happy-path": "uuid-efgh" } } ``` -**Cross-environment workflow:** +--- -Each environment has its own state file with its own credential UUIDs. The same resource file works across all environments — only the state file differs: +## Project Structure ``` -.vapi-state.dev.json → "my-cred": "uuid-for-dev" -.vapi-state.stg.json → "my-cred": "uuid-for-stg" -.vapi-state.prod.json → "my-cred": "uuid-for-prod" -``` - -> **Note:** Credentials are auto-discovered from the Vapi API by name. Create credentials with the same name in each environment's Vapi org, and pull will populate the mappings automatically. - -### State File - -Tracks mapping between resource IDs and Vapi UUIDs: - -```json -{ - "credentials": { - "my-server-credential": "uuid-0000" - }, - "tools": { - "my-tool": "uuid-1234" - }, - "assistants": { - "my-assistant": "uuid-5678" - }, - "squads": { - "my-squad": "uuid-abcd" - }, - "personalities": { - "skeptical-sam": "uuid-efgh" - } -} +vapi-gitops/ +├── docs/ +│ ├── Vapi Prompt Optimization Guide.md +│ └── changelog.md +├── src/ +│ ├── setup.ts # Interactive setup wizard +│ ├── interactive.ts # Interactive pull/push/apply/call/cleanup flows +│ ├── searchableCheckbox.ts # Custom multi-select prompt component +│ ├── pull.ts # Pull platform state +│ ├── push.ts # Push local state to platform +│ ├── apply.ts # Orchestrator: pull → merge → push +│ ├── call.ts # WebSocket call script +│ ├── eval.ts # Eval runner +│ ├── cleanup.ts # Orphan cleanup +│ ├── pull-cmd.ts # Entry point: interactive or direct pull +│ ├── push-cmd.ts # Entry point: interactive or direct push +│ ├── apply-cmd.ts # Entry point: interactive or direct apply +│ ├── call-cmd.ts # Entry point: interactive or direct call +│ ├── cleanup-cmd.ts # Entry point: interactive or direct cleanup +│ ├── types.ts # TypeScript interfaces +│ ├── config.ts # Environment & configuration +│ ├── api.ts # Vapi HTTP client +│ ├── state.ts # State file management +│ ├── resources.ts # Resource loading (YAML, MD, TS) +│ ├── resolver.ts # Reference resolution +│ ├── credentials.ts # Credential resolution (name ↔ UUID) +│ └── delete.ts # Deletion & orphan checks +├── resources/ +│ └── / # One directory per configured org +│ ├── assistants/ +│ ├── tools/ +│ ├── squads/ +│ ├── structuredOutputs/ +│ ├── evals/ +│ └── simulations/ +│ ├── personalities/ +│ ├── scenarios/ +│ ├── tests/ +│ └── suites/ +├── scripts/ +│ └── mock-vapi-webhook-server.ts +├── .env. # API token per org (gitignored) +└── .vapi-state..json # State file per org ``` --- @@ -763,13 +550,7 @@ Tracks mapping between resource IDs and Vapi UUIDs: | `VAPI_TOKEN` | ✅ | API authentication token | | `VAPI_BASE_URL` | ❌ | API base URL (defaults to `https://api.vapi.ai`) | -### Excluded Fields - -Some fields are excluded when writing to files (server-managed): - -- `id`, `orgId`, `createdAt`, `updatedAt` -- `analyticsMetadata`, `isDeleted` -- `isServerUrlSecretSet`, `workflowIds` +These are stored in `.env.` files, one per configured organization. --- @@ -795,22 +576,18 @@ The referenced resource doesn't exist. Check: Check the state file has correct UUID: -1. Open `.vapi-state.{env}.json` +1. Open `.vapi-state..json` 2. Find the resource entry 3. If incorrect, delete entry and re-run push ### "Credential with ID not found" errors -The credential UUID doesn't exist in the target environment. Fix: +The credential UUID doesn't exist in the target org. Fix: -1. Run `npm run pull:{env}` to fetch credentials into the state file -2. If the credential doesn't exist in the target org, create it in the Vapi dashboard with the same name +1. Run `npm run pull -- ` to fetch credentials into the state file +2. If the credential doesn't exist, create it in the Vapi dashboard with the same name 3. Pull again — the mapping will be auto-populated -### "Unresolved credential" warnings - -A resource file has a `credentialId` that couldn't be resolved to a UUID. This means the credential name isn't in the state file. Run `pull` to populate credential mappings. - ### "property X should not exist" API errors Some properties can't be updated after creation. Add them to `UPDATE_EXCLUDED_KEYS` in `src/config.ts`. @@ -823,3 +600,4 @@ Some properties can't be updated after creation. Add them to `UPDATE_EXCLUDED_KE - [Tools API](https://docs.vapi.ai/api-reference/tools/create) - [Structured Outputs API](https://docs.vapi.ai/api-reference/structured-outputs/structured-output-controller-create) - [Squads API](https://docs.vapi.ai/api-reference/squads/create) +- [Evals API](https://docs.vapi.ai/api-reference/evals) diff --git a/docs/environment-scoped-resources.md b/docs/environment-scoped-resources.md deleted file mode 100644 index fb6f57f..0000000 --- a/docs/environment-scoped-resources.md +++ /dev/null @@ -1,102 +0,0 @@ -# Environment-Scoped Resource Directories - -## Overview - -Resources are stored in environment-scoped subdirectories under `resources/`. The engine reads only from the directory matching the active environment, preventing cross-environment contamination. - -``` -resources/ - dev/ <- push:dev reads/writes here - stg/ <- push:stg reads/writes here - prod/ <- push:prod reads/writes here -``` - -Each environment directory mirrors the same internal structure (`assistants/`, `tools/`, `squads/`, etc.). - -## How It Works - -The engine resolves `RESOURCES_DIR` as `resources//` based on the environment argument passed to every command (`dev`, `stg`, `prod`). All modules (`push.ts`, `pull.ts`, `resources.ts`, `delete.ts`, `resolver.ts`) operate relative to this single path. - -| Command | Reads from | State file | Env file | -|---------|-----------|------------|----------| -| `push:dev` | `resources/dev/` | `.vapi-state.dev.json` | `.env.dev` | -| `push:stg` | `resources/stg/` | `.vapi-state.stg.json` | `.env.stg` | -| `push:prod` | `resources/prod/` | `.vapi-state.prod.json` | `.env.prod` | -| `pull:dev` | writes to `resources/dev/` | `.vapi-state.dev.json` | `.env.dev` | -| `pull:stg` | writes to `resources/stg/` | `.vapi-state.stg.json` | `.env.stg` | -| `pull:prod` | writes to `resources/prod/` | `.vapi-state.prod.json` | `.env.prod` | - -Resource IDs are computed relative to the resource type directory inside the environment scope, so existing state file mappings remain valid. - -## The Three Environments - -### dev — Experimentation and rapid iteration - -- Build and test new assistants, tools, squads -- Try prompt changes, test handoff flows, validate tool integrations against mock servers -- Anyone can push at any time, no approval needed -- May contain resources pointing to mock/ngrok tool endpoints for local testing - -### stg — Pre-production validation - -- Validate resources in a production-like environment before going live -- Run simulation suites, conduct QA calls, verify tool integrations against real endpoints -- Resources should use production tool endpoints, never mock/ngrok -- Acts as a gate — if something breaks here, it does not reach prod - -### prod — Live - -- Resources actively handling real calls -- Only push after successful validation in stg -- Changes should be deliberate and reviewed - -## Promotion Workflow - -Resources flow in one direction: **dev -> stg -> prod**. - -### Promoting dev -> stg - -```bash -# Copy files -cp resources/dev/assistants/my-agent.md resources/stg/assistants/ -cp resources/dev/tools/my-tool.yml resources/stg/tools/ - -# Review: ensure tool URLs point to production endpoints, not mock/ngrok - -# Push to staging -npm run push:stg -``` - -**Note:** The first push to a new environment creates fresh resources in that Vapi org (new UUIDs). The state file for that environment (e.g. `.vapi-state.stg.json`) starts empty, so every resource is treated as a new creation. Subsequent pushes update the existing resources via their tracked UUIDs. - -### Promoting stg -> prod - -```bash -# Copy files -cp resources/stg/assistants/my-agent.md resources/prod/assistants/ -cp resources/stg/tools/my-tool.yml resources/prod/tools/ - -# Push to production -npm run push:prod -``` - -### Skipping staging (dev -> prod directly) - -For urgent fixes, copy directly from dev to prod. Ensure production endpoints are used. Not recommended for routine changes once staging is available. - -## Naming Conventions Within an Environment - -Within `resources/dev/`, the `*-mock` and `*-prod` suffixes on filenames indicate which **tool endpoints** the resource uses, not which Vapi org it targets: - -| Suffix | Tool endpoints | Vapi org | -|--------|---------------|----------| -| `*-mock` | ngrok mock server | dev | -| `*-prod` | production APIs | dev | - -When promoting from dev, only copy the `*-prod` endpoint variants forward. The `*-mock` variants are dev-only. - -## Setting Up a New Environment - -1. Create `.env.` with the `VAPI_TOKEN` for that Vapi org -2. Run `pull:` to populate `resources//` and `.vapi-state..json` -3. Or seed manually by copying files from another environment's directory diff --git a/package-lock.json b/package-lock.json index 154dba4..e73c7fc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "1.0.0", "license": "Apache-2.0", "dependencies": { + "@inquirer/prompts": "^8.4.1", "yaml": "^2.7.0" }, "devDependencies": { @@ -463,11 +464,339 @@ "node": ">=18" } }, + "node_modules/@inquirer/ansi": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@inquirer/ansi/-/ansi-2.0.5.tgz", + "integrity": "sha512-doc2sWgJpbFQ64UflSVd17ibMGDuxO1yKgOgLMwavzESnXjFWJqUeG8saYosqKpHp4kWiM5x1nXvEjbpx90gzw==", + "license": "MIT", + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^21.7.0 || ^20.12.0" + } + }, + "node_modules/@inquirer/checkbox": { + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/@inquirer/checkbox/-/checkbox-5.1.3.tgz", + "integrity": "sha512-+G7I8CT+EHv/hasNfUl3P37DVoMoZfpA+2FXmM54dA8MxYle1YqucxbacxHalw1iAFSdKNEDTGNV7F+j1Ldqcg==", + "license": "MIT", + "dependencies": { + "@inquirer/ansi": "^2.0.5", + "@inquirer/core": "^11.1.8", + "@inquirer/figures": "^2.0.5", + "@inquirer/type": "^4.0.5" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^21.7.0 || ^20.12.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/confirm": { + "version": "6.0.11", + "resolved": "https://registry.npmjs.org/@inquirer/confirm/-/confirm-6.0.11.tgz", + "integrity": "sha512-pTpHjg0iEIRMYV/7oCZUMf27/383E6Wyhfc/MY+AVQGEoUobffIYWOK9YLP2XFRGz/9i6WlTQh1CkFVIo2Y7XA==", + "license": "MIT", + "dependencies": { + "@inquirer/core": "^11.1.8", + "@inquirer/type": "^4.0.5" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^21.7.0 || ^20.12.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/core": { + "version": "11.1.8", + "resolved": "https://registry.npmjs.org/@inquirer/core/-/core-11.1.8.tgz", + "integrity": "sha512-/u+yJk2pOKNDOh1ZgdUH2RQaRx6OOH4I0uwL95qPvTFTIL38YBsuSC4r1yXBB3Q6JvNqFFc202gk0Ew79rrcjA==", + "license": "MIT", + "dependencies": { + "@inquirer/ansi": "^2.0.5", + "@inquirer/figures": "^2.0.5", + "@inquirer/type": "^4.0.5", + "cli-width": "^4.1.0", + "fast-wrap-ansi": "^0.2.0", + "mute-stream": "^3.0.0", + "signal-exit": "^4.1.0" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^21.7.0 || ^20.12.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/editor": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@inquirer/editor/-/editor-5.1.0.tgz", + "integrity": "sha512-6wlkYl65Qfayy48gPCfU4D7li6KCAGN79mLXa/tYHZH99OfZ820yY+HA+DgE88r8YwwgeuY6PQgNqMeK6LuMmw==", + "license": "MIT", + "dependencies": { + "@inquirer/core": "^11.1.8", + "@inquirer/external-editor": "^3.0.0", + "@inquirer/type": "^4.0.5" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^21.7.0 || ^20.12.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/expand": { + "version": "5.0.12", + "resolved": "https://registry.npmjs.org/@inquirer/expand/-/expand-5.0.12.tgz", + "integrity": "sha512-vOfrB33b7YIZfDauXS8vNNz2Z86FozTZLIt7e+7/dCaPJ1RXZsHCuI9TlcERzEUq57vkM+UdnBgxP0rFd23JYQ==", + "license": "MIT", + "dependencies": { + "@inquirer/core": "^11.1.8", + "@inquirer/type": "^4.0.5" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^21.7.0 || ^20.12.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/external-editor": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@inquirer/external-editor/-/external-editor-3.0.0.tgz", + "integrity": "sha512-lDSwMgg+M5rq6JKBYaJwSX6T9e/HK2qqZ1oxmOwn4AQoJE5D+7TumsxLGC02PWS//rkIVqbZv3XA3ejsc9FYvg==", + "license": "MIT", + "dependencies": { + "chardet": "^2.1.1", + "iconv-lite": "^0.7.2" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^21.7.0 || ^20.12.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/figures": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@inquirer/figures/-/figures-2.0.5.tgz", + "integrity": "sha512-NsSs4kzfm12lNetHwAn3GEuH317IzpwrMCbOuMIVytpjnJ90YYHNwdRgYGuKmVxwuIqSgqk3M5qqQt1cDk0tGQ==", + "license": "MIT", + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^21.7.0 || ^20.12.0" + } + }, + "node_modules/@inquirer/input": { + "version": "5.0.11", + "resolved": "https://registry.npmjs.org/@inquirer/input/-/input-5.0.11.tgz", + "integrity": "sha512-twUWidn4ocPO8qi6fRM7tNWt7W1FOnOZqQ+/+PsfLUacMR5rFLDPK9ql0nBPwxi0oELbo8T5NhRs8B2+qQEqFQ==", + "license": "MIT", + "dependencies": { + "@inquirer/core": "^11.1.8", + "@inquirer/type": "^4.0.5" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^21.7.0 || ^20.12.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/number": { + "version": "4.0.11", + "resolved": "https://registry.npmjs.org/@inquirer/number/-/number-4.0.11.tgz", + "integrity": "sha512-Vscmim9TCksQsfjPtka/JwPUcbLhqWYrgfPf1cHrCm24X/F2joFwnageD50yMKsaX14oNGOyKf/RNXAFkNjWpA==", + "license": "MIT", + "dependencies": { + "@inquirer/core": "^11.1.8", + "@inquirer/type": "^4.0.5" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^21.7.0 || ^20.12.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/password": { + "version": "5.0.11", + "resolved": "https://registry.npmjs.org/@inquirer/password/-/password-5.0.11.tgz", + "integrity": "sha512-9KZFeRaNHIcejtPb0wN4ddFc7EvobVoAFa049eS3LrDZFxI8O7xUXiITEOinBzkZFAIwY5V4yzQae/QfO9cbbg==", + "license": "MIT", + "dependencies": { + "@inquirer/ansi": "^2.0.5", + "@inquirer/core": "^11.1.8", + "@inquirer/type": "^4.0.5" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^21.7.0 || ^20.12.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/prompts": { + "version": "8.4.1", + "resolved": "https://registry.npmjs.org/@inquirer/prompts/-/prompts-8.4.1.tgz", + "integrity": "sha512-AH5xPQ997K7e0F0vulPlteIHke2awMkFi8F0dBemrDfmvtPmHJo82mdHbONC4F/t8d1NHwrbI5cGVI+RbLWdoQ==", + "license": "MIT", + "dependencies": { + "@inquirer/checkbox": "^5.1.3", + "@inquirer/confirm": "^6.0.11", + "@inquirer/editor": "^5.1.0", + "@inquirer/expand": "^5.0.12", + "@inquirer/input": "^5.0.11", + "@inquirer/number": "^4.0.11", + "@inquirer/password": "^5.0.11", + "@inquirer/rawlist": "^5.2.7", + "@inquirer/search": "^4.1.7", + "@inquirer/select": "^5.1.3" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^21.7.0 || ^20.12.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/rawlist": { + "version": "5.2.7", + "resolved": "https://registry.npmjs.org/@inquirer/rawlist/-/rawlist-5.2.7.tgz", + "integrity": "sha512-AqRMiD9+uE1lskDPrdqHwrV/EUmxKEBLX44SR7uxK3vD2413AmVfE5EQaPeNzYf5Pq5SitHJDYUFVF0poIr09w==", + "license": "MIT", + "dependencies": { + "@inquirer/core": "^11.1.8", + "@inquirer/type": "^4.0.5" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^21.7.0 || ^20.12.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/search": { + "version": "4.1.7", + "resolved": "https://registry.npmjs.org/@inquirer/search/-/search-4.1.7.tgz", + "integrity": "sha512-1y7+0N65AWk5RdlXH/Kn13txf3IjIQ7OEfhCEkDTU+h5wKMLq8DUF3P6z+/kLSxDGDtQT1dRBWEUC3o/VvImsQ==", + "license": "MIT", + "dependencies": { + "@inquirer/core": "^11.1.8", + "@inquirer/figures": "^2.0.5", + "@inquirer/type": "^4.0.5" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^21.7.0 || ^20.12.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/select": { + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/@inquirer/select/-/select-5.1.3.tgz", + "integrity": "sha512-zYyqWgGQi3NhBcNq4Isc5rB3oEdQEh1Q/EcAnOW0FK4MpnXWkvSBYgA4cYrTM4A9UB573omouZbnL9JJ74Mq3A==", + "license": "MIT", + "dependencies": { + "@inquirer/ansi": "^2.0.5", + "@inquirer/core": "^11.1.8", + "@inquirer/figures": "^2.0.5", + "@inquirer/type": "^4.0.5" + }, + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^21.7.0 || ^20.12.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/type": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-4.0.5.tgz", + "integrity": "sha512-aetVUNeKNc/VriqXlw1NRSW0zhMBB0W4bNbWRJgzRl/3d0QNDQFfk0GO5SDdtjMZVg6o8ZKEiadd7SCCzoOn5Q==", + "license": "MIT", + "engines": { + "node": ">=23.5.0 || ^22.13.0 || ^21.7.0 || ^20.12.0" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, "node_modules/@types/node": { "version": "22.19.7", "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.7.tgz", "integrity": "sha512-MciR4AKGHWl7xwxkBa6xUGxQJ4VBOmPTF7sL+iGzuahOFaO0jHCsuEfS80pan1ef4gWId1oWOweIhrDEYLuaOw==", - "dev": true, + "devOptional": true, "license": "MIT", "dependencies": { "undici-types": "~6.21.0" @@ -508,6 +837,21 @@ "license": "MIT", "optional": true }, + "node_modules/chardet": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/chardet/-/chardet-2.1.1.tgz", + "integrity": "sha512-PsezH1rqdV9VvyNhxxOW32/d75r01NY7TQCmOqomRo15ZSOKbpTFVsfjghxo6JloQUCGnH4k1LGu0R4yCLlWQQ==", + "license": "MIT" + }, + "node_modules/cli-width": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-4.1.0.tgz", + "integrity": "sha512-ouuZd4/dm2Sw5Gmqy6bGyNNNe1qt9RpmxveLSO7KcgsTnU7RXfsw+/bukWGo1abgBiMAic068rclZsO4IWmmxQ==", + "license": "ISC", + "engines": { + "node": ">= 12" + } + }, "node_modules/debug": { "version": "4.4.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", @@ -568,6 +912,30 @@ "@esbuild/win32-x64": "0.27.2" } }, + "node_modules/fast-string-truncated-width": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/fast-string-truncated-width/-/fast-string-truncated-width-3.0.3.tgz", + "integrity": "sha512-0jjjIEL6+0jag3l2XWWizO64/aZVtpiGE3t0Zgqxv0DPuxiMjvB3M24fCyhZUO4KomJQPj3LTSUnDP3GpdwC0g==", + "license": "MIT" + }, + "node_modules/fast-string-width": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/fast-string-width/-/fast-string-width-3.0.2.tgz", + "integrity": "sha512-gX8LrtNEI5hq8DVUfRQMbr5lpaS4nMIWV+7XEbXk2b8kiQIizgnlr12B4dA3ZEx3308ze0O4Q1R+cHts8kyUJg==", + "license": "MIT", + "dependencies": { + "fast-string-truncated-width": "^3.0.2" + } + }, + "node_modules/fast-wrap-ansi": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/fast-wrap-ansi/-/fast-wrap-ansi-0.2.0.tgz", + "integrity": "sha512-rLV8JHxTyhVmFYhBJuMujcrHqOT2cnO5Zxj37qROj23CP39GXubJRBUFF0z8KFK77Uc0SukZUf7JZhsVEQ6n8w==", + "license": "MIT", + "dependencies": { + "fast-string-width": "^3.0.2" + } + }, "node_modules/file-uri-to-path": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", @@ -603,6 +971,22 @@ "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" } }, + "node_modules/iconv-lite": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz", + "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/mic": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/mic/-/mic-2.1.2.tgz", @@ -617,6 +1001,15 @@ "license": "MIT", "optional": true }, + "node_modules/mute-stream": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-3.0.0.tgz", + "integrity": "sha512-dkEJPVvun4FryqBmZ5KhDo0K9iDXAwn08tMLDinNdRBNPcYEDiWYysLcc6k3mjTMlbP9KyylvRpd4wFtwrT9rw==", + "license": "ISC", + "engines": { + "node": "^20.17.0 || >=22.9.0" + } + }, "node_modules/resolve-pkg-maps": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", @@ -627,6 +1020,24 @@ "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" } }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/speaker": { "version": "0.5.5", "resolved": "https://registry.npmjs.org/speaker/-/speaker-0.5.5.tgz", @@ -681,7 +1092,7 @@ "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", - "dev": true, + "devOptional": true, "license": "MIT" }, "node_modules/yaml": { diff --git a/package.json b/package.json index d4bb54d..8d0ae39 100644 --- a/package.json +++ b/package.json @@ -6,33 +6,13 @@ "private": true, "license": "Apache-2.0", "scripts": { - "apply:dev": "tsx src/apply.ts dev", - "apply:stg": "tsx src/apply.ts stg", - "apply:prod": "tsx src/apply.ts prod", - "apply:dev:force": "tsx src/apply.ts dev --force", - "apply:stg:force": "tsx src/apply.ts stg --force", - "apply:prod:force": "tsx src/apply.ts prod --force", - "push:dev": "tsx src/push.ts dev", - "push:stg": "tsx src/push.ts stg", - "push:prod": "tsx src/push.ts prod", - "push:dev:force": "tsx src/push.ts dev --force", - "push:stg:force": "tsx src/push.ts stg --force", - "push:prod:force": "tsx src/push.ts prod --force", - "pull:dev": "tsx src/pull.ts dev", - "pull:stg": "tsx src/pull.ts stg", - "pull:dev:force": "tsx src/pull.ts dev --force", - "pull:stg:force": "tsx src/pull.ts stg --force", - "pull:prod": "tsx src/pull.ts prod", - "pull:prod:force": "tsx src/pull.ts prod --force", - "pull:dev:bootstrap": "tsx src/pull.ts dev --bootstrap", - "pull:stg:bootstrap": "tsx src/pull.ts stg --bootstrap", - "pull:prod:bootstrap": "tsx src/pull.ts prod --bootstrap", - "call:dev": "tsx src/call.ts dev", - "call:stg": "tsx src/call.ts stg", - "call:prod": "tsx src/call.ts prod", - "cleanup:dev": "tsx src/cleanup.ts dev", - "cleanup:stg": "tsx src/cleanup.ts stg", - "cleanup:prod": "tsx src/cleanup.ts prod", + "setup": "tsx src/setup.ts", + "apply": "tsx src/apply-cmd.ts", + "push": "tsx src/push-cmd.ts", + "pull": "tsx src/pull-cmd.ts", + "call": "tsx src/call-cmd.ts", + "cleanup": "tsx src/cleanup-cmd.ts", + "eval": "tsx src/eval.ts", "mock:webhook": "tsx scripts/mock-vapi-webhook-server.ts", "build": "tsc --noEmit" }, @@ -42,6 +22,7 @@ "typescript": "^5.0.0" }, "dependencies": { + "@inquirer/prompts": "^8.4.1", "yaml": "^2.7.0" }, "optionalDependencies": { diff --git a/resources/dev/simulations/personalities/.gitkeep b/resources/dev/simulations/personalities/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/resources/dev/simulations/scenarios/.gitkeep b/resources/dev/simulations/scenarios/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/resources/dev/simulations/suites/.gitkeep b/resources/dev/simulations/suites/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/resources/dev/simulations/tests/.gitkeep b/resources/dev/simulations/tests/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/resources/dev/squads/.gitkeep b/resources/dev/squads/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/resources/prod/assistants/.gitkeep b/resources/prod/assistants/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/resources/prod/simulations/personalities/.gitkeep b/resources/prod/simulations/personalities/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/resources/prod/simulations/scenarios/.gitkeep b/resources/prod/simulations/scenarios/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/resources/prod/simulations/suites/.gitkeep b/resources/prod/simulations/suites/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/resources/prod/simulations/tests/.gitkeep b/resources/prod/simulations/tests/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/resources/prod/squads/.gitkeep b/resources/prod/squads/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/resources/prod/structuredOutputs/.gitkeep b/resources/prod/structuredOutputs/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/resources/prod/tools/.gitkeep b/resources/prod/tools/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/resources/stg/assistants/.gitkeep b/resources/stg/assistants/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/resources/stg/simulations/personalities/.gitkeep b/resources/stg/simulations/personalities/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/resources/stg/simulations/scenarios/.gitkeep b/resources/stg/simulations/scenarios/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/resources/stg/simulations/suites/.gitkeep b/resources/stg/simulations/suites/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/resources/stg/simulations/tests/.gitkeep b/resources/stg/simulations/tests/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/resources/stg/squads/.gitkeep b/resources/stg/squads/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/resources/stg/structuredOutputs/.gitkeep b/resources/stg/structuredOutputs/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/resources/stg/tools/.gitkeep b/resources/stg/tools/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/src/apply-cmd.ts b/src/apply-cmd.ts new file mode 100644 index 0000000..3d4d6b9 --- /dev/null +++ b/src/apply-cmd.ts @@ -0,0 +1,36 @@ +// Entry point for `npm run apply`. Detects whether an org slug was provided: +// - With slug: forwards to apply.ts (existing non-interactive behavior) +// - Without slug: enters interactive mode (org selection + confirm) + +const SLUG_RE = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/; +const arg = process.argv[2]; +const isSlug = arg && SLUG_RE.test(arg); + +if (isSlug) { + const { runApply } = await import("./apply.ts"); + await runApply().catch((error: unknown) => { + console.error( + "\n❌ Apply failed:", + error instanceof Error ? error.message : error, + ); + process.exit(1); + }); +} else if (!arg) { + const { runInteractiveApply } = await import("./interactive.ts"); + await runInteractiveApply().catch((error: unknown) => { + console.error( + "\n❌ Apply failed:", + error instanceof Error ? error.message : error, + ); + process.exit(1); + }); +} else { + console.error(`❌ Invalid org name: ${arg}`); + console.error( + " Must be lowercase alphanumeric with optional hyphens (e.g., dev, my-org)", + ); + console.error( + " Usage: npm run apply | npm run apply (interactive)", + ); + process.exit(1); +} diff --git a/src/apply.ts b/src/apply.ts index 4f188ba..7071945 100644 --- a/src/apply.ts +++ b/src/apply.ts @@ -1,5 +1,5 @@ import { execSync } from "child_process"; -import { join, dirname } from "path"; +import { join, dirname, resolve } from "path"; import { fileURLToPath } from "url"; // ───────────────────────────────────────────────────────────────────────────── @@ -13,7 +13,7 @@ import { fileURLToPath } from "url"; const __dirname = dirname(fileURLToPath(import.meta.url)); const BASE_DIR = join(__dirname, ".."); -const VALID_ENVIRONMENTS = ["dev", "stg", "prod"] as const; +const SLUG_RE = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/; function runPassthrough(cmd: string): number { try { @@ -24,17 +24,16 @@ function runPassthrough(cmd: string): number { } } -async function main(): Promise { +export async function runApply(): Promise { const env = process.argv[2]; const allArgs = process.argv.slice(3); const hasForce = allArgs.includes("--force"); - // Pull never gets --force (apply's pull should always preserve local changes/deletions) const pullArgs = allArgs.filter(a => a !== "--force").join(" "); const pushArgs = allArgs.join(" "); - if (!env || !VALID_ENVIRONMENTS.includes(env as typeof VALID_ENVIRONMENTS[number])) { - console.error("Usage: npm run apply:dev [--force]"); + if (!env || !SLUG_RE.test(env)) { + console.error("Usage: npm run apply [--force]"); console.error(""); console.error(" Pull → Merge → Push (safe bidirectional sync)"); console.error(""); @@ -54,7 +53,6 @@ async function main(): Promise { } console.log("═══════════════════════════════════════════════════════════════\n"); - // Step 1: Pull (never forced — always preserves local deletions/changes) const pullCmd = `npx tsx src/pull.ts ${env} ${pullArgs}`.trim(); const pullExit = runPassthrough(pullCmd); if (pullExit !== 0) { @@ -62,7 +60,6 @@ async function main(): Promise { process.exit(1); } - // Step 2: Push merged state (--force forwarded here for deletions) console.log("\n🚀 Pushing merged state to platform...\n"); const pushCmd = `npx tsx src/push.ts ${env} ${pushArgs}`.trim(); const pushExit = runPassthrough(pushCmd); @@ -76,7 +73,12 @@ async function main(): Promise { console.log("═══════════════════════════════════════════════════════════════\n"); } -main().catch((error) => { - console.error("\n❌ Apply failed:", error); - process.exit(1); -}); +// Run when executed directly +const isMainModule = + resolve(process.argv[1] ?? "") === resolve(fileURLToPath(import.meta.url)); +if (isMainModule) { + runApply().catch((error) => { + console.error("\n❌ Apply failed:", error); + process.exit(1); + }); +} diff --git a/src/call-cmd.ts b/src/call-cmd.ts new file mode 100644 index 0000000..e38640c --- /dev/null +++ b/src/call-cmd.ts @@ -0,0 +1,36 @@ +// Entry point for `npm run call`. Detects whether an org slug was provided: +// - With slug + flags: forwards to call.ts (existing non-interactive behavior) +// - Without slug: enters interactive mode (org selection + resource picker) + +const SLUG_RE = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/; +const arg = process.argv[2]; +const isSlug = arg && SLUG_RE.test(arg); + +if (isSlug) { + const { runCall } = await import("./call.ts"); + await runCall().catch((error: unknown) => { + console.error( + "\n❌ Call failed:", + error instanceof Error ? error.message : error, + ); + process.exit(1); + }); +} else if (!arg) { + const { runInteractiveCall } = await import("./interactive.ts"); + await runInteractiveCall().catch((error: unknown) => { + console.error( + "\n❌ Call failed:", + error instanceof Error ? error.message : error, + ); + process.exit(1); + }); +} else { + console.error(`❌ Invalid org name: ${arg}`); + console.error( + " Must be lowercase alphanumeric with optional hyphens (e.g., dev, my-org)", + ); + console.error( + " Usage: npm run call -a | npm run call (interactive)", + ); + process.exit(1); +} diff --git a/src/call.ts b/src/call.ts index fdaaf21..c0f9b8f 100644 --- a/src/call.ts +++ b/src/call.ts @@ -1,10 +1,12 @@ import { existsSync, readFileSync } from "fs"; -import { join, dirname } from "path"; +import { join, dirname, resolve } from "path"; import { fileURLToPath } from "url"; import { execSync } from "child_process"; import * as readline from "readline"; +import { createRequire } from "module"; import type { Environment, StateFile } from "./types.ts"; -import { VALID_ENVIRONMENTS } from "./types.ts"; + +const require = createRequire(import.meta.url); // ───────────────────────────────────────────────────────────────────────────── // Configuration @@ -27,18 +29,19 @@ interface CallConfig { // Argument Parsing // ───────────────────────────────────────────────────────────────────────────── +const SLUG_RE = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/; + function printUsage(): void { - console.error("❌ Usage: bun run call: -a "); - console.error(" bun run call: -s "); + console.error("❌ Usage: npm run call -a "); + console.error(" npm run call -s "); console.error(""); console.error(" Options:"); console.error(" -a Call an assistant by name"); console.error(" -s Call a squad by name"); console.error(""); console.error(" Examples:"); - console.error(" bun run call:dev -a my-assistant"); - console.error(" bun run call:dev -a support-assistant"); - console.error(" bun run call:prod -s my-squad"); + console.error(" npm run call my-org -a my-assistant"); + console.error(" npm run call my-org -s my-squad"); } function parseArgs(): CallConfig { @@ -51,9 +54,11 @@ function parseArgs(): CallConfig { const env = args[0] as Environment; - if (!VALID_ENVIRONMENTS.includes(env)) { - console.error(`❌ Invalid environment: ${env}`); - console.error(` Must be one of: ${VALID_ENVIRONMENTS.join(", ")}`); + if (!SLUG_RE.test(env)) { + console.error(`❌ Invalid org name: ${env}`); + console.error( + " Must be lowercase alphanumeric with optional hyphens (e.g., dev, my-org)", + ); process.exit(1); } @@ -203,7 +208,7 @@ async function checkMicrophonePermission(): Promise { " If prompted, please grant microphone permission in System Preferences.", ); console.log( - " System Preferences > Security & Privacy > Privacy > Microphone\n", + " System Settings > Privacy & Security > Microphone\n", ); // Ask user to continue anyway @@ -267,7 +272,7 @@ function loadState(env: Environment): StateFile { if (!existsSync(stateFilePath)) { console.error(`❌ State file not found: .vapi-state.${env}.json`); console.error( - " Run 'npm run apply:" + env + "' first to create resources", + " Run 'npm run apply -- " + env + "' first to create resources", ); process.exit(1); } @@ -466,11 +471,21 @@ async function connectWebSocket( } }; - ws.onmessage = (event) => { - if (event.data instanceof Buffer || event.data instanceof ArrayBuffer) { - // Binary audio data from assistant + ws.onmessage = async (event) => { + const data = event.data; + // Binary audio data from assistant + if (data instanceof Buffer || data instanceof ArrayBuffer) { + if (audioContext) { + audioContext.playAudio(data); + } + } else if (typeof Blob !== "undefined" && data instanceof Blob) { + if (audioContext) { + const arrayBuffer = await data.arrayBuffer(); + audioContext.playAudio(arrayBuffer); + } + } else if (ArrayBuffer.isView(data)) { if (audioContext) { - audioContext.playAudio(event.data); + audioContext.playAudio(data.buffer as ArrayBuffer); } } else { // Control message (JSON) @@ -537,7 +552,20 @@ function handleControlMessage( } case "call-ended": { const cm = message as CallEndedMessage; - console.log(`\n📞 Call ended: ${cm.reason || "unknown reason"}`); + const reasonLabels: Record = { + "silence-timed-out": "Silence timeout (no speech detected)", + "assistant-ended-call": "Assistant ended the call", + "customer-ended-call": "Customer ended the call", + "max-duration-reached": "Maximum call duration reached", + "assistant-error": "Assistant error", + "pipeline-error": "Pipeline error", + "voicemail-reached": "Voicemail detected", + "customer-did-not-answer": "No answer", + "assistant-request-returned-error": "Assistant request error", + "assistant-not-found": "Assistant not found", + }; + const label = cm.reason ? (reasonLabels[cm.reason] ?? cm.reason) : "unknown reason"; + console.log(`\n📞 Call ended: ${label}`); break; } default: @@ -582,23 +610,27 @@ function createAudioContext(): { playAudio: (data: Buffer | ArrayBuffer) => void; close: () => void; } { - // Lazy load speaker module let Speaker: SpeakerConstructor | null = null; let speakerInstance: SpeakerInstance | null = null; try { - // Dynamic import for optional dependency Speaker = require("speaker") as SpeakerConstructor; speakerInstance = new Speaker!({ channels: 1, bitDepth: 16, sampleRate: 16000, }); - } catch { - console.warn( - "⚠️ 'speaker' module not installed. Audio playback disabled.", - ); - console.warn(" Install with: npm install speaker"); + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + if (msg.includes("Cannot find module")) { + console.warn("⚠️ 'speaker' module not installed. Audio playback disabled."); + console.warn(" Install with: npm install speaker"); + } else if (msg.includes("Could not locate the bindings file") || msg.includes("NODE_MODULE_VERSION")) { + console.warn("⚠️ 'speaker' native bindings not built for this Node version."); + console.warn(" Rebuild with: npm rebuild speaker"); + } else { + console.warn(`⚠️ Could not initialize speaker: ${msg}`); + } } return { @@ -645,9 +677,16 @@ function createMicrophoneStream(onData: (data: Buffer) => void): { micInstance!.start(); } catch (error) { - console.warn("⚠️ 'mic' module not installed or microphone unavailable."); - console.warn(" Install with: npm install mic"); - console.warn(" Error:", error); + const msg = error instanceof Error ? error.message : String(error); + if (msg.includes("Cannot find module")) { + console.warn("⚠️ 'mic' module not installed. Microphone input disabled."); + console.warn(" Install with: npm install mic"); + } else if (msg.includes("sox") || msg.includes("rec")) { + console.warn("⚠️ sox/rec not found. Required for microphone input."); + console.warn(" Install with: brew install sox (macOS) or apt install sox (Linux)"); + } else { + console.warn(`⚠️ Could not initialize microphone: ${msg}`); + } } return { @@ -663,14 +702,13 @@ function createMicrophoneStream(onData: (data: Buffer) => void): { // Main // ───────────────────────────────────────────────────────────────────────────── -async function main() { +export async function runCall(): Promise { const config = parseArgs(); console.log(`\n🚀 Starting WebSocket call`); console.log(` Environment: ${config.env}`); console.log(` ${config.resourceType}: ${config.target}\n`); - // Check microphone permissions first const hasPermission = await checkMicrophonePermission(); if (!hasPermission) { console.log("❌ Call cancelled due to microphone permission issues."); @@ -695,7 +733,11 @@ async function main() { await connectWebSocket(call.transport.websocketCallUrl, config); } -main().catch((error) => { - console.error("❌ Fatal error:", error); - process.exit(1); -}); +const isMainModule = + resolve(process.argv[1] ?? "") === resolve(fileURLToPath(import.meta.url)); +if (isMainModule) { + runCall().catch((error) => { + console.error("❌ Fatal error:", error); + process.exit(1); + }); +} diff --git a/src/cleanup-cmd.ts b/src/cleanup-cmd.ts new file mode 100644 index 0000000..6c304d6 --- /dev/null +++ b/src/cleanup-cmd.ts @@ -0,0 +1,36 @@ +// Entry point for `npm run cleanup`. Detects whether an org slug was provided: +// - With slug: forwards to cleanup.ts (existing non-interactive behavior) +// - Without slug: enters interactive mode (org selection + confirm) + +const SLUG_RE = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/; +const arg = process.argv[2]; +const isSlug = arg && SLUG_RE.test(arg); + +if (isSlug) { + const { runCleanup } = await import("./cleanup.ts"); + await runCleanup().catch((error: unknown) => { + console.error( + "\n❌ Cleanup failed:", + error instanceof Error ? error.message : error, + ); + process.exit(1); + }); +} else if (!arg) { + const { runInteractiveCleanup } = await import("./interactive.ts"); + await runInteractiveCleanup().catch((error: unknown) => { + console.error( + "\n❌ Cleanup failed:", + error instanceof Error ? error.message : error, + ); + process.exit(1); + }); +} else { + console.error(`❌ Invalid org name: ${arg}`); + console.error( + " Must be lowercase alphanumeric with optional hyphens (e.g., dev, my-org)", + ); + console.error( + " Usage: npm run cleanup | npm run cleanup (interactive)", + ); + process.exit(1); +} diff --git a/src/cleanup.ts b/src/cleanup.ts index 9d3319c..b4ca89c 100644 --- a/src/cleanup.ts +++ b/src/cleanup.ts @@ -1,3 +1,5 @@ +import { resolve } from "path"; +import { fileURLToPath } from "url"; import { VAPI_ENV, VAPI_BASE_URL, VAPI_TOKEN } from "./config.ts"; import { loadState } from "./state.ts"; @@ -103,7 +105,7 @@ async function main(): Promise { `❌ Refusing to run destructive cleanup without explicit confirmation.`, ); console.error( - ` Re-run with: npm run cleanup:${VAPI_ENV} -- --force --confirm ${VAPI_ENV}`, + ` Re-run with: npm run cleanup -- ${VAPI_ENV} --force --confirm ${VAPI_ENV}`, ); process.exit(1); } @@ -118,6 +120,7 @@ async function main(): Promise { ...Object.values(state.scenarios), ...Object.values(state.simulations), ...Object.values(state.simulationSuites), + ...Object.values(state.evals), ]); // A state file with zero tracked resources is almost always a fresh clone, @@ -129,7 +132,7 @@ async function main(): Promise { ); console.error( ` This usually means the state was never bootstrapped. Run ` + - `\`npm run pull:${VAPI_ENV}:bootstrap\` first, then retry.`, + `\`npm run pull -- ${VAPI_ENV} --bootstrap\` first, then retry.`, ); process.exit(1); } @@ -239,7 +242,7 @@ async function main(): Promise { ); console.log("🔒 DRY-RUN MODE - No resources were deleted"); console.log(" To actually delete, run:"); - console.log(` npm run cleanup:${VAPI_ENV} -- --force`); + console.log(` npm run cleanup -- ${VAPI_ENV} --force`); console.log( "═══════════════════════════════════════════════════════════════\n", ); @@ -271,7 +274,13 @@ async function main(): Promise { ); } -main().catch((error) => { - console.error("\n❌ Cleanup failed:", error); - process.exit(1); -}); +export { main as runCleanup }; + +const isMainModule = + resolve(process.argv[1] ?? "") === resolve(fileURLToPath(import.meta.url)); +if (isMainModule) { + main().catch((error) => { + console.error("\n❌ Cleanup failed:", error); + process.exit(1); + }); +} diff --git a/src/config.ts b/src/config.ts index fa5a41b..24b887f 100644 --- a/src/config.ts +++ b/src/config.ts @@ -2,7 +2,7 @@ import { existsSync, readFileSync } from "fs"; import { join, basename, dirname, resolve, relative } from "path"; import { fileURLToPath } from "url"; import type { Environment, ResourceType } from "./types.ts"; -import { VALID_ENVIRONMENTS, VALID_RESOURCE_TYPES } from "./types.ts"; +import { VALID_RESOURCE_TYPES } from "./types.ts"; // ───────────────────────────────────────────────────────────────────────────── // CLI Argument Parsing @@ -32,23 +32,27 @@ const RESOURCE_PATH_MAP: Record = { "simulations/suites": "simulationSuites", }; +const SLUG_RE = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/; + function parseEnvironment(): Environment { - const envArg = process.argv[2] as Environment | undefined; + const envArg = process.argv[2]; if (!envArg) { - console.error("❌ Environment argument is required"); - console.error(" Usage: npm run apply:dev | apply:stg | apply:prod"); + console.error("❌ Environment / org name argument is required"); + console.error(" Usage: npm run push -- "); console.error(" Flags: --force (enable deletions)"); console.error( - " --type (apply only specific resource type)", + " --type (apply only specific resource type, repeatable)", ); console.error(" -- (apply only specific files)"); process.exit(1); } - if (!VALID_ENVIRONMENTS.includes(envArg)) { - console.error(`❌ Invalid environment: ${envArg}`); - console.error(` Must be one of: ${VALID_ENVIRONMENTS.join(", ")}`); + if (!SLUG_RE.test(envArg)) { + console.error(`❌ Invalid environment / org name: ${envArg}`); + console.error( + " Must be lowercase alphanumeric with optional hyphens (e.g., dev, my-org)", + ); process.exit(1); } @@ -94,46 +98,50 @@ function parseFlags(): { applyFilter: {}, }; - // Parse --type or -t flag - const typeIndex = args.findIndex((a) => a === "--type" || a === "-t"); - if (typeIndex !== -1 && args[typeIndex + 1]) { - const typeArg = args[typeIndex + 1]!; - const resolved = resolveResourceTypes(typeArg); - if (!resolved) { - console.error(`❌ Invalid resource type: ${typeArg}`); - console.error(` Must be one of: ${VALID_TYPE_ARGS.join(", ")}`); - process.exit(1); - } - result.applyFilter.resourceTypes = resolved; - } - const resourceIds: string[] = []; - - // Parse file paths and positional resource types const filePaths: string[] = []; + for (let i = 0; i < args.length; i++) { const arg = args[i]; if (!arg) continue; - // Skip flags and their values - if ( - arg === "--force" || - arg === "--bootstrap" || - arg === "--id" || - arg === "--type" || - arg === "-t" - ) { - if (arg === "--type" || arg === "-t" || arg === "--id") i++; // skip the value too + + if (arg === "--force" || arg === "--bootstrap") continue; + + // --type / -t (repeatable): accumulate resource types + if ((arg === "--type" || arg === "-t") && args[i + 1]) { + const typeArg = args[i + 1]!; + const resolved = resolveResourceTypes(typeArg); + if (!resolved) { + console.error(`❌ Invalid resource type: ${typeArg}`); + console.error(` Must be one of: ${VALID_TYPE_ARGS.join(", ")}`); + process.exit(1); + } + if (!result.applyFilter.resourceTypes) { + result.applyFilter.resourceTypes = []; + } + result.applyFilter.resourceTypes.push(...resolved); + i++; continue; } - // Check if it's a resource type or group (positional) - if (!result.applyFilter.resourceTypes) { - const resolved = resolveResourceTypes(arg); - if (resolved) { - result.applyFilter.resourceTypes = resolved; - continue; + + // --id (repeatable) + if (arg === "--id" && args[i + 1]) { + resourceIds.push(args[i + 1]!); + i++; + continue; + } + + // Positional resource type / group + const resolved = resolveResourceTypes(arg); + if (resolved) { + if (!result.applyFilter.resourceTypes) { + result.applyFilter.resourceTypes = []; } + result.applyFilter.resourceTypes.push(...resolved); + continue; } - // If it looks like a file path (contains / or ends with .yml/.yaml/.md/.ts) + + // File path if (arg.includes("/") || /\.(yml|yaml|md|ts)$/.test(arg)) { filePaths.push(arg); } @@ -142,15 +150,6 @@ function parseFlags(): { if (filePaths.length > 0) { result.applyFilter.filePaths = filePaths; } - - for (let i = 0; i < args.length; i++) { - const arg = args[i]; - if (arg === "--id" && args[i + 1]) { - resourceIds.push(args[i + 1]!); - i++; - } - } - if (resourceIds.length > 0) { result.applyFilter.resourceIds = resourceIds; } @@ -246,6 +245,7 @@ export const UPDATE_EXCLUDED_KEYS: Record = { scenarios: [], simulations: [], simulationSuites: [], + evals: ["type"], }; export function removeExcludedKeys( diff --git a/src/delete.ts b/src/delete.ts index a226cbe..938fbf5 100644 --- a/src/delete.ts +++ b/src/delete.ts @@ -103,6 +103,7 @@ const DELETE_ENDPOINT_MAP: Record = { scenarios: "/eval/simulation/scenario", simulations: "/eval/simulation", simulationSuites: "/eval/simulation/suite", + evals: "/eval", }; // Map display type back to ReferenceableType for reference checking @@ -115,6 +116,7 @@ const REFERENCEABLE_TYPE_MAP: Record = { "simulation": "simulations", "simulation suite": null, // not referenceable by others "squad": null, // not referenceable by others + "eval": null, // not referenceable by others }; export async function deleteOrphanedResources( @@ -150,9 +152,13 @@ export async function deleteOrphanedResources( const orphanedSimulationSuites = shouldCheck("simulationSuites") ? findOrphanedResources(loadedResources.simulationSuites.map((s) => s.resourceId), state.simulationSuites) : []; + const orphanedEvals = shouldCheck("evals") + ? findOrphanedResources(loadedResources.evals.map((e) => e.resourceId), state.evals) + : []; // Collect all orphaned resources (in reverse dependency order for deletion) const allOrphaned = [ + ...orphanedEvals.map((r) => ({ ...r, type: "eval" as const, stateKey: "evals" as ResourceType })), ...orphanedSimulationSuites.map((r) => ({ ...r, type: "simulation suite" as const, stateKey: "simulationSuites" as ResourceType })), ...orphanedSimulations.map((r) => ({ ...r, type: "simulation" as const, stateKey: "simulations" as ResourceType })), ...orphanedScenarios.map((r) => ({ ...r, type: "scenario" as const, stateKey: "scenarios" as ResourceType })), @@ -214,7 +220,7 @@ export async function deleteOrphanedResources( } console.log(" ℹ️ These resources exist in Vapi but not in your local files."); console.log(" ℹ️ To delete them, run with --force flag:"); - console.log(" npm run apply:dev:force\n"); + console.log(" npm run push -- --force\n"); return; } diff --git a/src/eval.ts b/src/eval.ts new file mode 100644 index 0000000..7967060 --- /dev/null +++ b/src/eval.ts @@ -0,0 +1,568 @@ +import { existsSync, readFileSync, statSync } from "fs"; +import { join, dirname, basename, isAbsolute } from "path"; +import { fileURLToPath } from "url"; +import { parse as parseYaml } from "yaml"; +import { readFile } from "fs/promises"; +import type { Environment, StateFile } from "./types.ts"; + +// ───────────────────────────────────────────────────────────────────────────── +// Configuration +// ───────────────────────────────────────────────────────────────────────────── + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const BASE_DIR = join(__dirname, ".."); + +function resourcesDir(env: string): string { + return join(BASE_DIR, "resources", env); +} + +const POLL_INTERVAL_MS = 3000; +const POLL_TIMEOUT_MS = 180_000; + +// ───────────────────────────────────────────────────────────────────────────── +// Argument Parsing +// ───────────────────────────────────────────────────────────────────────────── + +interface EvalConfig { + env: Environment; + token: string; + baseUrl: string; + variablesFile?: string; + squadName?: string; + assistantName?: string; + evalFilter?: string; +} + +function printUsage(): void { + console.error("Usage: tsx src/eval.ts -s [options]"); + console.error(" tsx src/eval.ts -a [options]"); + console.error(""); + console.error("Runs Vapi Evals (mock conversation tests) against a transient or stored assistant/squad."); + console.error("Evals must be pushed first (npm run push -- evals). Assistants/squads can be transient."); + console.error(""); + console.error("Options:"); + console.error(" -s Target squad (by resource filename, loaded as transient)"); + console.error(" -a Assistant: resource id, or path to .md/.yml (cwd or repo root)"); + console.error(" -v Variable values JSON file (default: eval-variables.json)"); + console.error(" --filter Run only evals matching this substring"); + console.error(" --stored Use stored assistantId/squadId from state instead of transient"); + console.error(""); + console.error("Examples:"); + console.error(" tsx src/eval.ts dev -s everblue-voice-squad-20374c37"); + console.error(" tsx src/eval.ts dev -a everblue-main-agent-633ab678 --filter name-collection"); + console.error(" tsx src/eval.ts dev -a resources/assistants/qa-address-resolution-tester-e9ed5d49.md"); + console.error(" tsx src/eval.ts dev -s everblue-voice-squad-20374c37 --stored"); +} + +function parseArgs(): EvalConfig & { useStored: boolean } { + const args = process.argv.slice(2); + if (args.length < 3) { + printUsage(); + process.exit(1); + } + + const SLUG_RE = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/; + const env = args[0] as Environment; + if (!env || !SLUG_RE.test(env)) { + console.error(`❌ Invalid org name: ${env}`); + console.error( + " Must be lowercase alphanumeric with optional hyphens (e.g., dev, my-org)", + ); + process.exit(1); + } + + let squadName: string | undefined; + let assistantName: string | undefined; + let variablesFile: string | undefined; + let evalFilter: string | undefined; + let useStored = false; + + for (let i = 1; i < args.length; i++) { + const arg = args[i]; + if (arg === "-s" || arg === "--squad") { squadName = args[++i]; } + else if (arg === "-a" || arg === "--assistant") { assistantName = args[++i]; } + else if (arg === "-v" || arg === "--variables") { variablesFile = args[++i]; } + else if (arg === "--filter") { evalFilter = args[++i]; } + else if (arg === "--stored") { useStored = true; } + } + + if (!squadName && !assistantName) { + console.error("❌ Must specify -s or -a "); + printUsage(); + process.exit(1); + } + + const { token, baseUrl } = loadEnvFile(env); + return { env, token, baseUrl, variablesFile, squadName, assistantName, evalFilter, useStored }; +} + +function loadEnvFile(env: string): { token: string; baseUrl: string } { + const envFiles = [ + join(BASE_DIR, `.env.${env}`), + join(BASE_DIR, `.env.${env}.local`), + join(BASE_DIR, ".env.local"), + ]; + const envVars: Record = {}; + for (const envFile of envFiles) { + if (!existsSync(envFile)) continue; + for (const line of readFileSync(envFile, "utf-8").split("\n")) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith("#")) continue; + const eqIndex = trimmed.indexOf("="); + if (eqIndex === -1) continue; + const key = trimmed.slice(0, eqIndex).trim(); + let value = trimmed.slice(eqIndex + 1).trim(); + if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) { + value = value.slice(1, -1); + } + if (envVars[key] === undefined) envVars[key] = value; + } + } + const token = process.env.VAPI_TOKEN || envVars.VAPI_TOKEN; + const baseUrl = process.env.VAPI_BASE_URL || envVars.VAPI_BASE_URL || "https://api.vapi.ai"; + if (!token) { + console.error(`❌ VAPI_TOKEN not found. Create .env.${env} with VAPI_TOKEN=your-token`); + process.exit(1); + } + return { token, baseUrl }; +} + +// ───────────────────────────────────────────────────────────────────────────── +// State & Resource Loading +// ───────────────────────────────────────────────────────────────────────────── + +function loadState(env: Environment): StateFile { + const stateFile = join(BASE_DIR, `.vapi-state.${env}.json`); + if (!existsSync(stateFile)) { + console.error(`❌ State file not found: .vapi-state.${env}.json`); + console.error(" Run 'npm run push -- evals' first to create eval resources"); + process.exit(1); + } + const content = readFileSync(stateFile, "utf-8"); + const state = JSON.parse(content) as StateFile; + if (!state.evals) state.evals = {}; + return state; +} + +function parseFrontmatter(content: string): { config: Record; body: string } { + const match = content.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n?([\s\S]*)$/); + if (!match || !match[1]) throw new Error("Invalid frontmatter format"); + return { config: parseYaml(match[1]) as Record, body: (match[2] ?? "").trim() }; +} + +function assistantModelInjectMarkdownSystem(config: Record, body: string): void { + if (!body) return; + const model = (config.model as Record) || {}; + const existing = Array.isArray(model.messages) ? model.messages : []; + model.messages = [{ role: "system", content: body }, ...existing.filter((m: { role?: string }) => m.role !== "system")]; + config.model = model; +} + +async function loadAssistantFromFilePath(filePath: string): Promise> { + const lower = filePath.toLowerCase(); + if (lower.endsWith(".md")) { + const { config, body } = parseFrontmatter(await readFile(filePath, "utf-8")); + assistantModelInjectMarkdownSystem(config, body); + return config; + } + if (lower.endsWith(".yml") || lower.endsWith(".yaml")) { + return parseYaml(await readFile(filePath, "utf-8")) as Record; + } + throw new Error(`Unsupported assistant file (use .md, .yml, .yaml): ${filePath}`); +} + +/** True when -a value should be tried as a filesystem path before resources/assistants/. */ +function assistantArgLooksLikeFilePath(arg: string): boolean { + if (isAbsolute(arg)) return true; + if (arg.startsWith("./") || arg.startsWith("../")) return true; + if (arg.includes("/") || arg.includes("\\")) return true; + const lower = arg.toLowerCase(); + return lower.endsWith(".md") || lower.endsWith(".yml") || lower.endsWith(".yaml"); +} + +/** Resolves a path-like -a argument to an existing file, or undefined to use resource-name flow. */ +function assistantArgResolveExistingFile(arg: string): string | undefined { + if (!assistantArgLooksLikeFilePath(arg)) return undefined; + const candidates: string[] = []; + if (isAbsolute(arg)) { + candidates.push(arg); + } else { + candidates.push(join(BASE_DIR, arg)); + candidates.push(join(process.cwd(), arg)); + } + for (const p of candidates) { + if (!existsSync(p)) continue; + try { + if (statSync(p).isFile()) return p; + } catch { + /* broken symlink etc. */ + } + } + return undefined; +} + +async function loadAssistant(name: string, env: string): Promise> { + const dir = resourcesDir(env); + const mdPath = join(dir, "assistants", `${name}.md`); + if (existsSync(mdPath)) return loadAssistantFromFilePath(mdPath); + const ymlPath = join(dir, "assistants", `${name}.yml`); + if (existsSync(ymlPath)) return loadAssistantFromFilePath(ymlPath); + throw new Error(`Assistant not found: ${name}`); +} + +async function loadAssistantForEvalTarget(arg: string, env: string): Promise<{ config: Record; sourcePath?: string }> { + const resolved = assistantArgResolveExistingFile(arg); + if (resolved) { + return { config: await loadAssistantFromFilePath(resolved), sourcePath: resolved }; + } + if (assistantArgLooksLikeFilePath(arg)) { + throw new Error( + `Assistant file not found: ${arg} (tried ${join(BASE_DIR, arg)} and ${join(process.cwd(), arg)})`, + ); + } + return { config: await loadAssistant(arg, env) }; +} + +async function loadSquad(name: string, env: string): Promise> { + const filePath = join(resourcesDir(env), "squads", `${name}.yml`); + if (!existsSync(filePath)) throw new Error(`Squad not found: ${filePath}`); + return parseYaml(await readFile(filePath, "utf-8")) as Record; +} + +function loadVariables(config: EvalConfig): Record | undefined { + const candidates = [config.variablesFile, "eval-variables.json", "resources/eval-variables.json"].filter(Boolean) as string[]; + for (const f of candidates) { + const resolved = f.startsWith("/") ? f : join(BASE_DIR, f); + if (existsSync(resolved)) { + console.log(`📋 Loading variables: ${basename(resolved)}`); + const raw = JSON.parse(readFileSync(resolved, "utf-8")); + return raw.squadOverrides?.variableValues ?? raw.assistantOverrides?.variableValues ?? raw.variableValues ?? raw; + } + } + return undefined; +} + +// ───────────────────────────────────────────────────────────────────────────── +// Reference Resolution +// ───────────────────────────────────────────────────────────────────────────── + +const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +function resolveId(id: string, stateSection: Record): string { + const clean = id.split("##")[0]?.trim() ?? ""; + if (UUID_RE.test(clean)) return clean; + return stateSection[clean] ?? clean; +} + +function resolveAssistantConfig(config: Record, state: StateFile): Record { + const resolved = JSON.parse(JSON.stringify(config)) as Record; + const model = resolved.model as Record | undefined; + if (model && Array.isArray(model.toolIds)) { + model.toolIds = (model.toolIds as string[]).map(id => resolveId(id, state.tools)); + } + const ap = resolved.artifactPlan as Record | undefined; + if (ap && Array.isArray(ap.structuredOutputIds)) { + ap.structuredOutputIds = (ap.structuredOutputIds as string[]).map(id => resolveId(id, state.structuredOutputs)); + } + if (Array.isArray(resolved.hooks)) { + for (const hook of resolved.hooks as Record[]) { + if (Array.isArray(hook.do)) { + for (const action of hook.do as Record[]) { + if (typeof action.toolId === "string" && !UUID_RE.test(action.toolId)) { + action.toolId = resolveId(action.toolId, state.tools); + } + } + } + } + } + // Resolve credentials + const credMap = new Map(Object.entries(state.credentials)); + if (credMap.size > 0) return deepReplace(resolved, credMap) as Record; + return resolved; +} + +async function resolveSquadConfig(config: Record, state: StateFile, expandTransient: boolean, env: string): Promise> { + const resolved = JSON.parse(JSON.stringify(config)) as Record; + if (Array.isArray(resolved.members)) { + for (const member of resolved.members as Record[]) { + if (typeof member.assistantId === "string") { + const localId = member.assistantId.split("##")[0]?.trim() ?? ""; + if (expandTransient && !UUID_RE.test(localId)) { + try { + const assistantConfig = await loadAssistant(localId, env); + delete member.assistantId; + member.assistant = resolveAssistantConfig(assistantConfig, state); + } catch { member.assistantId = resolveId(localId, state.assistants); } + } else { + member.assistantId = resolveId(localId, state.assistants); + } + } + const overrides = member.assistantOverrides as Record | undefined; + const toolsAppend = overrides?.["tools:append"] as Record[] | undefined; + if (Array.isArray(toolsAppend)) { + for (const tool of toolsAppend) { + if (Array.isArray(tool.destinations)) { + for (const dest of tool.destinations as Record[]) { + if (typeof dest.assistantId === "string" && !UUID_RE.test(dest.assistantId)) { + dest.assistantId = resolveId(dest.assistantId, state.assistants); + } + } + } + } + } + } + } + const credMap = new Map(Object.entries(state.credentials)); + if (credMap.size > 0) return deepReplace(resolved, credMap) as Record; + return resolved; +} + +function deepReplace(value: unknown, map: Map): unknown { + if (typeof value === "string") return map.get(value) ?? value; + if (Array.isArray(value)) return value.map(v => deepReplace(v, map)); + if (value && typeof value === "object") { + const result: Record = {}; + for (const [k, v] of Object.entries(value as Record)) result[k] = deepReplace(v, map); + return result; + } + return value; +} + +// ───────────────────────────────────────────────────────────────────────────── +// Eval Loading — reads resources/evals/*.yml and resolves to platform UUIDs +// ───────────────────────────────────────────────────────────────────────────── + +interface EvalDefinition { + resourceId: string; + evalId: string; // platform UUID from state + name: string; +} + +function loadEvals(state: StateFile, filter?: string): EvalDefinition[] { + const evalState = state.evals ?? {}; + const evals: EvalDefinition[] = []; + + for (const [resourceId, uuid] of Object.entries(evalState)) { + if (filter && !resourceId.toLowerCase().includes(filter.toLowerCase())) continue; + evals.push({ resourceId, evalId: uuid, name: resourceId }); + } + + return evals; +} + +// ───────────────────────────────────────────────────────────────────────────── +// Vapi API +// ───────────────────────────────────────────────────────────────────────────── + +interface EvalRunResult { + id?: string; + evalRunId?: string; + status?: string; + endedReason?: string; + endedMessage?: string; + results?: Array<{ + status?: string; + failureReason?: string; + [key: string]: unknown; + }>; + cost?: number; + error?: string; + [key: string]: unknown; +} + +async function apiRequest(config: EvalConfig, method: string, endpoint: string, body?: unknown): Promise { + const url = `${config.baseUrl}${endpoint}`; + const response = await fetch(url, { + method, + headers: { Authorization: `Bearer ${config.token}`, "Content-Type": "application/json" }, + ...(body ? { body: JSON.stringify(body) } : {}), + }); + if (!response.ok) { + const text = await response.text(); + throw new Error(`API ${method} ${endpoint} → ${response.status}: ${text}`); + } + return response.json(); +} + +function sleep(ms: number): Promise { + return new Promise(r => setTimeout(r, ms)); +} + +async function createEvalRun(config: EvalConfig, evalId: string, target: Record): Promise { + const body = { + type: "eval", + evalId, + target, + }; + const result = await apiRequest(config, "POST", "/eval/run", body) as EvalRunResult; + const runId = result.evalRunId ?? result.id; + if (typeof result.error === "string" && result.error) { + throw new Error(result.error); + } + if (!runId) { + throw new Error( + `POST /eval/run returned no evalRunId (keys: ${Object.keys(result).join(", ")})`, + ); + } + return runId; +} + +async function pollEvalRun(config: EvalConfig, runId: string): Promise { + const start = Date.now(); + while (Date.now() - start < POLL_TIMEOUT_MS) { + await sleep(POLL_INTERVAL_MS); + const result = await apiRequest(config, "GET", `/eval/run/${runId}`) as EvalRunResult; + if (result.status === "ended") return result; + process.stdout.write("."); + } + throw new Error(`Eval run ${runId} timed out after ${POLL_TIMEOUT_MS / 1000}s`); +} + +// ───────────────────────────────────────────────────────────────────────────── +// Main +// ───────────────────────────────────────────────────────────────────────────── + +async function main(): Promise { + const config = parseArgs(); + const state = loadState(config.env); + const variableValues = loadVariables(config); + + console.log("═══════════════════════════════════════════════════════════════"); + console.log(`🧪 Vapi GitOps Eval Runner — Environment: ${config.env}`); + console.log(` API: ${config.baseUrl}`); + if (config.squadName) console.log(` Squad: ${config.squadName}${config.useStored ? " (stored)" : " (transient)"}`); + if (config.assistantName) console.log(` Assistant: ${config.assistantName}${config.useStored ? " (stored)" : " (transient)"}`); + if (variableValues) console.log(` Variables: ${Object.keys(variableValues).length} keys`); + if (config.evalFilter) console.log(` Filter: "${config.evalFilter}"`); + console.log("═══════════════════════════════════════════════════════════════\n"); + + // Build the target (transient or stored) + let target: Record; + + if (config.squadName) { + if (config.useStored) { + const squadId = state.squads[config.squadName]; + if (!squadId) { + console.error(`❌ Squad not found in state: ${config.squadName}`); + console.error(" Available: " + Object.keys(state.squads).join(", ")); + process.exit(1); + } + target = { + type: "squad", + squadId, + ...(variableValues ? { assistantOverrides: { variableValues } } : {}), + }; + } else { + console.log("📂 Loading squad as transient config...\n"); + const squadConfig = await loadSquad(config.squadName, config.env); + const resolved = await resolveSquadConfig(squadConfig, state, true, config.env); + target = { + type: "squad", + squad: resolved, + ...(variableValues ? { assistantOverrides: { variableValues } } : {}), + }; + } + } else { + if (config.useStored) { + const assistantId = state.assistants[config.assistantName!]; + if (!assistantId) { + console.error(`❌ Assistant not found in state: ${config.assistantName}`); + process.exit(1); + } + target = { + type: "assistant", + assistantId, + ...(variableValues ? { assistantOverrides: { variableValues } } : {}), + }; + } else { + const { config: assistantConfig, sourcePath } = await loadAssistantForEvalTarget(config.assistantName!, config.env); + console.log( + sourcePath + ? `📂 Loading assistant from file: ${sourcePath}\n` + : "📂 Loading assistant as transient config...\n", + ); + const resolved = resolveAssistantConfig(assistantConfig, state); + target = { + type: "assistant", + assistant: resolved, + ...(variableValues ? { assistantOverrides: { variableValues } } : {}), + }; + } + } + + // Load eval definitions from state (they must be pushed first) + const evals = loadEvals(state, config.evalFilter); + if (evals.length === 0) { + console.error("❌ No evals found in state" + (config.evalFilter ? ` matching "${config.evalFilter}"` : "")); + console.error(" Push evals first: npm run push -- evals"); + console.error(" Eval files go in: resources/evals/"); + process.exit(1); + } + + console.log(`📋 Running ${evals.length} eval(s)...\n`); + + // Run each eval + const results: Array<{ eval: string; runId: string; passed: boolean; failureReason?: string; cost?: number }> = []; + + for (const evalDef of evals) { + process.stdout.write(` 🧪 ${evalDef.name} `); + try { + const runId = await createEvalRun(config, evalDef.evalId, target); + process.stdout.write(`[${runId}] `); + + const result = await pollEvalRun(config, runId); + const allPassed = (result.results ?? []).every(r => r.status === "pass"); + const passed = result.endedReason === "mockConversation.done" && allPassed; + + results.push({ + eval: evalDef.name, + runId, + passed, + failureReason: !passed ? (result.endedMessage ?? result.endedReason) : undefined, + cost: result.cost as number | undefined, + }); + + if (passed) { + console.log(" ✅ PASS"); + } else { + console.log(" ❌ FAIL"); + if (result.endedMessage) console.log(` Reason: ${result.endedMessage}`); + for (const r of result.results ?? []) { + if (r.status === "fail") { + console.log(` → ${r.failureReason || JSON.stringify(r)}`); + } + } + } + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + console.log(` ❌ ERROR: ${msg}`); + results.push({ eval: evalDef.name, runId: "n/a", passed: false, failureReason: msg }); + } + } + + // Summary + const passed = results.filter(r => r.passed).length; + const failed = results.length - passed; + const totalCost = results.reduce((sum, r) => sum + (r.cost ?? 0), 0); + + console.log("\n═══════════════════════════════════════════════════════════════"); + console.log(`📊 Results: ${passed}/${results.length} passed, ${failed} failed`); + if (totalCost > 0) console.log(`💰 Total cost: $${totalCost.toFixed(4)}`); + console.log("═══════════════════════════════════════════════════════════════\n"); + + if (failed > 0) { + console.log("❌ Failed evals:"); + for (const r of results.filter(r => !r.passed)) { + console.log(` - ${r.eval}: ${r.failureReason || "unknown"}`); + } + console.log("\n💡 Fix the issues before pushing assistant/squad changes."); + process.exit(1); + } + + console.log("✅ All evals passed! Safe to push assistant/squad changes."); +} + +main().catch((error) => { + console.error("\n❌ Eval failed:", error instanceof Error ? error.message : error); + process.exit(1); +}); diff --git a/src/interactive.ts b/src/interactive.ts new file mode 100644 index 0000000..bb9520c --- /dev/null +++ b/src/interactive.ts @@ -0,0 +1,1018 @@ +import { execSync } from "child_process"; +import { existsSync, readdirSync, readFileSync, statSync } from "fs"; +import { join, dirname, relative, extname } from "path"; +import { fileURLToPath } from "url"; +import { select, confirm } from "@inquirer/prompts"; +import searchableCheckbox, { BACK_SENTINEL } from "./searchableCheckbox.js"; +import type { StateFile } from "./types.ts"; + +// ───────────────────────────────────────────────────────────────────────────── +// Constants +// ───────────────────────────────────────────────────────────────────────────── + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const BASE_DIR = join(__dirname, ".."); + +const SLUG_RE = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/; + +interface ResourceTypeDef { + key: string; + label: string; + endpoint: string; + folder: string; +} + +const RESOURCE_TYPES: ResourceTypeDef[] = [ + { + key: "assistants", + label: "Assistants", + endpoint: "/assistant", + folder: "assistants", + }, + { key: "tools", label: "Tools", endpoint: "/tool", folder: "tools" }, + { key: "squads", label: "Squads", endpoint: "/squad", folder: "squads" }, + { + key: "structuredOutputs", + label: "Structured Outputs", + endpoint: "/structured-output", + folder: "structuredOutputs", + }, + { + key: "personalities", + label: "Personalities", + endpoint: "/eval/simulation/personality", + folder: "simulations/personalities", + }, + { + key: "scenarios", + label: "Scenarios", + endpoint: "/eval/simulation/scenario", + folder: "simulations/scenarios", + }, + { + key: "simulations", + label: "Simulations", + endpoint: "/eval/simulation", + folder: "simulations/tests", + }, + { + key: "simulationSuites", + label: "Simulation Suites", + endpoint: "/eval/simulation/suite", + folder: "simulations/suites", + }, + { + key: "evals", + label: "Evals", + endpoint: "/eval", + folder: "evals", + }, +]; + +// ───────────────────────────────────────────────────────────────────────────── +// ANSI helpers +// ───────────────────────────────────────────────────────────────────────────── + +const c = { + bold: (s: string) => `\x1b[1m${s}\x1b[0m`, + dim: (s: string) => `\x1b[2m${s}\x1b[0m`, + green: (s: string) => `\x1b[32m${s}\x1b[0m`, + yellow: (s: string) => `\x1b[33m${s}\x1b[0m`, + red: (s: string) => `\x1b[31m${s}\x1b[0m`, + cyan: (s: string) => `\x1b[36m${s}\x1b[0m`, +}; + +function isBack(result: string[]): boolean { + return result.length === 1 && result[0] === BACK_SENTINEL; +} + +// ───────────────────────────────────────────────────────────────────────────── +// Org Detection +// ───────────────────────────────────────────────────────────────────────────── + +interface OrgInfo { + slug: string; + hasEnv: boolean; + hasResources: boolean; +} + +function detectOrgs(): OrgInfo[] { + const slugs = new Map(); + + // Scan .env.* files + const baseEntries = readdirSync(BASE_DIR); + for (const entry of baseEntries) { + const match = entry.match(/^\.env\.(.+)$/); + if (!match) continue; + const slug = match[1]!; + if (slug === "example" || slug === "local" || slug.endsWith(".local")) + continue; + if (!SLUG_RE.test(slug)) continue; + if (!slugs.has(slug)) + slugs.set(slug, { slug, hasEnv: false, hasResources: false }); + slugs.get(slug)!.hasEnv = true; + } + + // Scan resources/ directories + const resourcesDir = join(BASE_DIR, "resources"); + if (existsSync(resourcesDir)) { + for (const entry of readdirSync(resourcesDir, { withFileTypes: true })) { + if (!entry.isDirectory()) continue; + if (!SLUG_RE.test(entry.name)) continue; + if (!slugs.has(entry.name)) + slugs.set(entry.name, { + slug: entry.name, + hasEnv: false, + hasResources: false, + }); + slugs.get(entry.name)!.hasResources = true; + } + } + + return [...slugs.values()].sort((a, b) => a.slug.localeCompare(b.slug)); +} + +// ───────────────────────────────────────────────────────────────────────────── +// Env File Loading +// ───────────────────────────────────────────────────────────────────────────── + +function loadOrgEnv(slug: string): { token: string; baseUrl: string } { + const envPath = join(BASE_DIR, `.env.${slug}`); + if (!existsSync(envPath)) { + throw new Error( + `No .env.${slug} file found. Run "npm run setup" first to configure this org.`, + ); + } + + const vars: Record = {}; + const content = readFileSync(envPath, "utf-8"); + for (const line of content.split("\n")) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith("#")) continue; + const eq = trimmed.indexOf("="); + if (eq === -1) continue; + let val = trimmed.slice(eq + 1).trim(); + if ( + (val.startsWith('"') && val.endsWith('"')) || + (val.startsWith("'") && val.endsWith("'")) + ) { + val = val.slice(1, -1); + } + vars[trimmed.slice(0, eq).trim()] = val; + } + + const token = vars.VAPI_TOKEN; + if (!token) { + throw new Error( + `.env.${slug} is missing VAPI_TOKEN. Run "npm run setup" to fix.`, + ); + } + + return { + token, + baseUrl: vars.VAPI_BASE_URL || "https://api.vapi.ai", + }; +} + +// ───────────────────────────────────────────────────────────────────────────── +// Org Selection Prompt +// ───────────────────────────────────────────────────────────────────────────── + +async function selectOrg(action: string): Promise { + const orgs = detectOrgs(); + + if (orgs.length === 0) { + console.error( + c.red( + '\n No configured orgs found. Run "npm run setup" to add one.\n', + ), + ); + process.exit(1); + } + + if (orgs.length === 1) { + const org = orgs[0]!; + console.log(c.dim(` Using org: ${org.slug}\n`)); + return org.slug; + } + + const slug = await select({ + message: `Select org to ${action}`, + choices: orgs.map((org) => { + const tags: string[] = []; + if (org.hasEnv) tags.push("env"); + if (org.hasResources) tags.push("resources"); + return { + name: `${org.slug} ${c.dim(`(${tags.join(", ")})`)}`, + value: org.slug, + }; + }), + }); + + return slug; +} + +// ───────────────────────────────────────────────────────────────────────────── +// API Client +// ───────────────────────────────────────────────────────────────────────────── + +async function apiGet( + token: string, + baseUrl: string, + endpoint: string, +): Promise { + const response = await fetch(`${baseUrl}${endpoint}`, { + method: "GET", + headers: { Authorization: `Bearer ${token}` }, + }); + if (!response.ok) { + const text = await response.text(); + throw new Error( + `API GET ${endpoint} failed (${response.status}): ${text}`, + ); + } + return response.json(); +} + +function normaliseList(data: unknown): Record[] { + if (Array.isArray(data)) return data as Record[]; + if ( + data && + typeof data === "object" && + "results" in data && + Array.isArray((data as Record).results) + ) { + return (data as Record).results as Record< + string, + unknown + >[]; + } + return []; +} + +// ───────────────────────────────────────────────────────────────────────────── +// Display helpers +// ───────────────────────────────────────────────────────────────────────────── + +function resourceDisplayName(r: Record): string { + if (typeof r.name === "string" && r.name) return r.name; + const fn = r.function as Record | undefined; + const fnName = typeof fn?.name === "string" ? fn.name : ""; + const rType = typeof r.type === "string" ? r.type : ""; + if (fnName && rType) return `${fnName} (${rType})`; + if (fnName) return fnName; + if (rType) return `${rType} (${(r.id as string).slice(0, 8)}…)`; + return r.id as string; +} + +function quickExtractName(filePath: string): string | null { + try { + const content = readFileSync(filePath, "utf-8"); + if (filePath.endsWith(".md")) { + const match = content.match(/^---\r?\n[\s\S]*?^name:\s*(.+)/m); + return match?.[1]?.trim().replace(/^['"]|['"]$/g, "") ?? null; + } + const match = content.match(/^name:\s*(.+)/m); + if (match?.[1]) { + let val = match[1].trim(); + if ( + (val.startsWith('"') && val.endsWith('"')) || + (val.startsWith("'") && val.endsWith("'")) + ) + val = val.slice(1, -1); + return val; + } + // For tools: function.name + const fnMatch = content.match( + /^function:\s*\n\s+name:\s*(.+)/m, + ); + if (fnMatch?.[1]) { + return fnMatch[1].trim().replace(/^['"]|['"]$/g, ""); + } + } catch { + /* ignore parse errors */ + } + return null; +} + +// ───────────────────────────────────────────────────────────────────────────── +// Local Resource Scanning +// ───────────────────────────────────────────────────────────────────────────── + +interface LocalResource { + typeKey: string; + typeLabel: string; + resourceId: string; + filePath: string; + displayName: string; +} + +function scanLocalResources(slug: string): LocalResource[] { + const resourcesDir = join(BASE_DIR, "resources", slug); + if (!existsSync(resourcesDir)) return []; + + const resources: LocalResource[] = []; + + for (const typeDef of RESOURCE_TYPES) { + const typeDir = join(resourcesDir, typeDef.folder); + if (!existsSync(typeDir)) continue; + + const walk = (dir: string): void => { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.name.startsWith(".")) continue; + const fullPath = join(dir, entry.name); + if (entry.isDirectory()) { + walk(fullPath); + continue; + } + const ext = extname(entry.name); + if (![".yml", ".yaml", ".md", ".ts"].includes(ext)) continue; + + const relPath = relative(typeDir, fullPath); + const resourceId = relPath.slice(0, -ext.length); + const name = quickExtractName(fullPath); + const displayName = name || resourceId; + + resources.push({ + typeKey: typeDef.key, + typeLabel: typeDef.label, + resourceId, + filePath: fullPath, + displayName, + }); + } + }; + + walk(typeDir); + } + + return resources; +} + +// ───────────────────────────────────────────────────────────────────────────── +// Git Status Detection +// ───────────────────────────────────────────────────────────────────────────── + +type GitStatusCode = "M" | "A" | "D" | "?" | ""; + +function getGitFileStatuses(slug: string): Map { + const statuses = new Map(); + try { + const output = execSync("git status --porcelain", { + cwd: BASE_DIR, + encoding: "utf-8", + stdio: ["pipe", "pipe", "pipe"], + }).trim(); + + if (!output) return statuses; + + const prefix = `resources/${slug}/`; + for (const line of output.split("\n")) { + if (!line.trim()) continue; + const xy = line.slice(0, 2); + let filePath = line.slice(3); + const arrowIdx = filePath.indexOf(" -> "); + if (arrowIdx !== -1) filePath = filePath.slice(arrowIdx + 4); + filePath = filePath.replace(/^"|"$/g, "").trim(); + + if (!filePath.startsWith(prefix)) continue; + + let code: GitStatusCode = ""; + if (xy.includes("M")) code = "M"; + else if (xy.includes("A") || xy === "??") code = "A"; + else if (xy.includes("D")) code = "D"; + + if (code) { + const absPath = join(BASE_DIR, filePath); + statuses.set(absPath, code); + } + } + } catch { + /* not a git repo or git not available */ + } + return statuses; +} + +function gitStatusLabel(code: GitStatusCode): string { + switch (code) { + case "M": + return c.yellow("[modified]"); + case "A": + return c.green("[new]"); + case "D": + return c.red("[deleted]"); + default: + return ""; + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// State File — detect which remote resources are already pulled locally +// ───────────────────────────────────────────────────────────────────────────── + +function loadKnownUuids(slug: string): Set { + const statePath = join(BASE_DIR, `.vapi-state.${slug}.json`); + if (!existsSync(statePath)) return new Set(); + try { + const raw = readFileSync(statePath, "utf-8"); + const state = JSON.parse(raw) as Record>; + const uuids = new Set(); + for (const section of Object.values(state)) { + if (typeof section !== "object" || section === null) continue; + for (const uuid of Object.values(section)) { + if (typeof uuid === "string") uuids.add(uuid); + } + } + return uuids; + } catch { + return new Set(); + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// Subprocess helpers +// ───────────────────────────────────────────────────────────────────────────── + +function spawnScript(args: string[]): void { + const binDir = join(BASE_DIR, "node_modules", ".bin"); + const pathSep = process.platform === "win32" ? ";" : ":"; + execSync(["tsx", ...args].join(" "), { + cwd: BASE_DIR, + stdio: "inherit", + env: { + ...process.env, + PATH: `${binDir}${pathSep}${process.env.PATH ?? ""}`, + }, + }); +} + +// ───────────────────────────────────────────────────────────────────────────── +// Interactive Pull +// ───────────────────────────────────────────────────────────────────────────── + +interface ResourceSnapshot { + key: string; + label: string; + resources: Record[]; +} + +export async function runInteractivePull(): Promise { + console.log(""); + console.log( + c.bold( + "═══════════════════════════════════════════════════════════════", + ), + ); + console.log(c.bold(" Vapi GitOps — Interactive Pull")); + console.log( + c.bold( + "═══════════════════════════════════════════════════════════════", + ), + ); + console.log(""); + + type Step = "org" | "scope" | "pick" | "confirm" | "execute"; + let step: Step = "org"; + + let slug = ""; + let token = ""; + let baseUrl = ""; + let snapshots: ResourceSnapshot[] = []; + let nonEmpty: ResourceSnapshot[] = []; + let totalCount = 0; + let picked: string[] = []; + + while (step !== "execute") { + switch (step) { + // ── Org selection ─────────────────────────────────────────────── + case "org": { + slug = await selectOrg("pull"); + ({ token, baseUrl } = loadOrgEnv(slug)); + + console.log(c.dim(" Fetching remote resources...\n")); + + let fetchFailed = false; + snapshots = await Promise.all( + RESOURCE_TYPES.map( + async (type): Promise => { + try { + const data = await apiGet(token, baseUrl, type.endpoint); + return { + key: type.key, + label: type.label, + resources: normaliseList(data), + }; + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + if (msg.includes("401") || msg.includes("403") || msg.includes("authentication") || msg.includes("unauthorized")) { + fetchFailed = true; + } + console.log(c.red(` ✗ Failed to fetch ${type.label}: ${msg}`)); + return { key: type.key, label: type.label, resources: [] }; + } + }, + ), + ); + + if (fetchFailed) { + console.log(c.red("\n ⚠ API authentication failed. Check your VAPI_TOKEN in .env." + slug)); + console.log(c.red(" Run \"npm run setup\" to reconfigure.\n")); + return; + } + + nonEmpty = snapshots.filter((s) => s.resources.length > 0); + totalCount = nonEmpty.reduce( + (n, s) => n + s.resources.length, + 0, + ); + + if (nonEmpty.length === 0) { + console.log(c.yellow(" No remote resources found.\n")); + return; + } + + step = "scope"; + break; + } + + // ── All / Pick scope ──────────────────────────────────────────── + case "scope": { + const scope = await select({ + message: "Which resources to pull?", + choices: [ + { + name: `All (${totalCount} resources across ${nonEmpty.length} types)`, + value: "all" as const, + }, + { name: "Let me pick…", value: "pick" as const }, + { name: c.dim("← Back"), value: "back" as const }, + ], + }); + + if (scope === "back") { + step = "org"; + break; + } + + if (scope === "all") { + console.log(c.dim("\n Pulling all resources...\n")); + spawnScript(["src/pull.ts", slug, "--force"]); + console.log(c.green("\n Done!\n")); + return; + } + + step = "pick"; + break; + } + + // ── Individual resource picker ────────────────────────────────── + case "pick": { + const knownUuids = loadKnownUuids(slug); + const localCount = nonEmpty.reduce( + (n, s) => + n + + s.resources.filter((r) => knownUuids.has(r.id as string)) + .length, + 0, + ); + if (localCount > 0) { + console.log( + c.dim( + ` ${localCount}/${totalCount} already pulled locally (marked ✔)\n`, + ), + ); + } + + const allChoices = nonEmpty.flatMap((snap) => + snap.resources.map((r) => { + const isLocal = knownUuids.has(r.id as string); + const tag = isLocal ? c.dim(" ✔ local") : ""; + return { + value: `${snap.key}::${r.id as string}`, + name: `${resourceDisplayName(r)}${tag}`, + group: snap.label, + checked: false, + }; + }), + ); + + picked = await searchableCheckbox({ + message: "Select resources to pull", + choices: allChoices, + pageSize: 20, + }); + + if (isBack(picked)) { + step = "scope"; + break; + } + + if (picked.length === 0) { + console.log(c.dim("\n Nothing selected.\n")); + return; + } + + step = "confirm"; + break; + } + + // ── Confirm ───────────────────────────────────────────────────── + case "confirm": { + const selectedIds = new Set(picked); + + console.log("\n Pull list:"); + for (const snap of snapshots) { + const count = [...selectedIds].filter((v) => + v.startsWith(`${snap.key}::`), + ).length; + if (count > 0) { + console.log( + ` ${c.green("✓")} ${snap.label} (${count}/${snap.resources.length})`, + ); + } + } + console.log(""); + + const action = await select({ + message: `Pull ${picked.length} resource(s)?`, + choices: [ + { name: "Yes, pull", value: "yes" as const }, + { name: "No, cancel", value: "no" as const }, + { name: c.dim("← Back to selection"), value: "back" as const }, + ], + }); + + if (action === "back") { + step = "pick"; + break; + } + if (action === "no") { + console.log(c.dim("\n Cancelled.\n")); + return; + } + + step = "execute"; + break; + } + } + } + + // ── Execute pull ────────────────────────────────────────────────────── + const byType = new Map(); + for (const id of picked) { + const sep = id.indexOf("::"); + const typeKey = id.substring(0, sep); + const uuid = id.substring(sep + 2); + if (!byType.has(typeKey)) byType.set(typeKey, []); + byType.get(typeKey)!.push(uuid); + } + + console.log(c.dim("\n Pulling...\n")); + + for (const [typeKey, uuids] of byType) { + const idArgs = uuids.flatMap((id) => ["--id", id]); + spawnScript([ + "src/pull.ts", + slug, + "--force", + "--type", + typeKey, + ...idArgs, + ]); + } + + console.log(c.green("\n Done!\n")); +} + +// ───────────────────────────────────────────────────────────────────────────── +// Interactive Push +// ───────────────────────────────────────────────────────────────────────────── + +export async function runInteractivePush(): Promise { + console.log(""); + console.log( + c.bold( + "═══════════════════════════════════════════════════════════════", + ), + ); + console.log(c.bold(" Vapi GitOps — Interactive Push")); + console.log( + c.bold( + "═══════════════════════════════════════════════════════════════", + ), + ); + console.log(""); + + type Step = "org" | "scope" | "pick" | "confirm" | "execute"; + let step: Step = "org"; + + let slug = ""; + let resources: LocalResource[] = []; + let gitStatuses = new Map(); + let picked: string[] = []; + + while (step !== "execute") { + switch (step) { + // ── Org selection ─────────────────────────────────────────────── + case "org": { + slug = await selectOrg("push"); + resources = scanLocalResources(slug); + + if (resources.length === 0) { + console.log( + c.yellow(` No local resources found in resources/${slug}/.\n`), + ); + return; + } + + gitStatuses = getGitFileStatuses(slug); + const modifiedCount = [...gitStatuses.values()].filter( + (s) => s === "M", + ).length; + const newCount = [...gitStatuses.values()].filter( + (s) => s === "A", + ).length; + + if (modifiedCount > 0 || newCount > 0) { + const parts: string[] = []; + if (modifiedCount > 0) parts.push(`${modifiedCount} modified`); + if (newCount > 0) parts.push(`${newCount} new`); + console.log(c.dim(` Git status: ${parts.join(", ")}\n`)); + } + + step = "scope"; + break; + } + + // ── All / Pick scope ──────────────────────────────────────────── + case "scope": { + const scope = await select({ + message: "Which resources to push?", + choices: [ + { + name: `All (${resources.length} resources)`, + value: "all" as const, + }, + { name: "Let me pick…", value: "pick" as const }, + { name: c.dim("← Back"), value: "back" as const }, + ], + }); + + if (scope === "back") { + step = "org"; + break; + } + + if (scope === "all") { + console.log(c.dim("\n Pushing all resources...\n")); + spawnScript(["src/push.ts", slug]); + console.log(c.green("\n Done!\n")); + return; + } + + step = "pick"; + break; + } + + // ── Individual resource picker ────────────────────────────────── + case "pick": { + const allChoices = resources.map((r) => { + const status = gitStatuses.get(r.filePath); + const statusTag = status ? ` ${gitStatusLabel(status)}` : ""; + return { + value: r.filePath, + name: `${r.displayName}${statusTag}`, + group: r.typeLabel, + checked: false, + }; + }); + + allChoices.sort((a, b) => { + if (a.group !== b.group) return 0; + const aStatus = gitStatuses.get(a.value) || ""; + const bStatus = gitStatuses.get(b.value) || ""; + if (aStatus && !bStatus) return -1; + if (!aStatus && bStatus) return 1; + return 0; + }); + + picked = await searchableCheckbox({ + message: "Select resources to push", + choices: allChoices, + pageSize: 20, + }); + + if (isBack(picked)) { + step = "scope"; + break; + } + + if (picked.length === 0) { + console.log(c.dim("\n Nothing selected.\n")); + return; + } + + step = "confirm"; + break; + } + + // ── Confirm ───────────────────────────────────────────────────── + case "confirm": { + const selectedSet = new Set(picked); + const byGroup = new Map(); + for (const r of resources) { + if (!selectedSet.has(r.filePath)) continue; + byGroup.set(r.typeLabel, (byGroup.get(r.typeLabel) ?? 0) + 1); + } + + console.log("\n Push list:"); + for (const [group, count] of byGroup) { + console.log(` ${c.green("✓")} ${group} (${count})`); + } + console.log(""); + + const action = await select({ + message: `Push ${picked.length} resource(s)?`, + choices: [ + { name: "Yes, push", value: "yes" as const }, + { name: "No, cancel", value: "no" as const }, + { name: c.dim("← Back to selection"), value: "back" as const }, + ], + }); + + if (action === "back") { + step = "pick"; + break; + } + if (action === "no") { + console.log(c.dim("\n Cancelled.\n")); + return; + } + + step = "execute"; + break; + } + } + } + + // ── Execute push ────────────────────────────────────────────────────── + const relPaths = picked.map((p) => relative(BASE_DIR, p)); + console.log(c.dim("\n Pushing...\n")); + spawnScript(["src/push.ts", slug, ...relPaths]); + console.log(c.green("\n Done!\n")); +} + +// ───────────────────────────────────────────────────────────────────────────── +// Interactive Apply (Pull → Push) +// ───────────────────────────────────────────────────────────────────────────── + +export async function runInteractiveApply(): Promise { + console.log(""); + console.log( + c.bold( + "═══════════════════════════════════════════════════════════════", + ), + ); + console.log(c.bold(" Vapi GitOps — Interactive Apply (Pull → Push)")); + console.log( + c.bold( + "═══════════════════════════════════════════════════════════════", + ), + ); + console.log(""); + + const slug = await selectOrg("apply"); + + const useForce = await confirm({ + message: + "Enable force mode? (deletions: resources removed locally will also be deleted remotely)", + default: false, + }); + + const args = ["src/apply.ts", slug]; + if (useForce) args.push("--force"); + + console.log(c.dim("\n Running pull → push...\n")); + spawnScript(args); + console.log(c.green("\n Done!\n")); +} + +// ───────────────────────────────────────────────────────────────────────────── +// Interactive Call +// ───────────────────────────────────────────────────────────────────────────── + +export async function runInteractiveCall(): Promise { + console.log(""); + console.log( + c.bold( + "═══════════════════════════════════════════════════════════════", + ), + ); + console.log(c.bold(" Vapi GitOps — Interactive Call")); + console.log( + c.bold( + "═══════════════════════════════════════════════════════════════", + ), + ); + console.log(""); + + const slug = await selectOrg("call"); + + const stateFile = join(BASE_DIR, `.vapi-state.${slug}.json`); + if (!existsSync(stateFile)) { + console.log( + c.yellow(`\n No state file found (.vapi-state.${slug}.json).`), + ); + console.log(c.yellow(" Run pull first to populate resource mappings.\n")); + return; + } + + let state: StateFile; + try { + state = JSON.parse(readFileSync(stateFile, "utf-8")) as StateFile; + } catch { + console.log(c.red(`\n Failed to parse state file.\n`)); + return; + } + + const assistantNames = Object.keys(state.assistants ?? {}); + const squadNames = Object.keys( + (state as StateFile & { squads?: Record }).squads ?? {}, + ); + + if (assistantNames.length === 0 && squadNames.length === 0) { + console.log( + c.yellow( + "\n No assistants or squads found in state. Run pull first.\n", + ), + ); + return; + } + + const choices: { name: string; value: string }[] = []; + for (const name of assistantNames) { + choices.push({ + name: `${name} ${c.dim("(assistant)")}`, + value: `-a ${name}`, + }); + } + for (const name of squadNames) { + choices.push({ + name: `${name} ${c.dim("(squad)")}`, + value: `-s ${name}`, + }); + } + + const target = await select({ + message: "Which resource to call?", + choices, + }); + + console.log(c.dim("\n Starting call...\n")); + spawnScript(["src/call.ts", slug, ...target.split(" ")]); +} + +// ───────────────────────────────────────────────────────────────────────────── +// Interactive Cleanup +// ───────────────────────────────────────────────────────────────────────────── + +export async function runInteractiveCleanup(): Promise { + console.log(""); + console.log( + c.bold( + "═══════════════════════════════════════════════════════════════", + ), + ); + console.log(c.bold(" Vapi GitOps — Interactive Cleanup")); + console.log( + c.bold( + "═══════════════════════════════════════════════════════════════", + ), + ); + console.log(""); + + const slug = await selectOrg("cleanup"); + + console.log( + c.yellow( + " ⚠ Cleanup deletes remote resources that are NOT in your local state file.\n", + ), + ); + + console.log(c.dim("\n Running dry-run preview...\n")); + spawnScript(["src/cleanup.ts", slug]); + + const proceed = await confirm({ + message: "Proceed with actual deletion?", + default: false, + }); + + if (!proceed) { + console.log(c.dim("\n Cancelled.\n")); + return; + } + + console.log(c.dim("\n Running cleanup with --force...\n")); + spawnScript(["src/cleanup.ts", slug, "--force"]); + console.log(c.green("\n Done!\n")); +} diff --git a/src/pull-cmd.ts b/src/pull-cmd.ts new file mode 100644 index 0000000..5e9dd21 --- /dev/null +++ b/src/pull-cmd.ts @@ -0,0 +1,34 @@ +// Entry point for `npm run pull`. Detects whether an org slug was provided: +// - With slug: forwards to pull.ts (existing non-interactive behavior) +// - Without slug: enters interactive mode (org selection + resource picker) + +const SLUG_RE = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/; +const arg = process.argv[2]; +const isSlug = arg && SLUG_RE.test(arg); + +if (isSlug) { + const { runPull } = await import("./pull.ts"); + await runPull().catch((error: unknown) => { + console.error( + "\n❌ Pull failed:", + error instanceof Error ? error.message : error, + ); + process.exit(1); + }); +} else if (!arg) { + const { runInteractivePull } = await import("./interactive.ts"); + await runInteractivePull().catch((error: unknown) => { + console.error( + "\n❌ Pull failed:", + error instanceof Error ? error.message : error, + ); + process.exit(1); + }); +} else { + console.error(`❌ Invalid org name: ${arg}`); + console.error( + " Must be lowercase alphanumeric with optional hyphens (e.g., dev, my-org)", + ); + console.error(" Usage: npm run pull | npm run pull (interactive)"); + process.exit(1); +} diff --git a/src/pull.ts b/src/pull.ts index d96bd8e..f7c9d6f 100644 --- a/src/pull.ts +++ b/src/pull.ts @@ -1,5 +1,5 @@ import { execSync } from "child_process"; -import { existsSync, readdirSync } from "fs"; +import { existsSync, readdirSync, statSync } from "fs"; import { mkdir, writeFile } from "fs/promises"; import { join, dirname, relative, resolve } from "path"; import { fileURLToPath } from "url"; @@ -52,6 +52,7 @@ const ENDPOINT_MAP: Record = { scenarios: "/eval/simulation/scenario", simulations: "/eval/simulation", simulationSuites: "/eval/simulation/suite", + evals: "/eval", }; // Map resource types to their folder paths (relative to resources/) @@ -64,6 +65,7 @@ const FOLDER_MAP: Record = { scenarios: "simulations/scenarios", simulations: "simulations/tests", simulationSuites: "simulations/suites", + evals: "evals", }; // ───────────────────────────────────────────────────────────────────────────── @@ -710,6 +712,28 @@ export async function pullResourceType( } } + // Skip locally edited files even without git (mtime-based detection) + // If the resource file is newer than the state file, it was locally modified + if (!bootstrap && !force && !isNew && !changedFiles) { + const dir = join(RESOURCES_DIR, folderPath); + const localFile = + [join(dir, `${resourceId}.md`), join(dir, `${resourceId}.yml`), join(dir, `${resourceId}.yaml`)] + .find((p) => existsSync(p)); + if (localFile) { + const stateFilePath = join(BASE_DIR, `.vapi-state.${VAPI_ENV}.json`); + if (existsSync(stateFilePath)) { + const localMtime = statSync(localFile).mtimeMs; + const stateMtime = statSync(stateFilePath).mtimeMs; + if (localMtime > stateMtime) { + console.log(` ⏭️ ${resourceId} (locally modified, skipping)`); + newStateSection[resourceId] = resource.id; + skipped++; + continue; + } + } + } + } + // Skip resources whose local file was deleted (works without git). // A resource that was previously tracked in state but now has no local // file is treated as an intentional deletion. To stop tracking it @@ -789,7 +813,7 @@ export async function runPull(options: PullOptions = {}): Promise { if (resourceIds?.length) { if (!typeFilter?.length || typeFilter.length !== 1) { throw new Error( - "Single-resource pull requires exactly one resource type. Example: npm run pull:dev -- squads --id ", + "Single-resource pull requires exactly one resource type. Example: npm run pull -- --type squads --id ", ); } } @@ -869,6 +893,7 @@ export async function runPull(options: PullOptions = {}): Promise { scenarios: { ...zero }, simulations: { ...zero }, simulationSuites: { ...zero }, + evals: { ...zero }, }; // Pull in reverse-resolution order: pull resources that are referenced by others first, @@ -932,6 +957,13 @@ export async function runPull(options: PullOptions = {}): Promise { bootstrap, resourceIds, }); + if (shouldPull("evals")) + stats.evals = await pullResourceType("evals", state, { + changedFiles, + force, + bootstrap, + resourceIds, + }); await saveState(state); @@ -962,7 +994,9 @@ export async function runPull(options: PullOptions = {}): Promise { console.log(" 🚫 = matched .vapi-ignore (not tracked)"); console.log(" ✏️ = locally modified (preserved)"); console.log(" 🗑️ = locally deleted (intent in state)"); - console.log(" Run with --force to overwrite: npm run pull:dev:force"); + console.log( + ` Run with --force to overwrite: npm run pull -- ${VAPI_ENV} --force`, + ); } return { state, stats, force, bootstrap }; diff --git a/src/push-cmd.ts b/src/push-cmd.ts new file mode 100644 index 0000000..4fdc654 --- /dev/null +++ b/src/push-cmd.ts @@ -0,0 +1,34 @@ +// Entry point for `npm run push`. Detects whether an org slug was provided: +// - With slug: forwards to push.ts (existing non-interactive behavior) +// - Without slug: enters interactive mode (org selection + resource picker) + +const SLUG_RE = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/; +const arg = process.argv[2]; +const isSlug = arg && SLUG_RE.test(arg); + +if (isSlug) { + const { runPush } = await import("./push.ts"); + await runPush().catch((error: unknown) => { + console.error( + "\n❌ Push failed:", + error instanceof Error ? error.message : error, + ); + process.exit(1); + }); +} else if (!arg) { + const { runInteractivePush } = await import("./interactive.ts"); + await runInteractivePush().catch((error: unknown) => { + console.error( + "\n❌ Push failed:", + error instanceof Error ? error.message : error, + ); + process.exit(1); + }); +} else { + console.error(`❌ Invalid org name: ${arg}`); + console.error( + " Must be lowercase alphanumeric with optional hyphens (e.g., dev, my-org)", + ); + console.error(" Usage: npm run push | npm run push (interactive)"); + process.exit(1); +} diff --git a/src/push.ts b/src/push.ts index e0303d9..8fe4dd6 100644 --- a/src/push.ts +++ b/src/push.ts @@ -1,9 +1,12 @@ +import { resolve } from "path"; +import { fileURLToPath } from "url"; import { vapiRequest, VapiApiError } from "./api.ts"; import { VAPI_ENV, VAPI_BASE_URL, FORCE_DELETE, APPLY_FILTER, + BASE_DIR, removeExcludedKeys, } from "./config.ts"; import { loadState, saveState } from "./state.ts"; @@ -502,6 +505,27 @@ export async function applySimulationSuite( }); } +export async function applyEval( + resource: ResourceFile, + state: StateFile, +): Promise { + const { resourceId, data } = resource; + const existingUuid = state.evals[resourceId]; + + const payload = data as Record; + + if (existingUuid) { + const updatePayload = removeExcludedKeys(payload, "evals"); + console.log(` 🔄 Updating eval: ${resourceId} (${existingUuid})`); + await vapiRequest("PATCH", `/eval/${existingUuid}`, updatePayload); + return existingUuid; + } else { + console.log(` ✨ Creating eval: ${resourceId}`); + const result = await vapiRequest("POST", "/eval", payload); + return result.id; + } +} + // ───────────────────────────────────────────────────────────────────────────── // Post-Apply: Update Tools with Assistant References (for handoff tools) // ───────────────────────────────────────────────────────────────────────────── @@ -587,11 +611,12 @@ function isPartialApply(): boolean { } function shouldApplyResourceType(type: ResourceType): boolean { - // If filtering by specific files, check if any file matches this type if (APPLY_FILTER.filePaths?.length) { - return true; // We'll filter by resourceId later + const folder = FOLDER_MAP[type]; + return APPLY_FILTER.filePaths.some( + (fp) => fp.includes(`/${folder}/`) || fp.includes(`\\${folder}\\`), + ); } - // If filtering by types, only include matching types if (APPLY_FILTER.resourceTypes?.length) { return APPLY_FILTER.resourceTypes.includes(type); } @@ -604,13 +629,14 @@ function filterResourcesByPaths( ): ResourceFile[] { if (!APPLY_FILTER.filePaths?.length) return resources; - // Get all resourceIds that match the file paths for this type const matchingIds = new Set(); for (const filePath of APPLY_FILTER.filePaths) { - // Try to match the file path to a resourceId + const resolvedInput = resolve(BASE_DIR, filePath); + for (const resource of resources) { if ( + resource.filePath === resolvedInput || resource.filePath.endsWith(filePath) || filePath.endsWith(resource.resourceId + ".yml") || filePath.endsWith(resource.resourceId + ".yaml") || @@ -814,6 +840,7 @@ async function main(): Promise { scenarios: 0, simulations: 0, simulationSuites: 0, + evals: 0, }; // From here on, any path out of the function (success OR thrown error) must @@ -837,6 +864,8 @@ async function main(): Promise { await loadResources>("simulations"); const allSimulationSuitesRaw = await loadResources>("simulationSuites"); + const allEvalsRaw = + await loadResources>("evals"); const loadedResources: LoadedResources = { tools: allToolsRaw, @@ -847,6 +876,7 @@ async function main(): Promise { scenarios: allScenariosRaw, simulations: allSimulationsRaw, simulationSuites: allSimulationSuitesRaw, + evals: allEvalsRaw, }; state = await maybeBootstrapState(loadedResources, state); @@ -904,6 +934,7 @@ async function main(): Promise { const allSimulationSuites = resolveCredentials( filterDefaults(allSimulationSuitesRaw), ); + const allEvals = resolveCredentials(filterDefaults(allEvalsRaw)); // Filter resources based on apply filter const tools = shouldApplyResourceType("tools") @@ -930,6 +961,9 @@ async function main(): Promise { const simulationSuites = shouldApplyResourceType("simulationSuites") ? filterResourcesByPaths(allSimulationSuites, "simulationSuites") : []; + const evals = shouldApplyResourceType("evals") + ? filterResourcesByPaths(allEvals, "evals") + : []; // Auto-dependency resolution context const autoApplied = new Set(); @@ -963,6 +997,7 @@ async function main(): Promise { if (scenarios.length > 0) typesToDelete.push("scenarios"); if (simulations.length > 0) typesToDelete.push("simulations"); if (simulationSuites.length > 0) typesToDelete.push("simulationSuites"); + if (evals.length > 0) typesToDelete.push("evals"); } } @@ -983,6 +1018,7 @@ async function main(): Promise { scenarios: allScenariosRaw, simulations: allSimulationsRaw, simulationSuites: allSimulationSuitesRaw, + evals: allEvalsRaw, }, state, typesToDelete, @@ -995,6 +1031,7 @@ async function main(): Promise { // 4. Simulation building blocks (personalities, scenarios) // 5. Simulations (references personalities, scenarios) // 6. Simulation suites (references simulations) + // 7. Evals if (tools.length > 0) { console.log("\n🔧 Applying tools...\n"); @@ -1136,6 +1173,20 @@ async function main(): Promise { } } + if (evals.length > 0) { + console.log("\n🧪 Applying evals...\n"); + for (const evalResource of evals) { + try { + const uuid = await applyEval(evalResource, state); + state.evals[evalResource.resourceId] = uuid; + applied.evals++; + } catch (error) { + console.error(formatApiError(evalResource.resourceId, error)); + throw error; + } + } + } + // Second pass: Link resources to assistants (include auto-applied deps) const allAppliedTools = [...tools, ...autoAppliedTools]; if (allAppliedTools.length > 0) { @@ -1179,6 +1230,7 @@ async function main(): Promise { console.log(` Simulations: ${applied.simulations}`); if (applied.simulationSuites > 0) console.log(` Simulation Suites: ${applied.simulationSuites}`); + if (applied.evals > 0) console.log(` Evals: ${applied.evals}`); } else { console.log("📋 Summary:"); console.log(` Tools: ${Object.keys(state.tools).length}`); @@ -1193,6 +1245,7 @@ async function main(): Promise { console.log( ` Simulation Suites: ${Object.keys(state.simulationSuites).length}`, ); + console.log(` Evals: ${Object.keys(state.evals).length}`); } } finally { // Always flush state, even on partial failure — resources that already @@ -1209,15 +1262,24 @@ async function main(): Promise { } } -// Run the apply engine -main().catch((error) => { - if (error instanceof VapiApiError) { - console.error(`\n❌ Apply failed: ${error.apiMessage}`); - } else { - console.error( - "\n❌ Apply failed:", - error instanceof Error ? error.message : error, - ); - } - process.exit(1); -}); +export async function runPush(): Promise { + return main(); +} + +const isMainModule = + process.argv[1] !== undefined && + resolve(process.argv[1]) === fileURLToPath(import.meta.url); + +if (isMainModule) { + main().catch((error) => { + if (error instanceof VapiApiError) { + console.error(`\n❌ Apply failed: ${error.apiMessage}`); + } else { + console.error( + "\n❌ Apply failed:", + error instanceof Error ? error.message : error, + ); + } + process.exit(1); + }); +} diff --git a/src/resources.ts b/src/resources.ts index e59da28..d64cecd 100644 --- a/src/resources.ts +++ b/src/resources.ts @@ -15,6 +15,7 @@ export const FOLDER_MAP: Record = { scenarios: "simulations/scenarios", simulations: "simulations/tests", simulationSuites: "simulations/suites", + evals: "evals", }; // Reverse map: folder path to resource type diff --git a/src/searchableCheckbox.ts b/src/searchableCheckbox.ts new file mode 100644 index 0000000..e7cbdac --- /dev/null +++ b/src/searchableCheckbox.ts @@ -0,0 +1,372 @@ +import { + createPrompt, + useState, + useKeypress, + isUpKey, + isDownKey, + isSpaceKey, + isEnterKey, +} from "@inquirer/core"; + +// ───────────────────────────────────────────────────────────────────────────── +// Types +// ───────────────────────────────────────────────────────────────────────────── + +interface Choice { + value: string; + name: string; + group: string; + checked?: boolean; +} + +interface Config { + message: string; + choices: Choice[]; + pageSize?: number; + allowBack?: boolean; + /** Start with all groups collapsed (default: false) */ + collapsed?: boolean; +} + +export const BACK_SENTINEL = "__BACK__"; + +interface HeaderEntry { + type: "header"; + group: string; + /** selected / total counts for display */ + sel: number; + total: number; + expanded: boolean; + matchCount: number; +} + +interface ItemEntry { + type: "item"; + /** Index into the original choices array */ + ci: number; +} + +type DisplayEntry = HeaderEntry | ItemEntry; + +// ───────────────────────────────────────────────────────────────────────────── +// ANSI helpers +// ───────────────────────────────────────────────────────────────────────────── + +const esc = { + bold: (s: string) => `\x1b[1m${s}\x1b[0m`, + dim: (s: string) => `\x1b[2m${s}\x1b[0m`, + green: (s: string) => `\x1b[32m${s}\x1b[0m`, + cyan: (s: string) => `\x1b[36m${s}\x1b[0m`, + yellow: (s: string) => `\x1b[33m${s}\x1b[0m`, + cursorHide: "\x1b[?25l", +}; + +// ───────────────────────────────────────────────────────────────────────────── +// Helpers +// ───────────────────────────────────────────────────────────────────────────── + +/** Extract unique group names in the order they first appear */ +function groupNames(choices: Choice[]): string[] { + const seen = new Set(); + const groups: string[] = []; + for (const c of choices) { + if (!seen.has(c.group)) { + seen.add(c.group); + groups.push(c.group); + } + } + return groups; +} + +// ───────────────────────────────────────────────────────────────────────────── +// Prompt +// ───────────────────────────────────────────────────────────────────────────── + +export default createPrompt((config, done) => { + const { choices, pageSize = 20 } = config; + const allGroups = groupNames(choices); + + const [status, setStatus] = useState("active"); + const [selected, setSelected] = useState>( + () => + new Set( + choices.reduce((acc, c, i) => { + if (c.checked === true) acc.push(i); + return acc; + }, []), + ), + ); + const [filter, setFilter] = useState(""); + // cursor indexes into the display[] array (headers + visible items) + const [cursor, setCursor] = useState(0); + const [collapsedGroups, setCollapsedGroups] = useState>( + () => new Set(config.collapsed ? allGroups : []), + ); + + // ── Build filtered indices per group ──────────────────────────────────── + + const isSearching = filter.length > 0; + const lower = filter.toLowerCase(); + + // Map: group → array of original choice indices that match the filter + const filteredByGroup = new Map(); + for (const group of allGroups) { + filteredByGroup.set(group, []); + } + for (let i = 0; i < choices.length; i++) { + const c = choices[i]!; + if ( + !isSearching || + c.name.toLowerCase().includes(lower) || + c.group.toLowerCase().includes(lower) + ) { + filteredByGroup.get(c.group)!.push(i); + } + } + + // ── Build display list ────────────────────────────────────────────────── + + const display: DisplayEntry[] = []; + + for (const group of allGroups) { + const matchingIndices = filteredByGroup.get(group)!; + if (matchingIndices.length === 0) continue; + + const totalInGroup = choices.filter((c) => c.group === group).length; + const selInGroup = choices.filter( + (c, i) => c.group === group && selected.has(i), + ).length; + + // When searching, force expand groups with matches + const isExpanded = isSearching || !collapsedGroups.has(group); + + display.push({ + type: "header", + group, + sel: selInGroup, + total: totalInGroup, + expanded: isExpanded, + matchCount: matchingIndices.length, + }); + + if (isExpanded) { + for (const ci of matchingIndices) { + display.push({ type: "item", ci }); + } + } + } + + const maxCursor = Math.max(0, display.length - 1); + const safeCursor = Math.max(0, Math.min(cursor, maxCursor)); + + // Helper: get the group name for the current cursor position + const currentEntry = display[safeCursor]; + const currentGroup: string | undefined = + currentEntry?.type === "header" + ? currentEntry.group + : currentEntry?.type === "item" + ? choices[currentEntry.ci]!.group + : undefined; + + // ── Keypress handler ──────────────────────────────────────────────────── + + useKeypress((key) => { + if (isEnterKey(key)) { + setStatus("done"); + done(choices.filter((_, i) => selected.has(i)).map((c) => c.value)); + return; + } + + if (isUpKey(key)) { + setCursor(Math.max(0, safeCursor - 1)); + return; + } + + if (isDownKey(key)) { + setCursor(Math.min(maxCursor, safeCursor + 1)); + return; + } + + // Right arrow: expand group (on header) or no-op on item + if (key.name === "right") { + if (currentEntry?.type === "header" && !isSearching) { + const next = new Set(collapsedGroups); + next.delete(currentEntry.group); + setCollapsedGroups(next); + } + return; + } + + // Left arrow: collapse group (on header), or jump to group header (on item) + if (key.name === "left") { + if (!isSearching) { + if (currentEntry?.type === "header") { + const next = new Set(collapsedGroups); + next.add(currentEntry.group); + setCollapsedGroups(next); + } else if (currentEntry?.type === "item" && currentGroup) { + // Jump cursor to this item's group header + const headerIdx = display.findIndex( + (d) => d.type === "header" && d.group === currentGroup, + ); + if (headerIdx >= 0) setCursor(headerIdx); + } + } + return; + } + + if (isSpaceKey(key)) { + if (currentEntry?.type === "header") { + // Toggle all items in this group + const groupIndices = filteredByGroup.get(currentEntry.group) ?? []; + const allChecked = groupIndices.every((i) => selected.has(i)); + const next = new Set(selected); + for (const i of groupIndices) { + if (allChecked) next.delete(i); + else next.add(i); + } + setSelected(next); + } else if (currentEntry?.type === "item") { + const ci = currentEntry.ci; + const next = new Set(selected); + if (next.has(ci)) next.delete(ci); + else next.add(ci); + setSelected(next); + } + return; + } + + // Ctrl+A: toggle all visible + if (key.ctrl && key.name === "a") { + const visibleIndices = display + .filter((d): d is ItemEntry => d.type === "item") + .map((d) => d.ci); + // Also include collapsed group items so ctrl+a truly means "all filtered" + const allFilteredIndices = new Set(); + for (const indices of filteredByGroup.values()) { + for (const i of indices) allFilteredIndices.add(i); + } + const target = visibleIndices.length > 0 ? [...allFilteredIndices] : []; + const allChecked = target.every((i) => selected.has(i)); + const next = new Set(selected); + for (const i of target) { + if (allChecked) next.delete(i); + else next.add(i); + } + setSelected(next); + return; + } + + // Ctrl+G: toggle all in the current group + if (key.ctrl && key.name === "g") { + if (currentGroup) { + const groupIndices = filteredByGroup.get(currentGroup) ?? []; + const allChecked = groupIndices.every((i) => selected.has(i)); + const next = new Set(selected); + for (const i of groupIndices) { + if (allChecked) next.delete(i); + else next.add(i); + } + setSelected(next); + } + return; + } + + if (key.name === "backspace") { + if (filter.length > 0) { + setFilter(filter.slice(0, -1)); + setCursor(0); + } + return; + } + + if (key.name === "escape") { + if (filter) { + setFilter(""); + setCursor(0); + } else if (config.allowBack !== false) { + setStatus("done"); + done([BACK_SENTINEL]); + } + return; + } + + // Printable character + if ( + !key.ctrl && + !key.shift && + key.name && + key.name.length === 1 && + key.name.charCodeAt(0) >= 33 && + key.name.charCodeAt(0) <= 126 + ) { + setFilter(filter + key.name); + setCursor(0); + } + }); + + // ── Render ────────────────────────────────────────────────────────────── + + const prefix = status === "done" ? esc.green("✔") : esc.green("?"); + + if (status === "done") { + return `${prefix} ${esc.bold(config.message)} ${esc.cyan(`${selected.size} selected`)}`; + } + + // Paginate around cursor position + const half = Math.floor(pageSize / 2); + let start = Math.max(0, safeCursor - half); + start = Math.min(start, Math.max(0, display.length - pageSize)); + const end = Math.min(start + pageSize, display.length); + + const lines: string[] = []; + lines.push(`${prefix} ${esc.bold(config.message)}`); + + if (filter) { + lines.push(` ${esc.dim("Search:")} ${filter}▏ ${esc.dim("(esc to clear)")}`); + } else { + lines.push(` ${esc.dim("Type to search… ←/→: collapse/expand (esc to go back)")}`); + } + lines.push(""); + + if (display.length === 0) { + lines.push(` ${esc.dim("No matches")}`); + } else { + if (start > 0) lines.push(` ${esc.dim(" ↑ more above")}`); + + for (let di = start; di < end; di++) { + const entry = display[di]!; + const isCursor = di === safeCursor; + + if (entry.type === "header") { + const arrow = entry.expanded ? "▾" : "▸"; + const counts = isSearching + ? `${entry.matchCount} match${entry.matchCount === 1 ? "" : "es"}` + : `${entry.sel}/${entry.total}`; + const ptr = isCursor ? esc.cyan("❯") : " "; + const label = isCursor + ? esc.bold(`${arrow} ${entry.group} (${counts})`) + : esc.dim(`${arrow} ${entry.group} (${counts})`); + lines.push(` ${ptr} ${label}`); + } else { + const choice = choices[entry.ci]!; + const isChecked = selected.has(entry.ci); + const ptr = isCursor ? esc.cyan("❯") : " "; + const ico = isChecked ? esc.green("◉") : esc.dim("◯"); + const lbl = isCursor ? esc.bold(choice.name) : choice.name; + lines.push(` ${ptr} ${ico} ${lbl}`); + } + } + + const remaining = display.length - end; + if (remaining > 0) lines.push(` ${esc.dim(` ↓ ${remaining} more below`)}`); + } + + lines.push(""); + const backHint = config.allowBack !== false ? " · esc: back" : ""; + lines.push( + ` ${esc.dim(`${selected.size}/${choices.length} selected · space: toggle · ctrl+g: group · ctrl+a: all · enter: confirm${backHint}`)}`, + ); + + return `${lines.join("\n")}${esc.cursorHide}`; +}); diff --git a/src/setup.ts b/src/setup.ts new file mode 100644 index 0000000..b5d46db --- /dev/null +++ b/src/setup.ts @@ -0,0 +1,646 @@ +import { existsSync, readdirSync } from "fs"; +import { mkdir, writeFile, rm } from "fs/promises"; +import { join, dirname } from "path"; +import { fileURLToPath } from "url"; +import { execSync } from "child_process"; +import { input, password, confirm, select } from "@inquirer/prompts"; +import searchableCheckbox, { BACK_SENTINEL } from "./searchableCheckbox.js"; + +// ───────────────────────────────────────────────────────────────────────────── +// Constants +// ───────────────────────────────────────────────────────────────────────────── + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const BASE_DIR = join(__dirname, ".."); +const VAPI_REGIONS: Record = { + us: "https://api.vapi.ai", + eu: "https://api.eu.vapi.ai", +}; +let vapiBaseUrl = VAPI_REGIONS.us!; +const UUID_RE = + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +const SLUG_RE = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/; + +interface ResourceTypeDef { + key: string; + label: string; + endpoint: string; +} + +const RESOURCE_TYPES: ResourceTypeDef[] = [ + { key: "assistants", label: "Assistants", endpoint: "/assistant" }, + { key: "tools", label: "Tools", endpoint: "/tool" }, + { key: "squads", label: "Squads", endpoint: "/squad" }, + { + key: "structuredOutputs", + label: "Structured Outputs", + endpoint: "/structured-output", + }, + { + key: "personalities", + label: "Personalities", + endpoint: "/eval/simulation/personality", + }, + { + key: "scenarios", + label: "Scenarios", + endpoint: "/eval/simulation/scenario", + }, + { key: "simulations", label: "Simulations", endpoint: "/eval/simulation" }, + { + key: "simulationSuites", + label: "Simulation Suites", + endpoint: "/eval/simulation/suite", + }, + { + key: "evals", + label: "Evals", + endpoint: "/eval", + }, +]; + +// ───────────────────────────────────────────────────────────────────────────── +// Terminal helpers +// ───────────────────────────────────────────────────────────────────────────── + +const c = { + bold: (s: string) => `\x1b[1m${s}\x1b[0m`, + dim: (s: string) => `\x1b[2m${s}\x1b[0m`, + green: (s: string) => `\x1b[32m${s}\x1b[0m`, + yellow: (s: string) => `\x1b[33m${s}\x1b[0m`, + red: (s: string) => `\x1b[31m${s}\x1b[0m`, + cyan: (s: string) => `\x1b[36m${s}\x1b[0m`, +}; + +// ───────────────────────────────────────────────────────────────────────────── +// API client +// ───────────────────────────────────────────────────────────────────────────── + +async function apiGet(token: string, endpoint: string): Promise { + const response = await fetch(`${vapiBaseUrl}${endpoint}`, { + method: "GET", + headers: { Authorization: `Bearer ${token}` }, + }); + + if (!response.ok) { + const text = await response.text(); + throw new Error( + `API GET ${endpoint} failed (${response.status}): ${text}`, + ); + } + + return response.json(); +} + +async function validateToken(token: string): Promise { + try { + await apiGet(token, "/assistant?limit=1"); + return true; + } catch { + return false; + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// Resource fetching +// ───────────────────────────────────────────────────────────────────────────── + +interface ResourceSnapshot { + key: string; + label: string; + count: number; + resources: Record[]; +} + +function normaliseList(data: unknown): Record[] { + if (Array.isArray(data)) return data as Record[]; + if ( + data && + typeof data === "object" && + "results" in data && + Array.isArray((data as Record).results) + ) { + return (data as Record).results as Record< + string, + unknown + >[]; + } + return []; +} + +async function fetchAllResourceSnapshots( + token: string, +): Promise { + const results = await Promise.all( + RESOURCE_TYPES.map(async (type): Promise => { + try { + const data = await apiGet(token, type.endpoint); + const list = normaliseList(data); + return { + key: type.key, + label: type.label, + count: list.length, + resources: list, + }; + } catch { + return { key: type.key, label: type.label, count: 0, resources: [] }; + } + }), + ); + return results; +} + +// ───────────────────────────────────────────────────────────────────────────── +// Slug helpers +// ───────────────────────────────────────────────────────────────────────────── + +function slugify(name: string): string { + return name + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, "") + .replace(/-+/g, "-"); +} + +// ───────────────────────────────────────────────────────────────────────────── +// Dependency detection — scan selected resources for UUID references +// to resources that aren't yet selected +// ───────────────────────────────────────────────────────────────────────────── + +function detectMissingDependencies( + snapshots: ResourceSnapshot[], + selectedIds: Set, +): Map> { + const refs = new Map>(); + + const addRef = (type: string, value: unknown) => { + if (typeof value !== "string" || !UUID_RE.test(value)) return; + if (selectedIds.has(`${type}::${value}`)) return; + if (!refs.has(type)) refs.set(type, new Set()); + refs.get(type)!.add(value); + }; + + for (const snap of snapshots) { + for (const r of snap.resources) { + const id = r.id as string; + if (!selectedIds.has(`${snap.key}::${id}`)) continue; + + const model = r.model as Record | undefined; + if (model && Array.isArray(model.toolIds)) { + for (const tid of model.toolIds) addRef("tools", tid); + } + + if (Array.isArray(r.toolIds)) { + for (const tid of r.toolIds) addRef("tools", tid); + } + + const ap = r.artifactPlan as Record | undefined; + if (ap && Array.isArray(ap.structuredOutputIds)) { + for (const sid of ap.structuredOutputIds) + addRef("structuredOutputs", sid); + } + + if (Array.isArray(r.members)) { + for (const m of r.members as Record[]) { + addRef("assistants", m.assistantId); + if (Array.isArray(m.assistantDestinations)) { + for (const d of m.assistantDestinations as Record< + string, + unknown + >[]) { + addRef("assistants", d.assistantId); + } + } + } + } + + if (Array.isArray(r.destinations)) { + for (const d of r.destinations as Record[]) { + addRef("assistants", d.assistantId); + } + } + + if (Array.isArray(r.assistantIds)) { + for (const aid of r.assistantIds) addRef("assistants", aid); + } + + addRef("personalities", r.personalityId); + addRef("scenarios", r.scenarioId); + + if (Array.isArray(r.simulationIds)) { + for (const sid of r.simulationIds) addRef("simulations", sid); + } + + if (Array.isArray(r.evaluations)) { + for (const ev of r.evaluations as Record[]) { + addRef("structuredOutputs", ev.structuredOutputId); + } + } + } + } + + // Only keep refs to resources that actually exist in our snapshots + const knownIds = new Set(); + for (const snap of snapshots) { + for (const r of snap.resources) { + knownIds.add(`${snap.key}::${r.id as string}`); + } + } + + const missing = new Map>(); + for (const [type, uuids] of refs) { + const existing = new Set(); + for (const uuid of uuids) { + if (knownIds.has(`${type}::${uuid}`)) existing.add(uuid); + } + if (existing.size > 0) missing.set(type, existing); + } + return missing; +} + +// ───────────────────────────────────────────────────────────────────────────── +// File system helpers +// ───────────────────────────────────────────────────────────────────────────── + +async function writeEnvFile( + slug: string, + token: string, + baseUrl: string, +): Promise { + const envPath = join(BASE_DIR, `.env.${slug}`); + let content = `VAPI_TOKEN=${token}\n`; + if (baseUrl !== VAPI_REGIONS.us) { + content += `VAPI_BASE_URL=${baseUrl}\n`; + } + await writeFile(envPath, content); +} + +async function deleteExistingOrg(slug: string): Promise { + const resourceDir = join(BASE_DIR, "resources", slug); + const stateFile = join(BASE_DIR, `.vapi-state.${slug}.json`); + + if (existsSync(resourceDir)) { + await rm(resourceDir, { recursive: true, force: true }); + } + if (existsSync(stateFile)) { + await rm(stateFile); + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// Pull integration +// ───────────────────────────────────────────────────────────────────────────── + +function invokePull(slug: string, selectedIds: Set): void { + // Group selected resources by type so we can pass --id per invocation + const byType = new Map(); + for (const id of selectedIds) { + const sep = id.indexOf("::"); + const typeKey = id.substring(0, sep); + const uuid = id.substring(sep + 2); + if (!byType.has(typeKey)) byType.set(typeKey, []); + byType.get(typeKey)!.push(uuid); + } + + const binDir = join(BASE_DIR, "node_modules", ".bin"); + const pathSep = process.platform === "win32" ? ";" : ":"; + const env = { + ...process.env, + PATH: `${binDir}${pathSep}${process.env.PATH ?? ""}`, + }; + + for (const [typeKey, uuids] of byType) { + const idArgs = uuids.flatMap((id) => ["--id", id]); + const cmd = [ + "tsx", + "src/pull.ts", + slug, + "--force", + "--type", + typeKey, + ...idArgs, + ].join(" "); + execSync(cmd, { cwd: BASE_DIR, stdio: "inherit", env }); + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// Display helpers +// ───────────────────────────────────────────────────────────────────────────── + +function resourceDisplayName(r: Record): string { + // Top-level name (assistants, squads, structured outputs, simulations, etc.) + if (typeof r.name === "string" && r.name) return r.name; + + // Tools: meaningful name is in function.name, type gives context + const fn = r.function as Record | undefined; + const fnName = typeof fn?.name === "string" ? fn.name : ""; + const rType = typeof r.type === "string" ? r.type : ""; + + if (fnName && rType) return `${fnName} (${rType})`; + if (fnName) return fnName; + if (rType) return `${rType} (${(r.id as string).slice(0, 8)}…)`; + + // Last resort + return r.id as string; +} + +// ───────────────────────────────────────────────────────────────────────────── +// Main wizard +// ───────────────────────────────────────────────────────────────────────────── + +async function main(): Promise { + if (!existsSync(join(BASE_DIR, "node_modules"))) { + console.log(c.dim("\n Installing dependencies...\n")); + execSync("npm install", { cwd: BASE_DIR, stdio: "inherit" }); + } + + console.log(""); + console.log( + c.bold( + "═══════════════════════════════════════════════════════════════", + ), + ); + console.log(c.bold(" Vapi GitOps — Setup Wizard")); + console.log( + c.bold( + "═══════════════════════════════════════════════════════════════", + ), + ); + console.log(""); + + // ── Step 1: API key + region ──────────────────────────────────────── + + let trimmedKey = ""; + + // eslint-disable-next-line no-constant-condition + while (true) { + const apiKey = await password({ + message: "Paste your Vapi private API key", + mask: "•", + validate: (value) => { + if (!value.trim()) return "API key is required"; + return true; + }, + }); + + trimmedKey = apiKey.trim(); + console.log(c.dim(` Validating against ${vapiBaseUrl}…`)); + + const valid = await validateToken(trimmedKey); + if (valid) { + const region = vapiBaseUrl === VAPI_REGIONS.eu ? "EU" : "US"; + console.log(c.green(` ✓ Connected to Vapi (${region})\n`)); + break; + } + + console.log( + c.red(" ✗ Could not authenticate — invalid key or wrong region.\n"), + ); + + const recovery = await select({ + message: "What would you like to do?", + choices: [ + { name: "Try a different API key", value: "retry" as const }, + { + name: `Switch to ${vapiBaseUrl === VAPI_REGIONS.eu ? "US" : "EU"} region and retry`, + value: "switch" as const, + }, + { name: "Cancel setup", value: "cancel" as const }, + ], + }); + + if (recovery === "cancel") { + console.log(c.dim("\n Setup cancelled.")); + process.exit(0); + } + + if (recovery === "switch") { + vapiBaseUrl = + vapiBaseUrl === VAPI_REGIONS.eu ? VAPI_REGIONS.us! : VAPI_REGIONS.eu!; + console.log(c.dim(` Switched to ${vapiBaseUrl}\n`)); + // Re-validate same key against the new region + console.log(c.dim(` Validating against ${vapiBaseUrl}…`)); + const retryValid = await validateToken(trimmedKey); + if (retryValid) { + const region = vapiBaseUrl === VAPI_REGIONS.eu ? "EU" : "US"; + console.log(c.green(` ✓ Connected to Vapi (${region})\n`)); + break; + } + console.log( + c.red(" ✗ Still could not authenticate. Try a different key.\n"), + ); + } + + // "retry" or failed switch → loop back to password prompt + } + + // ── Step 2: Folder slug ─────────────────────────────────────────────── + + const rawSlug = await input({ + message: "Folder name for this org (e.g. my-org, my-org-prod)", + validate: (value) => { + const slug = slugify(value); + if (!slug || !SLUG_RE.test(slug)) { + return "Must be lowercase alphanumeric with hyphens (e.g. my-org-name)"; + } + return true; + }, + transformer: (value) => { + const slug = slugify(value); + if (slug && slug !== value) return `${value} → ${c.dim(slug)}`; + return value; + }, + }); + + const slug = slugify(rawSlug); + + // Check if org already exists locally + const resourceDir = join(BASE_DIR, "resources", slug); + const stateFile = join(BASE_DIR, `.vapi-state.${slug}.json`); + + if (existsSync(resourceDir) || existsSync(stateFile)) { + console.log(c.yellow(`\n ⚠ Org "${slug}" already exists locally.`)); + + const override = await confirm({ + message: "Override? (deletes existing files and re-pulls)", + default: false, + }); + + if (!override) { + console.log( + `\n Use ${c.cyan(`npm run pull -- ${slug}`)} to update existing resources.`, + ); + process.exit(0); + } + + console.log(c.dim(" Removing existing files...")); + await deleteExistingOrg(slug); + console.log(c.green(" ✓ Cleaned up\n")); + } else { + console.log(c.green(`\n ✓ Will create: resources/${slug}/\n`)); + } + + // ── Step 3: Resource selection ──────────────────────────────────────── + + console.log(c.dim(" Fetching available resources...\n")); + + const snapshots = await fetchAllResourceSnapshots(trimmedKey); + const nonEmpty = snapshots.filter((s) => s.count > 0); + + if (nonEmpty.length === 0) { + console.log(c.yellow(" No resources found in this org.")); + console.log(" Writing environment file only.\n"); + await writeEnvFile(slug, trimmedKey, vapiBaseUrl); + await mkdir(resourceDir, { recursive: true }); + printSummary(slug); + return; + } + + const totalCount = nonEmpty.reduce((n, s) => n + s.count, 0); + + // selectedIds: "typeKey::resourceUUID" + let selectedIds: Set; + + // eslint-disable-next-line no-constant-condition + while (true) { + const scope = await select({ + message: "Which resources to download?", + choices: [ + { + name: `All (${totalCount} resources across ${nonEmpty.length} types)`, + value: "all" as const, + }, + { name: "Let me pick…", value: "pick" as const }, + ], + }); + + if (scope === "pick") { + const allChoices = nonEmpty.flatMap((snap) => + snap.resources.map((r) => ({ + value: `${snap.key}::${r.id as string}`, + name: resourceDisplayName(r), + group: snap.label, + checked: false, + })), + ); + + const picked = await searchableCheckbox({ + message: "Select resources", + choices: allChoices, + pageSize: 20, + }); + + if (picked.length === 1 && picked[0] === BACK_SENTINEL) continue; + + selectedIds = new Set(picked); + } else { + selectedIds = new Set( + nonEmpty.flatMap((snap) => + snap.resources.map((r) => `${snap.key}::${r.id as string}`), + ), + ); + } + break; + } + + // ── Step 3b: Dependency detection (iterative) ───────────────────────── + + console.log(c.dim("\n Checking dependencies...\n")); + + let iterations = 0; + while (iterations < 5) { + const missing = detectMissingDependencies(snapshots, selectedIds); + if (missing.size === 0) break; + + console.log( + c.yellow(" ⚠ Selected resources reference additional items:"), + ); + for (const [type, uuids] of missing) { + const def = RESOURCE_TYPES.find((t) => t.key === type); + console.log(` • ${uuids.size} ${def?.label ?? type}`); + } + console.log(""); + + const includeDeps = await confirm({ + message: "Also download referenced resources?", + default: true, + }); + + if (!includeDeps) break; + + for (const [type, uuids] of missing) { + for (const uuid of uuids) { + selectedIds.add(`${type}::${uuid}`); + } + } + iterations++; + } + + // Show final download list + console.log("\n Download list:"); + for (const snap of snapshots) { + const typeSelected = [...selectedIds].filter((v) => + v.startsWith(`${snap.key}::`), + ).length; + if (typeSelected > 0) { + console.log( + ` ${c.green("✓")} ${snap.label} (${typeSelected}/${snap.count})`, + ); + } + } + console.log(""); + + // ── Step 4: Write env file & pull ───────────────────────────────────── + + await writeEnvFile(slug, trimmedKey, vapiBaseUrl); + console.log(c.green(` ✓ Created .env.${slug}\n`)); + + console.log(c.bold(" Downloading...\n")); + + invokePull(slug, selectedIds); + + // ── Done ────────────────────────────────────────────────────────────── + + printSummary(slug); +} + +function printSummary(slug: string): void { + console.log(""); + console.log( + c.bold( + "═══════════════════════════════════════════════════════════════", + ), + ); + console.log(c.bold(" ✅ Setup Complete!")); + console.log( + c.bold( + "═══════════════════════════════════════════════════════════════", + ), + ); + console.log(""); + console.log(` 📁 Resources: resources/${slug}/`); + console.log(` 🔑 Env file: .env.${slug}`); + console.log(` 📄 State file: .vapi-state.${slug}.json`); + console.log(""); + console.log(" Next steps:"); + console.log( + ` ${c.cyan(`npm run pull -- ${slug}`)} Pull latest from Vapi`, + ); + console.log( + ` ${c.cyan(`npm run push -- ${slug}`)} Push local changes to Vapi`, + ); + console.log( + ` ${c.cyan(`npm run pull -- ${slug} --force`)} Force overwrite local files`, + ); + console.log(""); +} + +main().catch((error) => { + console.error( + c.red( + `\n ✗ Setup failed: ${error instanceof Error ? error.message : error}`, + ), + ); + process.exit(1); +}); diff --git a/src/state.ts b/src/state.ts index fcaa199..152bd54 100644 --- a/src/state.ts +++ b/src/state.ts @@ -18,6 +18,7 @@ function createEmptyState(): StateFile { scenarios: {}, simulations: {}, simulationSuites: {}, + evals: {}, }; } diff --git a/src/types.ts b/src/types.ts index a3f9b1c..fd51eaf 100644 --- a/src/types.ts +++ b/src/types.ts @@ -12,6 +12,7 @@ export interface StateFile { scenarios: Record; simulations: Record; simulationSuites: Record; + evals: Record; } export interface ResourceFile> { @@ -33,15 +34,14 @@ export type ResourceType = | "personalities" | "scenarios" | "simulations" - | "simulationSuites"; + | "simulationSuites" + | "evals"; -export type Environment = "dev" | "stg" | "prod"; +// Any slug-like string: "dev", "prod", "roofr-production", etc. +export type Environment = string; -export const VALID_ENVIRONMENTS: readonly Environment[] = [ - "dev", - "stg", - "prod", -]; +// Well-known names kept for backward-compatible npm scripts +export const VALID_ENVIRONMENTS: readonly string[] = ["dev", "stg", "prod"]; export const VALID_RESOURCE_TYPES: readonly ResourceType[] = [ "tools", @@ -52,6 +52,7 @@ export const VALID_RESOURCE_TYPES: readonly ResourceType[] = [ "scenarios", "simulations", "simulationSuites", + "evals", ]; export interface LoadedResources { @@ -63,6 +64,7 @@ export interface LoadedResources { scenarios: ResourceFile>[]; simulations: ResourceFile>[]; simulationSuites: ResourceFile>[]; + evals: ResourceFile>[]; } export interface OrphanedResource {