diff --git a/README.md b/README.md index 98c007f..7727e6c 100755 --- a/README.md +++ b/README.md @@ -764,6 +764,10 @@ repository: build_commands: - npm install - npm run build + env_keys: # auto-discovered from .env.example + - LINKEDIN_EMAIL # values live in MCP_ENV_FILE + - LINKEDIN_PASSWORD # (the proxy's .env) and are written + - LINKEDIN_LI_AT # into /.env on every build / spawn tools: - name: search_jobs # advertised as linkedin__search_jobs description: Search LinkedIn job postings. @@ -777,13 +781,56 @@ tools: The `package.command` is what spawns the MCP server (just like a regular package provider). The new `repository:` block tells the server **how to materialize the workdir on startup**. +#### Secrets from `.env.example` + +If the cloned repo contains a `.env.example` (or `.env.sample` / `.env.template`) +at its root, mcpproxy parses it after the clone step and surfaces every +`KEY=` line in two places: + +1. The wizard's **Secrets** step (so you can fill in values immediately). +2. The provider's `repository.env_keys` list in YAML (editable in the + **πŸ“‚ Repository** editor box). + +Values themselves live in `MCP_ENV_FILE` (the proxy's `.env`) β€” the same +storage every other secret uses. At spawn time and on every restart, the +server: + +1. Reads the current values from `MCP_ENV_FILE` and the process environment. +2. Writes a `.env` file inside `` containing only the keys that + are actually set (empty / unset keys are skipped). +3. Passes the same values as environment variables to the spawned MCP + subprocess. + +This covers both server styles: code that calls `dotenv.config()` / +`tsx --env-file=.env` reads the on-disk file, while code that reads +`process.env.X` / `os.environ[X]` sees the env vars directly. + +#### Build failures while secrets are missing + +A common failure mode: a build command like `npm install` triggers a +`postinstall` script that requires secrets, but the user hasn't filled +them in yet. mcpproxy's wizard handles this gracefully: + +- The clone step runs first, then `.env.example` is parsed. +- If a build command then fails, the wizard surfaces the error inline + and **still** continues to the Secrets step with the discovered keys. +- After you fill in the secrets and save, `materialize_repository` + re-runs the build on the next server start β€” with `/.env` + now populated β€” and the build succeeds. + #### Editing a repository provider -The editor shows a **πŸ“‚ Repository** box with the git URL, ref, and build commands. -- **↻ Re-clone & build** β€” re-runs `git pull` (or `git clone` on a fresh container) - and the build commands, then re-introspects the spawn command. -- After saving, click **Restart MCP Server** to apply changes β€” on startup the server - walks every repository provider and re-runs clone/build before registering tools. +The editor shows a **πŸ“‚ Repository** box with the git URL, ref, build +commands, and the auto-discovered env keys list. +- **↻ Re-clone & build** β€” re-runs `git pull` (or `git clone` on a fresh + container) and the build commands, then re-introspects the spawn + command. Newly-discovered env keys are merged into the list. +- **↻ Re-scan** on the env keys row β€” re-parses `.env.example` without + re-running the build (useful if you've just pulled a new commit + that adds variables). +- After saving, click **Restart MCP Server** to apply changes β€” on + startup the server walks every repository provider, re-clones / pulls + / re-builds, writes `/.env`, then registers tools. #### Environment variables @@ -976,6 +1023,10 @@ repository: build_commands: # shell commands run in before spawn - npm install - npm run build + env_keys: # optional β€” KEY names whose values live + - MY_API_KEY # in MCP_ENV_FILE. A .env file is written + - SECRET_TOKEN # into before every build / spawn. + # Auto-discovered from .env.example. # ── Shared optional fields (all provider types) ─────────────────────────────── diff --git a/frontend/app.py b/frontend/app.py index 44a488b..58bd49f 100644 --- a/frontend/app.py +++ b/frontend/app.py @@ -86,9 +86,48 @@ def _extract_secret_env_keys(spec: dict[str, Any]) -> list[str]: for key in (tool.get("secrets") or {}).get("env", {}).values(): if key not in keys: keys.append(key) + # Repository providers may declare extra env keys (auto-discovered from + # the cloned repo's .env.example) that drive the underlying server. + for key in (spec.get("repository") or {}).get("env_keys") or []: + if key and key not in keys: + keys.append(key) return keys +_ENV_EXAMPLE_CANDIDATES = (".env.example", ".env.sample", ".env.template") + + +def _write_workdir_env_file(workdir: str | Path, env_keys: list[str]) -> Path: + """Write a ``.env`` file inside ``workdir`` populated from ``os.environ``. + + Only keys with a non-empty value in the current process environment are + written. This lets dotenv-style loaders inside the cloned repo (such as + ``tsx --env-file=.env``) pick up secrets supplied via the proxy's + Secrets UI without leaking unset placeholders. Returns the written path. + """ + target = Path(workdir) / ".env" + target.parent.mkdir(parents=True, exist_ok=True) + lines: list[str] = [] + for key in env_keys: + val = os.environ.get(key) + if val: + lines.append(f"{key}={val}") + target.write_text("\n".join(lines) + ("\n" if lines else ""), encoding="utf-8") + return target + + +def _parse_env_example(workdir: str | Path) -> list[str]: + """Return the ordered list of KEY names from the first .env.example-style + file found in ``workdir``. Returns ``[]`` when no candidate exists. + """ + wd = Path(workdir) + for name in _ENV_EXAMPLE_CANDIDATES: + candidate = wd / name + if candidate.exists(): + return list(_read_env_file(candidate).keys()) + return [] + + # --------------------------------------------------------------------------- # Package manager detection (for logging / display β€” execution is identical) # --------------------------------------------------------------------------- @@ -174,6 +213,7 @@ def _provider_to_structured(name: str, spec: dict[str, Any]) -> dict[str, Any]: repo_ref = (repo_sub.get("ref") or "").strip() build_commands = list(repo_sub.get("build_commands") or []) workdir = _repository_workdir(name, repo_sub.get("workdir")) + repo_env_keys = list(repo_sub.get("env_keys") or []) elif pkg_sub is not None: ptype = "package" command = (pkg_sub.get("command") or "").strip() @@ -181,6 +221,7 @@ def _provider_to_structured(name: str, spec: dict[str, Any]) -> dict[str, Any]: repo_ref = "" build_commands = [] workdir = "" + repo_env_keys = [] else: ptype = "code" command = "" @@ -188,6 +229,7 @@ def _provider_to_structured(name: str, spec: dict[str, Any]) -> dict[str, Any]: repo_ref = "" build_commands = [] workdir = "" + repo_env_keys = [] return { "name": name, @@ -200,6 +242,7 @@ def _provider_to_structured(name: str, spec: dict[str, Any]) -> dict[str, Any]: "repo_url": repo_url, "repo_ref": repo_ref, "build_commands": build_commands, + "repo_env_keys": repo_env_keys, "workdir": workdir, "tools": tools_out, } @@ -231,6 +274,9 @@ def _structured_to_yaml(provider: dict[str, Any]) -> str: build_commands = [c for c in (provider.get("build_commands") or []) if c] if build_commands: repo_block["build_commands"] = build_commands + env_keys = [k for k in (provider.get("repo_env_keys") or []) if k] + if env_keys: + repo_block["env_keys"] = env_keys spec["repository"] = repo_block else: code = (provider.get("code") or "").strip() @@ -484,6 +530,7 @@ async def introspect_package(request: Request) -> dict: requirements: list[str] = body.get("requirements") or [] setup_commands: list[str] = body.get("setup_commands") or [] cwd = (body.get("cwd") or "").strip() or None + env_keys = list(body.get("env_keys") or []) or None if not command: raise HTTPException(400, "command is required") @@ -507,7 +554,7 @@ async def introspect_package(request: Request) -> dict: # 3. Introspect the MCP server from process_runner import introspect - tools = await introspect(command, cwd=cwd) + tools = await introspect(command, cwd=cwd, env_keys=env_keys) return {"ok": True, "tools": tools, "package_manager": pm} except Exception as exc: traceback.print_exc() @@ -543,25 +590,70 @@ async def clone_and_build(request: Request) -> dict: workdir = _repository_workdir(name, explicit_workdir) + # The clone step is required β€” if that fails we have nothing useful + # to return. Build-command failures are tolerated: they're often + # caused by missing .env values, and the user needs the discovered + # env_keys back so they can populate secrets and retry on next + # restart (when materialize_repository writes .env first). try: Path(workdir).parent.mkdir(parents=True, exist_ok=True) if (Path(workdir) / ".git").exists(): subprocess.run(["git", "-C", workdir, "pull", "--ff-only"], check=True) else: subprocess.run(["git", "clone", url, workdir], check=True) - if ref: subprocess.run(["git", "-C", workdir, "checkout", ref], check=True) + except Exception as exc: + traceback.print_exc() + return { + "ok": False, + "error": str(exc), + "workdir": workdir, + "env_keys": [], + } - for cmd in build_commands: - if not cmd: - continue - subprocess.run(shlex.split(cmd), check=True, cwd=workdir) + # Parse .env.example BEFORE running build commands so the user + # gets the secret list back even if a build command fails. + env_keys = _parse_env_example(workdir) + + # Write a best-effort .env from currently-set environment values so + # build commands that already invoke dotenv loaders (e.g. + # `tsx --env-file=.env`) succeed when secrets are present. + if env_keys: + try: + _write_workdir_env_file(workdir, env_keys) + except Exception: + traceback.print_exc() - return {"ok": True, "workdir": workdir} + for cmd in build_commands: + if not cmd: + continue + try: + subprocess.run(shlex.split(cmd), check=True, cwd=workdir) + except Exception as exc: + traceback.print_exc() + return { + "ok": False, + "error": str(exc), + "failed_command": cmd, + "workdir": workdir, + "env_keys": env_keys, + } + + return {"ok": True, "workdir": workdir, "env_keys": env_keys} + + @app.post("/api/scan-env-example") + async def scan_env_example(request: Request) -> dict: + """Re-scan ``/.env.example`` (or sibling) and return KEY names.""" + body = await request.json() + workdir = (body.get("workdir") or "").strip() + if not workdir: + raise HTTPException(400, "workdir is required") + try: + return {"ok": True, "env_keys": _parse_env_example(workdir)} except Exception as exc: traceback.print_exc() - return {"ok": False, "error": str(exc), "workdir": workdir} + return {"ok": False, "error": str(exc), "env_keys": []} # ── Function extractor (code providers) ────────────────────────────────── @@ -933,7 +1025,18 @@ async def index():
-
Shell commands run inside the workdir before the MCP server is spawned. Re-runs on every server start.
+
Shell commands run inside the workdir before the MCP server is spawned. Re-runs on every server start. Don't put long-running server start commands here.
+ +
+ +
+
A .env file is written into the workdir from your secrets before every build / spawn β€” so dotenv loaders (tsx --env-file=.env, etc.) pick them up.
Workdir: (auto)
@@ -1095,18 +1198,18 @@ async def index():
- +
-
e.g. npm install, npm run build. Re-runs on every server start so ephemeral containers rebuild.
+
e.g. npm install, npm run build. Do not put the long-running server start here (e.g. npm run start:dev) β€” that goes in Spawn command. Build commands re-run on every server start so ephemeral containers rebuild.
-
The command used to launch the stdio MCP server, run from inside the workdir after the build commands complete.
+
The long-running command that launches the stdio MCP server, run from inside the workdir after the build commands complete.
-
Clicking Next clones the repo, runs the build commands, then introspects the spawn command to populate the tool list. All settings are saved to YAML so the server can re-build on container restart.
+
Clicking Next clones the repo, parses .env.example (so its keys appear as secrets on the next step), runs the build commands, then introspects the spawn command to populate the tool list. If the build fails because secrets aren't set yet, you can still continue β€” the next server restart will re-build with the secrets in place.
@@ -1327,6 +1430,7 @@ async def index(): requirements: currentProvider.requirements || [], setup_commands: currentProvider.setup_commands || [], cwd: isRepo ? (currentProvider.workdir || '') : '', + env_keys: isRepo ? (currentProvider.repo_env_keys || []) : [], }); if (!r.ok) throw new Error(r.error || 'introspection failed'); knownFunctions = (r.tools || []).map(t => t.name).filter(Boolean); @@ -1422,6 +1526,7 @@ async def index(): document.getElementById('f-repo-ref').value = p.repo_ref || ''; document.getElementById('f-repo-workdir').textContent = p.workdir || '(auto)'; renderBuildCommands(p.build_commands || []); + renderEnvKeys(p.repo_env_keys || []); } if (isCode) { codeEditor.setValue(p.code || ''); @@ -1468,6 +1573,52 @@ async def index(): currentProvider[key] = val; } +// Env keys list (repository providers) +function renderEnvKeys(keys) { + const c = document.getElementById('env-keys-container'); + if (!keys.length) { c.innerHTML = '
(none discovered β€” no .env.example in the repo, or repo not built yet)
'; return; } + c.innerHTML = keys.map((k, i) => ` +
+ + +
`).join(''); +} + +function addEnvKey() { + ensureProvider(); + currentProvider.repo_env_keys = currentProvider.repo_env_keys || []; + currentProvider.repo_env_keys.push(''); + renderEnvKeys(currentProvider.repo_env_keys); +} + +function removeEnvKey(i) { + ensureProvider(); + currentProvider.repo_env_keys.splice(i, 1); + renderEnvKeys(currentProvider.repo_env_keys); +} + +function updateEnvKey(i, val) { + ensureProvider(); + currentProvider.repo_env_keys[i] = val.trim(); +} + +async function rescanEnvExample() { + if (!currentProvider || currentProvider.type !== 'repository') return; + const wd = currentProvider.workdir; + if (!wd) { toast('No workdir β€” click Re-clone & build first', false); return; } + try { + const r = await api('POST', '/api/scan-env-example', {workdir: wd}); + if (!r.ok) throw new Error(r.error || 'scan failed'); + // Merge: preserve any keys the user typed manually, add new ones from .env.example + const existing = new Set(currentProvider.repo_env_keys || []); + (r.env_keys || []).forEach(k => existing.add(k)); + currentProvider.repo_env_keys = Array.from(existing); + renderEnvKeys(currentProvider.repo_env_keys); + toast(`Discovered ${r.env_keys.length} env key(s)`); + } catch (e) { toast(e.message, false); } +} + async function rebuildRepository() { if (!currentProvider || currentProvider.type !== 'repository') return; const el = document.getElementById('rebuild-status'); @@ -1481,9 +1632,20 @@ async def index(): build_commands: currentProvider.build_commands || [], workdir: currentProvider.workdir || '', }); - if (!r.ok) throw new Error(r.error || 'build failed'); - currentProvider.workdir = r.workdir; - document.getElementById('f-repo-workdir').textContent = r.workdir; + currentProvider.workdir = r.workdir || currentProvider.workdir; + document.getElementById('f-repo-workdir').textContent = currentProvider.workdir; + // Merge discovered env_keys with whatever the user already has + if (r.env_keys && r.env_keys.length) { + const existing = new Set(currentProvider.repo_env_keys || []); + r.env_keys.forEach(k => existing.add(k)); + currentProvider.repo_env_keys = Array.from(existing); + renderEnvKeys(currentProvider.repo_env_keys); + } + if (!r.ok) { + el.className = 'fn-status error'; + el.textContent = (r.error || 'build failed') + (r.failed_command ? ` (running ${r.failed_command})` : ''); + return; + } el.className = 'fn-status ok'; el.textContent = `Built βœ“ (workdir: ${r.workdir})`; discoverFunctions().catch(() => {}); @@ -1733,6 +1895,7 @@ async def index(): p.repo_url = document.getElementById('f-repo-url').value.trim(); p.repo_ref = document.getElementById('f-repo-ref').value.trim(); p.build_commands = (currentProvider.build_commands || []).filter(c => c.trim()); + p.repo_env_keys = (currentProvider.repo_env_keys || []).filter(k => k.trim()); } else if (p.type === 'package') { p.command = document.getElementById('f-command').value.trim(); } else { @@ -2010,26 +2173,37 @@ async def index(): nextBtn.disabled = true; const origText = nextBtn.textContent; const resultEl = document.getElementById('wz-repo-result'); - let workdir = ''; + let workdir = '', env_keys = [], buildFailed = false; try { nextBtn.textContent = '⏳ Cloning & building…'; resultEl.innerHTML = 'Cloning repo and running build commands β€” this may take a while…'; const cb = await api('POST', '/api/clone-and-build', { name, repo_url: url, ref, build_commands, }); - if (!cb.ok) throw new Error(cb.error || 'clone/build failed'); - workdir = cb.workdir; - resultEl.innerHTML = `
βœ“ Built in ${esc(workdir)}
`; - nextBtn.textContent = '⏳ Introspecting…'; - const ir = await api('POST', '/api/introspect', { - command: cmd, cwd: workdir, - }); - if (!ir.ok) { - resultEl.innerHTML += `
⚠ Introspection failed (${esc(ir.error||'')}). Continuing β€” add tools manually in the editor.
`; - wzIntrospectedTools = []; + workdir = cb.workdir || ''; + env_keys = cb.env_keys || []; + if (!cb.ok) { + // Build failure is tolerated β€” we still have a workdir and the + // env_keys discovered from .env.example. Surface the error and + // let the user continue to the Secrets step. + buildFailed = true; + resultEl.innerHTML = `
⚠ Build failed: ${esc(cb.error || '')}${cb.failed_command ? ` (running ${esc(cb.failed_command)})` : ''}.
`; + if (env_keys.length) { + resultEl.innerHTML += `
Discovered ${env_keys.length} env key(s) from .env.example. Fill them in on the next step and the build will run again on the next server restart.
`; + } } else { - wzIntrospectedTools = ir.tools || []; - resultEl.innerHTML += `
βœ“ Found ${wzIntrospectedTools.length} tool(s)
`; + resultEl.innerHTML = `
βœ“ Built in ${esc(workdir)}${env_keys.length ? ` Β· Discovered ${env_keys.length} env key(s) from .env.example` : ''}
`; + nextBtn.textContent = '⏳ Introspecting…'; + const ir = await api('POST', '/api/introspect', { + command: cmd, cwd: workdir, env_keys, + }); + if (!ir.ok) { + resultEl.innerHTML += `
⚠ Introspection failed (${esc(ir.error||'')}). Continuing β€” add tools manually in the editor.
`; + wzIntrospectedTools = []; + } else { + wzIntrospectedTools = ir.tools || []; + resultEl.innerHTML += `
βœ“ Found ${wzIntrospectedTools.length} tool(s)
`; + } } } catch (e) { errEl.textContent = e.message; @@ -2041,12 +2215,13 @@ async def index(): nextBtn.disabled = false; nextBtn.textContent = origText; } - const provider = { - name, type: 'repository', command: cmd, documentation: '', code: '', - repo_url: url, repo_ref: ref, workdir, - build_commands, - requirements: [], setup_commands: [], - tools: wzIntrospectedTools.map(t => ({ + // Repository providers with a build failure may have no tools yet β€” + // add a placeholder so the create-provider validation passes. Users + // can replace it once secrets are populated and the next restart + // builds successfully. + let tools; + if (wzIntrospectedTools.length) { + tools = wzIntrospectedTools.map(t => ({ name: t.name, function: '', description: t.description || '', @@ -2054,13 +2229,27 @@ async def index(): enabled: true, parameters: _schemaToParams(t.inputSchema || t.input_schema || {}), secrets: [], - })), + })); + } else { + tools = [{ + name: '_placeholder', function: '', + description: 'Placeholder β€” re-introspect after the next successful build.', + documentation: '', enabled: false, parameters: [], secrets: [], + }]; + } + const provider = { + name, type: 'repository', command: cmd, documentation: '', code: '', + repo_url: url, repo_ref: ref, workdir, + build_commands, + repo_env_keys: env_keys, + requirements: [], setup_commands: [], + tools, }; try { const r = await api('POST', '/api/tools', {name, provider}); currentName = name; currentProvider = provider; loadList(); - await wzGoSecrets(r.secret_keys || []); + await wzGoSecrets(r.secret_keys || env_keys); } catch(e) { errEl.textContent = e.message; } return; } diff --git a/process_runner.py b/process_runner.py index bfe08fc..1affd13 100644 --- a/process_runner.py +++ b/process_runner.py @@ -17,6 +17,7 @@ import asyncio import json +import os import shlex import traceback from typing import Any @@ -25,9 +26,15 @@ class ProcessSession: """A long-lived connection to a single stdio MCP server process.""" - def __init__(self, command: str, cwd: str | None = None) -> None: + def __init__( + self, + command: str, + cwd: str | None = None, + env_keys: list[str] | None = None, + ) -> None: self.command = command self.cwd = cwd + self.env_keys = list(env_keys or []) self._parts: list[str] = shlex.split(command) self._proc: asyncio.subprocess.Process | None = None self._lock = asyncio.Lock() @@ -53,12 +60,14 @@ async def _recv(self, timeout: float = 30.0) -> dict[str, Any]: return json.loads(line) async def _start(self) -> None: + env = self._build_env() self._proc = await asyncio.create_subprocess_exec( *self._parts, stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, cwd=self.cwd, + env=env, ) # initialize handshake rid = self._new_id() @@ -74,6 +83,35 @@ async def _start(self) -> None: # notifications/initialized (no response expected) await self._send({"jsonrpc": "2.0", "method": "notifications/initialized"}) + def _build_env(self) -> dict[str, str]: + """Return the env dict for the subprocess. + + Starts from the current process env, then re-reads the proxy's + ``MCP_ENV_FILE`` (if any) so that secret values added via the UI + after server start are picked up on the next spawn without + requiring a full restart. Only ``env_keys`` are refreshed from + the file β€” everything else is inherited unchanged. + """ + env = os.environ.copy() + if not self.env_keys: + return env + env_file = os.environ.get("MCP_ENV_FILE", ".env") + try: + from pathlib import Path + p = Path(env_file) + if p.exists(): + for line in p.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, _, v = line.partition("=") + k = k.strip() + if k in self.env_keys: + env[k] = v.strip().strip('"').strip("'") + except Exception: + traceback.print_exc() + return env + def _alive(self) -> bool: return self._proc is not None and self._proc.returncode is None @@ -139,29 +177,36 @@ async def close(self) -> None: # Module-level session registry (one session per (command, cwd) pair) # --------------------------------------------------------------------------- -_sessions: dict[tuple[str, str | None], ProcessSession] = {} +_sessions: dict[tuple[str, str | None, tuple[str, ...]], ProcessSession] = {} -def get_session(command: str, cwd: str | None = None) -> ProcessSession: +def get_session( + command: str, + cwd: str | None = None, + env_keys: list[str] | None = None, +) -> ProcessSession: """Return (creating if needed) the persistent session for *command*. - Sessions are keyed on the (command, cwd) pair so that two providers that - happen to share a spawn command but live in different working directories - (e.g. two repository providers built from the same template) get distinct - subprocesses. + Sessions are keyed on (command, cwd, env_keys) so that two providers + that share a spawn command but live in different workdirs or use + different env-key sets get distinct subprocesses. """ - key = (command, cwd) + key = (command, cwd, tuple(env_keys or ())) if key not in _sessions: - _sessions[key] = ProcessSession(command, cwd=cwd) + _sessions[key] = ProcessSession(command, cwd=cwd, env_keys=env_keys) return _sessions[key] -async def introspect(command: str, cwd: str | None = None) -> list[dict[str, Any]]: +async def introspect( + command: str, + cwd: str | None = None, + env_keys: list[str] | None = None, +) -> list[dict[str, Any]]: """ Spawn a *fresh* process, fetch its tools/list, then shut it down. Used by the frontend wizard β€” does not affect the persistent session registry. """ - session = ProcessSession(command, cwd=cwd) + session = ProcessSession(command, cwd=cwd, env_keys=env_keys) try: await session._start() return await session.list_tools() diff --git a/server.py b/server.py index 4ebc24e..665d535 100755 --- a/server.py +++ b/server.py @@ -277,14 +277,17 @@ def _get_package_command(spec: dict[str, Any]) -> str | None: def _make_process_handler( - command: str, tool_name: str, cwd: str | None = None + command: str, + tool_name: str, + cwd: str | None = None, + env_keys: list[str] | None = None, ) -> Callable[..., Any]: """Return an async handler that proxies calls to a subprocess MCP process.""" from process_runner import get_session async def process_handler(context: dict[str, Any], **kwargs: Any) -> Any: try: - session = get_session(command, cwd=cwd) + session = get_session(command, cwd=cwd, env_keys=env_keys) return await session.call_tool(tool_name, kwargs) except Exception as exc: traceback.print_exc() @@ -330,6 +333,7 @@ def materialize_repository(spec: dict[str, Any]) -> None: workdir = repository_workdir(provider_name, spec) assert workdir is not None build_commands = list(repo.get("build_commands") or []) + env_keys = list(repo.get("env_keys") or []) try: wd_path = Path(workdir) @@ -346,6 +350,14 @@ def materialize_repository(spec: dict[str, Any]) -> None: print(f"Checking out ref {ref} in {workdir}") subprocess.run(["git", "-C", workdir, "checkout", ref], check=True) + # Materialise /.env from os.environ BEFORE running build + # commands. Some servers (e.g. those using `tsx --env-file=.env`) + # require the file to exist when the build script runs. Missing + # values are skipped β€” the build may still fail but on subsequent + # restarts (after the user fills secrets in the UI) it will succeed. + if env_keys: + write_workdir_env_file(workdir, env_keys) + for cmd in build_commands: if not cmd: continue @@ -357,6 +369,24 @@ def materialize_repository(spec: dict[str, Any]) -> None: raise +def write_workdir_env_file(workdir: str, env_keys: list[str]) -> Path: + """Write a ``.env`` inside ``workdir`` populated from ``os.environ``. + + Only keys with a non-empty value are written. Used by + ``materialize_repository`` so dotenv-style loaders inside the cloned + repo pick up secrets supplied via the proxy's Secrets UI. + """ + target = Path(workdir) / ".env" + target.parent.mkdir(parents=True, exist_ok=True) + lines: list[str] = [] + for key in env_keys: + val = os.environ.get(key) + if val: + lines.append(f"{key}={val}") + target.write_text("\n".join(lines) + ("\n" if lines else ""), encoding="utf-8") + return target + + def register_provider(spec: dict[str, Any]) -> None: """Register all tools declared in one provider spec. @@ -371,8 +401,10 @@ def register_provider(spec: dict[str, Any]) -> None: try: command = _get_package_command(spec) # Repository providers piggy-back on the package code path; the only - # difference is that their subprocess is spawned with cwd=. + # difference is that their subprocess is spawned with cwd= + # and env enriched with the repository.env_keys declared in YAML. cwd = repository_workdir(provider_name, spec) + env_keys = list((spec.get("repository") or {}).get("env_keys") or []) if command is not None: # ── package provider (npx / uvx / python -m / any binary) ────────── @@ -381,7 +413,7 @@ def register_provider(spec: dict[str, Any]) -> None: if not tool_is_enabled(tool_spec): print(f"Skipping disabled tool: {advertised_tool_name(provider_name, tool_name)}") continue - handler = _make_process_handler(command, tool_name, cwd=cwd) + handler = _make_process_handler(command, tool_name, cwd=cwd, env_keys=env_keys) register_tool( tool_spec, handler, diff --git a/tests/test_frontend.py b/tests/test_frontend.py index 8b9cc3a..09f933e 100644 --- a/tests/test_frontend.py +++ b/tests/test_frontend.py @@ -10,11 +10,13 @@ _detect_package_manager, _extract_functions, _extract_secret_env_keys, + _parse_env_example, _provider_to_structured, _read_env_file, _structured_to_yaml, _validate_provider, _write_env_file, + _write_workdir_env_file, create_app, ) @@ -411,7 +413,12 @@ def test_no_raw_yaml_editor(self, client): assert "mode/yaml" not in r.text def test_no_discover_tab(self, client): - assert "Discover" not in client.get("/").text + # The legacy "Discover" tab and its modal must not appear in the UI. + # (Substring "Discover" is allowed e.g. in "Discovered N env keys" + # toast text introduced for repository providers.) + text = client.get("/").text + assert "id=\"discover-tab\"" not in text + assert "Discover Tools" not in text def test_contains_api_calls(self, client): assert "/api/tools" in client.get("/").text @@ -877,7 +884,7 @@ class TestIntrospectCwd: def test_cwd_passed_through(self, client): captured = {} - async def fake_introspect(command, cwd=None): + async def fake_introspect(command, cwd=None, env_keys=None): captured["command"] = command captured["cwd"] = cwd return [] @@ -893,10 +900,187 @@ async def fake_introspect(command, cwd=None): def test_no_cwd_when_omitted(self, client): captured = {} - async def fake_introspect(command, cwd=None): + async def fake_introspect(command, cwd=None, env_keys=None): captured["cwd"] = cwd return [] with patch("process_runner.introspect", new=fake_introspect): client.post("/api/introspect", json={"command": "echo hi"}) assert captured["cwd"] is None + + +# --------------------------------------------------------------------------- +# .env.example parsing + workdir .env writing +# --------------------------------------------------------------------------- + +class TestParseEnvExample: + def test_returns_keys_in_order(self, tmp_path): + (tmp_path / ".env.example").write_text( + "# comment line\n" + "\n" + "FOO=bar\n" + "BAZ=\n" + 'QUOTED="value with spaces"\n' + ) + assert _parse_env_example(tmp_path) == ["FOO", "BAZ", "QUOTED"] + + def test_empty_when_no_file(self, tmp_path): + assert _parse_env_example(tmp_path) == [] + + def test_falls_back_to_env_sample(self, tmp_path): + (tmp_path / ".env.sample").write_text("MY_KEY=x\n") + assert _parse_env_example(tmp_path) == ["MY_KEY"] + + def test_falls_back_to_env_template(self, tmp_path): + (tmp_path / ".env.template").write_text("TEMPLATE_KEY=x\n") + assert _parse_env_example(tmp_path) == ["TEMPLATE_KEY"] + + def test_env_example_wins_over_sample(self, tmp_path): + (tmp_path / ".env.example").write_text("A=1\n") + (tmp_path / ".env.sample").write_text("B=2\n") + assert _parse_env_example(tmp_path) == ["A"] + + +class TestWriteWorkdirEnvFile: + def test_writes_only_set_keys(self, tmp_path, monkeypatch): + monkeypatch.setenv("MY_FOO", "fooval") + monkeypatch.delenv("MY_BAR", raising=False) + target = _write_workdir_env_file(tmp_path, ["MY_FOO", "MY_BAR"]) + text = target.read_text() + assert "MY_FOO=fooval" in text + assert "MY_BAR" not in text + + def test_creates_workdir_if_missing(self, tmp_path, monkeypatch): + wd = tmp_path / "sub" / "wd" + monkeypatch.setenv("X", "1") + _write_workdir_env_file(wd, ["X"]) + assert (wd / ".env").exists() + + def test_empty_file_when_no_keys_set(self, tmp_path, monkeypatch): + monkeypatch.delenv("UNSET_KEY", raising=False) + target = _write_workdir_env_file(tmp_path, ["UNSET_KEY"]) + assert target.read_text() == "" + + +class TestExtractSecretEnvKeysIncludesRepo: + def test_repo_env_keys_added(self): + spec = { + "tools": [{"secrets": {"env": {"a": "TOOL_KEY"}}}], + "repository": {"env_keys": ["REPO_A", "REPO_B"]}, + } + keys = _extract_secret_env_keys(spec) + assert keys == ["TOOL_KEY", "REPO_A", "REPO_B"] + + def test_dedup_across_tool_and_repo(self): + spec = { + "tools": [{"secrets": {"env": {"a": "SHARED"}}}], + "repository": {"env_keys": ["SHARED", "EXTRA"]}, + } + keys = _extract_secret_env_keys(spec) + assert keys == ["SHARED", "EXTRA"] + + +class TestRepositoryRoundTripEnvKeys: + def test_round_trip_with_env_keys(self): + provider = {**REPOSITORY_PROVIDER, "repo_env_keys": ["LINKEDIN_EMAIL", "LINKEDIN_PASSWORD"]} + yaml_str = _structured_to_yaml(provider) + spec = yaml.safe_load(yaml_str) + assert spec["repository"]["env_keys"] == ["LINKEDIN_EMAIL", "LINKEDIN_PASSWORD"] + structured = _provider_to_structured("linkedin", spec) + assert structured["repo_env_keys"] == ["LINKEDIN_EMAIL", "LINKEDIN_PASSWORD"] + + def test_empty_env_keys_omitted_from_yaml(self): + yaml_str = _structured_to_yaml(REPOSITORY_PROVIDER) # repo_env_keys not set + spec = yaml.safe_load(yaml_str) + assert "env_keys" not in spec.get("repository", {}) + + +class TestCloneAndBuildEnvKeys: + def _patch_repos_dir(self, monkeypatch, tmp_path): + monkeypatch.setattr("frontend.app.REPOS_DIR", tmp_path) + + def test_returns_env_keys_from_dot_env_example(self, client, tmp_path, monkeypatch): + self._patch_repos_dir(monkeypatch, tmp_path) + + # Fake git clone: when called with "git clone ", write a + # .env.example into as if the repo contained it. + def fake_run(args, **kwargs): + if len(args) >= 4 and args[0:2] == ["git", "clone"]: + wd = Path(args[3]) + wd.mkdir(parents=True, exist_ok=True) + (wd / ".env.example").write_text("API_KEY=\nUSERNAME=\n") + class _R: returncode = 0 + return _R() + + with patch("frontend.app.subprocess.run", side_effect=fake_run): + r = client.post("/api/clone-and-build", json={ + "name": "linkedin", + "repo_url": "https://e.com/r.git", + "build_commands": [], + }) + assert r.json()["ok"] is True + assert r.json()["env_keys"] == ["API_KEY", "USERNAME"] + + def test_returns_env_keys_even_when_build_fails(self, client, tmp_path, monkeypatch): + import subprocess as sp + self._patch_repos_dir(monkeypatch, tmp_path) + + def fake_run(args, **kwargs): + if args[0:2] == ["git", "clone"]: + wd = Path(args[3]) + wd.mkdir(parents=True, exist_ok=True) + (wd / ".env.example").write_text("NEED_ME=\n") + class _R: returncode = 0 + return _R() + if "npm" in args: + # build fails because the .env doesn't have the needed value + raise sp.CalledProcessError(9, args) + class _R: returncode = 0 + return _R() + + with patch("frontend.app.subprocess.run", side_effect=fake_run): + r = client.post("/api/clone-and-build", json={ + "name": "linkedin", + "repo_url": "https://e.com/r.git", + "build_commands": ["npm install", "npm run build"], + }) + data = r.json() + assert data["ok"] is False + # Crucially: env_keys still returned so the wizard can populate Secrets + assert data["env_keys"] == ["NEED_ME"] + assert data["failed_command"] == "npm install" + + def test_writes_env_file_before_build(self, client, tmp_path, monkeypatch): + self._patch_repos_dir(monkeypatch, tmp_path) + monkeypatch.setenv("NEED_ME", "supplied") + + def fake_run(args, **kwargs): + if args[0:2] == ["git", "clone"]: + wd = Path(args[3]) + wd.mkdir(parents=True, exist_ok=True) + (wd / ".env.example").write_text("NEED_ME=\n") + class _R: returncode = 0 + return _R() + + with patch("frontend.app.subprocess.run", side_effect=fake_run): + r = client.post("/api/clone-and-build", json={ + "name": "linkedin", + "repo_url": "https://e.com/r.git", + "build_commands": ["npm install"], + }) + wd = Path(r.json()["workdir"]) + env_file = wd / ".env" + assert env_file.exists() + assert "NEED_ME=supplied" in env_file.read_text() + + +class TestScanEnvExampleEndpoint: + def test_returns_keys_for_existing_workdir(self, client, tmp_path): + (tmp_path / ".env.example").write_text("A=\nB=\n") + r = client.post("/api/scan-env-example", json={"workdir": str(tmp_path)}) + assert r.json()["ok"] is True + assert r.json()["env_keys"] == ["A", "B"] + + def test_missing_workdir_400(self, client): + r = client.post("/api/scan-env-example", json={}) + assert r.status_code == 400 diff --git a/tests/test_process_runner.py b/tests/test_process_runner.py index d6f0d69..9c252fe 100644 --- a/tests/test_process_runner.py +++ b/tests/test_process_runner.py @@ -36,3 +36,42 @@ def test_no_cwd_is_distinct_from_explicit_none(self): a = process_runner.get_session("echo hi") b = process_runner.get_session("echo hi", cwd=None) assert a is b + + def test_different_env_keys_are_distinct(self): + a = process_runner.get_session("echo hi", env_keys=["A"]) + b = process_runner.get_session("echo hi", env_keys=["B"]) + assert a is not b + assert a.env_keys == ["A"] + assert b.env_keys == ["B"] + + def test_same_env_keys_returns_same_session(self): + a = process_runner.get_session("echo hi", env_keys=["A", "B"]) + b = process_runner.get_session("echo hi", env_keys=["A", "B"]) + assert a is b + + +class TestBuildEnv: + def test_inherits_os_environ(self, monkeypatch): + monkeypatch.setenv("MY_INHERITED", "yes") + s = process_runner.ProcessSession("echo hi", env_keys=["MY_INHERITED"]) + env = s._build_env() + assert env["MY_INHERITED"] == "yes" + + def test_reads_from_mcp_env_file(self, tmp_path, monkeypatch): + # Simulate a user adding a secret via the UI after server start β€” + # the value should be picked up from MCP_ENV_FILE on next spawn. + env_file = tmp_path / ".env" + env_file.write_text("MY_NEW_SECRET=freshvalue\n") + monkeypatch.setenv("MCP_ENV_FILE", str(env_file)) + monkeypatch.delenv("MY_NEW_SECRET", raising=False) + s = process_runner.ProcessSession("echo hi", env_keys=["MY_NEW_SECRET"]) + env = s._build_env() + assert env["MY_NEW_SECRET"] == "freshvalue" + + def test_no_env_keys_skips_file_read(self, tmp_path, monkeypatch): + # When env_keys is empty, the session should not touch MCP_ENV_FILE + # β€” it just inherits os.environ. + monkeypatch.setenv("MCP_ENV_FILE", str(tmp_path / "nonexistent")) + s = process_runner.ProcessSession("echo hi") + env = s._build_env() # must not raise + assert isinstance(env, dict) diff --git a/tests/test_server.py b/tests/test_server.py index 3b002d0..6478700 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -35,6 +35,7 @@ resolve_env_defaults, run_provider_setup, tool_is_enabled, + write_workdir_env_file, ) import tool_registry @@ -872,9 +873,10 @@ def test_session_created_with_repo_workdir(self, tmp_path: Path): def fake_decorator(**kwargs): return lambda fn: fn - def fake_get_session(command, cwd=None): + def fake_get_session(command, cwd=None, env_keys=None): captured["command"] = command captured["cwd"] = cwd + captured["env_keys"] = env_keys class _Sess: async def call_tool(self, *a, **kw): return {"ok": True} return _Sess() @@ -891,3 +893,100 @@ async def call_tool(self, *a, **kw): return {"ok": True} assert captured["cwd"] == "/app/repos/linkedin" assert captured["command"] == "node dist/main.js" + + +class TestWriteWorkdirEnvFile: + def test_writes_only_set_keys(self, tmp_path: Path, monkeypatch): + monkeypatch.setenv("LINKEDIN_EMAIL", "user@example.com") + monkeypatch.delenv("LINKEDIN_PASSWORD", raising=False) + target = write_workdir_env_file(str(tmp_path), ["LINKEDIN_EMAIL", "LINKEDIN_PASSWORD"]) + assert target == tmp_path / ".env" + text = target.read_text() + assert "LINKEDIN_EMAIL=user@example.com" in text + assert "LINKEDIN_PASSWORD" not in text + + def test_empty_keys_list_writes_empty_file(self, tmp_path: Path): + target = write_workdir_env_file(str(tmp_path), []) + assert target.exists() + assert target.read_text() == "" + + +class TestMaterializeRepositoryEnvFile: + def test_env_file_written_before_build(self, tmp_path: Path, monkeypatch): + wd = str(tmp_path / "wd") + spec = { + "_config_path": str(tmp_path / "r.yaml"), + "repository": { + "url": "https://example.com/r.git", + "workdir": wd, + "build_commands": ["npm install"], + "env_keys": ["MY_SECRET"], + }, + } + monkeypatch.setenv("MY_SECRET", "value123") + + observed_env_at_build = {} + + def fake_run(args, **kwargs): + if "npm" in args: + # When the build command runs, the .env file should exist + env_file = Path(wd) / ".env" + observed_env_at_build["exists"] = env_file.exists() + observed_env_at_build["content"] = env_file.read_text() if env_file.exists() else "" + class _R: returncode = 0 + return _R() + + with patch("server.subprocess.run", side_effect=fake_run): + materialize_repository(spec) + + assert observed_env_at_build["exists"] is True + assert "MY_SECRET=value123" in observed_env_at_build["content"] + + def test_no_env_keys_no_dot_env(self, tmp_path: Path): + wd = str(tmp_path / "wd") + spec = { + "_config_path": str(tmp_path / "r.yaml"), + "repository": { + "url": "https://example.com/r.git", + "workdir": wd, + "build_commands": [], + }, + } + with patch("server.subprocess.run"): + materialize_repository(spec) + assert not (Path(wd) / ".env").exists() + + +class TestRegisterProviderEnvKeys: + def test_env_keys_passed_to_get_session(self, tmp_path: Path): + spec = { + "_config_path": str(tmp_path / "linkedin.yaml"), + "package": {"command": "node dist/main.js"}, + "repository": { + "url": "https://example.com/r.git", + "workdir": "/app/repos/linkedin", + "build_commands": [], + "env_keys": ["A", "B"], + }, + "tools": [{ + "name": "do_thing", + "description": "x", + "input_schema": {"type": "object", "properties": {}, "required": []}, + }], + } + captured = {} + + def fake_get_session(command, cwd=None, env_keys=None): + captured["env_keys"] = env_keys + class _S: + async def call_tool(self, *a, **kw): return {"ok": True} + return _S() + + with patch("server.mcp") as mock_mcp, \ + patch("process_runner.get_session", side_effect=fake_get_session): + mock_mcp.tool.side_effect = lambda **k: lambda fn: fn + register_provider(spec) + entry = tool_registry.get("linkedin__do_thing") + import asyncio + asyncio.run(entry["handler"](None)) + assert captured["env_keys"] == ["A", "B"]