diff --git a/README-cn.md b/README-cn.md index fc45408..09f6273 100644 --- a/README-cn.md +++ b/README-cn.md @@ -50,8 +50,20 @@ uv run yada --task-file issue.md --workspace /workspace --yes Yada 会打印每轮 DeepSeek 调用和工具执行，最后报告任务是否通过验证门槛。默认 Trace 保存在目标仓库的 `.yada/runs/` 目录下。 -仓库测试可以执行任意代码。Yada 提供 Guardrail，但不是完整的操作系统沙箱； -处理陌生项目时请使用一次性 VM 或容器。 +陌生项目可能包含并执行任意代码。Yada 虽然提供了 Guardrail，但并不是完整的 +操作系统沙箱，运行这类项目仍可能危及系统安全。处理陌生项目时，请使用一次性 +VM 或容器。 + +## 检查每一个步骤 + +将任意 JSONL Trace 转换成完全离线、自包含的可视化页面，无需服务器、CDN 或 +额外运行时依赖： + +```bash +uv run yada-trace TRACE.jsonl --html +``` + +[![Yada 离线 Trace Viewer](docs/assets/yada-trace-viewer.jpg)](docs/assets/yada-trace-viewer.jpg) ## 更多文档 diff --git a/README.md b/README.md index a51e15f..b2a71e9 100644 --- a/README.md +++ b/README.md @@ -51,8 +51,21 @@ Yada prints each DeepSeek turn and tool execution, then reports whether the task passed its verification gate. Traces are written under the target repository's `.yada/runs/` directory by default. -Repository tests can execute arbitrary code. Yada provides guardrails, not a -complete OS sandbox; use a disposable VM or container for unfamiliar projects. +Unfamiliar projects may contain and execute arbitrary code. Yada provides +guardrails, but it is not a complete operating-system sandbox, so running such +projects can still put your system at risk. Use a disposable VM or container +when working with unfamiliar projects. + +## Inspect every step + +Turn any JSONL trace into a self-contained offline viewer—no server, CDN, or +additional runtime dependency required: + +```bash +uv run yada-trace TRACE.jsonl --html +``` + +[![Yada offline trace viewer](docs/assets/yada-trace-viewer.jpg)](docs/assets/yada-trace-viewer.jpg) ## Learn more diff --git a/docs/assets/yada-trace-viewer.jpg b/docs/assets/yada-trace-viewer.jpg new file mode 100644 index 0000000..0851d92 Binary files /dev/null and b/docs/assets/yada-trace-viewer.jpg differ diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 2e32401..7a29447 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -301,7 +301,7 @@ errors or non-verdict outcomes such as skipped grading. ## `yada-trace` ```text -yada-trace TRACE.jsonl [--step N | --verbose | --events] +yada-trace TRACE.jsonl [--step N | --verbose | --events | --html [PATH]] ``` | Option | Meaning | @@ -309,6 +309,7 @@ yada-trace TRACE.jsonl [--step N | --verbose | --events] | `--step N` | Expand one complete request → response → tools step. | | `--verbose` | Expand payloads inside every grouped step. | | `--events` | Show the flat event timeline with physical JSONL line numbers. | +| `--html [PATH]` | Write one self-contained offline HTML viewer. Without `PATH`, write it beside the trace and open it. | The default view groups records by agent step and shows physical JSONL line references for the model request/response and each tool call/result pair: @@ -317,8 +318,19 @@ references for the model request/response and each tool call/result pair: uv run yada-trace TRACE.jsonl uv run yada-trace TRACE.jsonl --step 8 uv run yada-trace TRACE.jsonl --events +uv run yada-trace TRACE.jsonl --html +uv run yada-trace TRACE.jsonl --html trace.html ``` +Without an output path, `--html` writes `TRACE.html` beside `TRACE.jsonl` and +opens it in the default browser. Pass a path to export the viewer without opening +it. The HTML viewer groups the same validated records by step and adds local step +filtering, failure/file-change filters, collapsed large payloads, and the final +diff. It contains inline CSS and JavaScript only: opening the file +does not start a server or request external resources. The generated file can +contain prompts, reasoning, source, patches, and command output from the source +trace, so handle it with the same care as the JSONL. + `yada-trace` returns `0` after rendering and `2` for an unreadable or invalid trace. See the [debugging guide](dev/debugging.md) for event semantics, trace levels, `jq` recipes, and reproduction workflows. diff --git a/docs/dev/debugging.md b/docs/dev/debugging.md index 60dd920..7542243 100644 --- a/docs/dev/debugging.md +++ b/docs/dev/debugging.md @@ -109,6 +109,23 @@ investigation: uv run yada-trace TRACE.jsonl --events ``` +For longer runs, generate a portable semantic view and open it directly in a +browser: + +```bash +uv run yada-trace TRACE.jsonl --html +``` + +This writes `TRACE.html` beside the JSONL and opens it in the default browser. +Pass an explicit path after `--html` to export without opening it. The single +HTML file works offline and groups requests, reasoning, responses, +plans, tool calls/results, failures, and the final diff by step. Step filtering +runs locally in the browser. Large prompts, patches, and command output +are collapsed by default. The viewer preserves redaction from the JSONL and +cannot recover omitted or redacted fields. + +[![Yada offline trace viewer](../assets/yada-trace-viewer.jpg)](../assets/yada-trace-viewer.jpg) + Then inspect exact records using the line references: ```bash diff --git a/src/yada/traces/__init__.py b/src/yada/traces/__init__.py index 0889a2f..0ce0bde 100644 --- a/src/yada/traces/__init__.py +++ b/src/yada/traces/__init__.py @@ -1,5 +1,6 @@ """Trace persistence and human-readable run diagnostics.""" +from yada.traces.html import render_trace_html, write_trace_html from yada.traces.jsonl import TRACE_LEVELS, TRACE_SCHEMA_VERSION, TraceWriter from yada.traces.report import ( LocatedTraceEvent, @@ -27,5 +28,7 @@ "read_located_trace", "read_trace", "reconstruct_model_request", + "render_trace_html", "render_trace_report", + "write_trace_html", ] diff --git a/src/yada/traces/cli.py b/src/yada/traces/cli.py index d1d1a9b..88de426 100644 --- a/src/yada/traces/cli.py +++ b/src/yada/traces/cli.py @@ -4,10 +4,14 @@ import argparse import sys +import webbrowser from pathlib import Path +from yada.traces.html import write_trace_html from yada.traces.report import TraceFormatError, render_trace_report +_AUTO_HTML = object() + def build_parser() -> argparse.ArgumentParser: """Create the parser for the lightweight ``yada-trace`` command.""" @@ -32,6 +36,17 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help="Show the legacy flat event timeline with physical line numbers.", ) + parser.add_argument( + "--html", + nargs="?", + const=_AUTO_HTML, + type=Path, + metavar="PATH", + help=( + "Write a self-contained offline HTML viewer; omit PATH to write " + "beside TRACE and open it." + ), + ) return parser @@ -40,8 +55,30 @@ def run_cli(argv: list[str] | None = None) -> int: args = build_parser().parse_args(argv) try: + trace_path = args.trace.expanduser().resolve() + if args.html is not None: + if args.step is not None or args.verbose or args.events: + raise TraceFormatError( + "--html cannot be combined with --step, --verbose, or --events" + ) + auto_open = args.html is _AUTO_HTML + output_path = ( + trace_path.with_suffix(".html") + if auto_open + else args.html.expanduser().resolve() + ) + write_trace_html(trace_path, output_path) + if auto_open and _open_html(output_path): + print(f"Wrote and opened offline trace viewer: {output_path}") + else: + print(f"Wrote offline trace viewer: {output_path}") + if auto_open: + print( + "Could not open it automatically; open the file in a browser." + ) + return 0 report = render_trace_report( - args.trace.expanduser().resolve(), + trace_path, step=args.step, verbose=args.verbose, events=args.events, @@ -63,6 +100,13 @@ def _positive_step(value: str) -> int: return step +def _open_html(path: Path) -> bool: + try: + return webbrowser.open(path.as_uri(), new=2) + except (OSError, webbrowser.Error): + return False + + def main() -> None: """Console-script entry point.""" diff --git a/src/yada/traces/html.py b/src/yada/traces/html.py new file mode 100644 index 0000000..5d2babb --- /dev/null +++ b/src/yada/traces/html.py @@ -0,0 +1,565 @@ +"""Self-contained offline HTML rendering for Yada traces.""" + +from __future__ import annotations + +import html +import json +from pathlib import Path +from typing import Any + +from yada.traces.report import ( + LocatedTraceEvent, + TraceFormatError, + TraceRun, + TraceStep, + TraceToolExecution, + build_trace_run, + read_located_trace, +) + +_EDIT_TOOLS = {"apply_patch", "replace_text"} +_COLLAPSE_CHARS = 4_000 +_TITLE_CHARS = 120 + + +def render_trace_html(path: Path) -> str: + """Render one validated JSONL trace as a portable offline HTML document.""" + + run = build_trace_run(read_located_trace(path)) + start = run.run_start.data if run.run_start else {} + run_id = _run_id(run) + panels = "".join( + _render_step(step, selected=index == 0) for index, step in enumerate(run.steps) + ) + navigation = "".join( + _render_step_button(step, selected=index == 0) + for index, step in enumerate(run.steps) + ) + if not run.steps: + panels = '

No agent steps were recorded.

' + + content = ( + _render_header(path, run) + + _render_run_details(run) + + '
' + + '" + + f'
{panels}
' + + "
" + ) + title = f"Yada trace · {run_id} · {start.get('model', 'unknown')}" + return _DOCUMENT.replace("__TITLE__", _escape(title), 1).replace( + "__CONTENT__", content, 1 + ) + + +def write_trace_html(trace_path: Path, output_path: Path) -> None: + """Validate ``trace_path`` and write its self-contained HTML view.""" + + trace_path = trace_path.resolve() + output_path = output_path.resolve() + if trace_path == output_path: + raise TraceFormatError("--html output must differ from the source trace") + document = render_trace_html(trace_path) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(document, encoding="utf-8") + + +def _render_header(path: Path, run: TraceRun) -> str: + start = run.run_start.data if run.run_start else {} + task = start.get("task") + outcome, outcome_class = _outcome(run) + token_count = _run_tokens(run) + latency = _last_elapsed(run) + interrupted = "" + if run.run_end is None: + interrupted = ( + '

Interrupted trace: missing run_end. ' + "The final state may be unavailable.

" + ) + return f""" +
+
+

Yada trace viewer · offline

+

{_escape(_task_title(task))}

+

{_escape(path)} · run {_escape(_run_id(run))}

+
+
+ {_metric("Outcome", outcome, outcome_class)} + {_metric("Model", start.get("model", "unknown"))} + {_metric("Tokens", f"{token_count:,}" if token_count is not None else "—")} + {_metric("Latency", _duration(latency) if latency is not None else "—")} + {_metric("Steps", len(run.steps))} +
+
+{interrupted} +""" + + +def _render_run_details(run: TraceRun) -> str: + start = run.run_start.data if run.run_start else {} + end = run.run_end.data if run.run_end else {} + details = '
' + details += _details( + "Task", + start.get("task", "Unavailable"), + open_by_default=False, + class_name="task-detail", + ) + details += _details( + "Model configuration", + start.get("model_config", "Unavailable"), + open_by_default=False, + ) + details += _details( + "Provenance", + start.get("provenance", "Unavailable"), + open_by_default=False, + ) + final_state = end.get("final_state") + if isinstance(final_state, dict): + details += _details( + "Final status", + { + "git_status": final_state.get("git_status"), + "diff_stat": final_state.get("diff_stat"), + }, + open_by_default=False, + ) + details += _details( + "Final diff", + final_state.get("diff") or "No diff recorded.", + open_by_default=False, + ) + else: + details += _details( + "Final diff", + "Unavailable: the trace has no captured final state.", + open_by_default=False, + ) + return details + "
" + + +def _render_filters() -> str: + filters = ( + ("model-error", "Model errors"), + ("protocol-violation", "Protocol violations"), + ("rejected-tool", "Rejected tools"), + ("nonzero-exit", "Non-zero exits"), + ("file-change", "File-changing steps"), + ) + controls = "".join( + f'' + for value, label in filters + ) + return f'
Filters{controls}
' + + +def _render_step_button(step: TraceStep, *, selected: bool) -> str: + flags = _step_flags(step) + response = step.model_response + duration = response.data.get("duration_ms") if response else None + tokens = _total_tokens(response.data.get("usage")) if response else None + facts = [] + if isinstance(duration, (int, float)): + facts.append(_duration(duration)) + if tokens is not None: + facts.append(f"{tokens:,} tokens") + if not facts: + facts.append("interrupted" if _step_interrupted(step) else "no metrics") + classes = "step-link selected" if selected else "step-link" + flag_text = " ".join(flags) + return ( + f'" + ) + + +def _render_step(step: TraceStep, *, selected: bool) -> str: + flags = _step_flags(step) + hidden = "" if selected else " hidden" + flag_text = " ".join(flags) + badges = "".join(f'{_escape(flag)}' for flag in flags) + warnings = _render_step_warnings(step) + request = _located(step.model_request) + response = _response(step.model_response) + plan = _located(step.plan_decision) + calls = [_located(execution.call) for execution in step.tool_executions] + results = [_located(execution.result) for execution in step.tool_executions] + return f""" +
+
+

Lines {step.first_line}–{step.last_line}

+

Step {step.number}

+
{badges}
+
+ {warnings} + {_details("Request", request or "Missing model request", open_by_default=False)} + {_details("Response", response or "Missing model response", open_by_default=True, searchable=True)} + {_details("Plan", plan or "No plan decision recorded", open_by_default=True, searchable=True)} + {_details("Tool Calls", calls or "No tool calls", open_by_default=True, searchable=True)} + {_details("Tool Results", results or "No tool results", open_by_default=True, searchable=True)} +
+""" + + +def _render_step_warnings(step: TraceStep) -> str: + warnings = [] + if step.model_request is None: + warnings.append("Missing model request") + if step.model_request is not None and step.model_response is None: + warnings.append("Missing model response") + for execution in step.tool_executions: + if execution.call is None: + warnings.append("Unmatched tool result") + if execution.result is None: + warnings.append("Missing tool result") + if not warnings: + return "" + unique = " · ".join(dict.fromkeys(warnings)) + return f'

{_escape(unique)}

' + + +def _step_flags(step: TraceStep) -> tuple[str, ...]: + flags = [] + if step.model_response and step.model_response.name == "model_error": + flags.append("model-error") + if any(event.name == "protocol_violation" for event in step.protocol_events): + flags.append("protocol-violation") + if any( + execution.call and execution.call.data.get("rejected") is True + for execution in step.tool_executions + ): + flags.append("rejected-tool") + if any(_nonzero_exit(execution) for execution in step.tool_executions): + flags.append("nonzero-exit") + if any(_changes_files(execution) for execution in step.tool_executions): + flags.append("file-change") + if _step_interrupted(step): + flags.append("incomplete") + return tuple(flags) + + +def _step_interrupted(step: TraceStep) -> bool: + return bool( + (step.model_request and step.model_response is None) + or any( + execution.call is None or execution.result is None + for execution in step.tool_executions + ) + ) + + +def _nonzero_exit(execution: TraceToolExecution) -> bool: + result = _tool_result(execution) + exit_code = result.get("exit_code") + return isinstance(exit_code, int) and exit_code != 0 + + +def _changes_files(execution: TraceToolExecution) -> bool: + result = _tool_result(execution) + if result.get("ok") is not True: + return False + changed = result.get("changed_files") + if isinstance(changed, list) and changed: + return True + source = execution.call or execution.result + return bool(source and source.data.get("tool") in _EDIT_TOOLS) + + +def _tool_result(execution: TraceToolExecution) -> dict[str, Any]: + if execution.result is None: + return {} + result = execution.result.data.get("result") + return result if isinstance(result, dict) else {} + + +def _response(response: LocatedTraceEvent | None) -> dict[str, Any] | None: + located = _located(response) + if located is None: + return located + located["data"] = { + key: value + for key, value in located["data"].items() + if key != "message_field_presence" + } + return located + + +def _located(event: LocatedTraceEvent | None) -> dict[str, Any] | None: + if event is None: + return None + return { + "jsonl_line": event.line_number, + "event": event.name, + "data": event.data, + } + + +def _details( + title: str, + value: Any, + *, + open_by_default: bool, + searchable: bool = False, + class_name: str = "", +) -> str: + rendered = _json_text(value) + open_attribute = ( + " open" if open_by_default and len(rendered) <= _COLLAPSE_CHARS else "" + ) + search_attribute = " data-searchable" if searchable else "" + classes = f"trace-section {class_name}".rstrip() + return ( + f'
' + f"{_escape(title)}
{_escape(rendered)}
" + ) + + +def _json_text(value: Any) -> str: + if isinstance(value, str): + return value + return json.dumps( + value, + ensure_ascii=False, + indent=2, + sort_keys=True, + default=str, + ) + + +def _metric(label: str, value: Any, class_name: str = "") -> str: + css_class = f' class="{class_name}"' if class_name else "" + return ( + '
' + f"{_escape(label)}{_escape(value)}" + "
" + ) + + +def _outcome(run: TraceRun) -> tuple[str, str]: + if run.run_end is None: + return "Interrupted", "error-text" + steps = run.run_end.data.get("steps") + suffix = "" + if isinstance(steps, int): + suffix = f" · {steps} {'step' if steps == 1 else 'steps'}" + if run.run_end.data.get("finished") is True: + return f"Resolved{suffix}", "success-text" + return f"Unfinished{suffix}", "error-text" + + +def _run_id(run: TraceRun) -> str: + return str( + next( + ( + event.record.get("run_id") + for event in run.events + if event.record.get("run_id") + ), + "legacy", + ) + ) + + +def _run_tokens(run: TraceRun) -> int | None: + if run.run_end: + total = _total_tokens(run.run_end.data.get("usage")) + if total is not None: + return total + totals = [ + _total_tokens(event.data.get("usage")) + for event in run.events + if event.name == "assistant" + ] + values = [value for value in totals if value is not None] + return sum(values) if values else None + + +def _total_tokens(usage: Any) -> int | None: + if not isinstance(usage, dict): + return None + total = usage.get("total_tokens") + if isinstance(total, int): + return total + prompt = usage.get("prompt_tokens") + completion = usage.get("completion_tokens") + if isinstance(prompt, int) and isinstance(completion, int): + return prompt + completion + return None + + +def _last_elapsed(run: TraceRun) -> int | float | None: + values = [ + event.record.get("elapsed_ms") + for event in run.events + if isinstance(event.record.get("elapsed_ms"), (int, float)) + ] + return max(values) if values else None + + +def _duration(milliseconds: int | float) -> str: + if milliseconds < 1000: + return f"{milliseconds:g} ms" + seconds = milliseconds / 1000 + return f"{seconds:.1f} s" if seconds < 10 else f"{seconds:g} s" + + +def _task_title(task: Any) -> str: + if not isinstance(task, str): + return "Untitled run" + title = next((line.strip() for line in task.splitlines() if line.strip()), "") + if not title: + return "Untitled run" + if len(title) <= _TITLE_CHARS: + return title + return title[: _TITLE_CHARS - 1].rstrip() + "…" + + +def _escape(value: Any) -> str: + return html.escape(str(value), quote=True) + + +_DOCUMENT = """ + + + + + +__TITLE__ + + +
__CONTENT__
+ + + +""" diff --git a/tests/traces/test_trace_html.py b/tests/traces/test_trace_html.py new file mode 100644 index 0000000..c456d46 --- /dev/null +++ b/tests/traces/test_trace_html.py @@ -0,0 +1,277 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from yada.traces import TRACE_SCHEMA_VERSION, TraceWriter, render_trace_html +from yada.traces.cli import run_cli + + +def test_completed_trace_renders_offline_semantic_view( + tmp_path: Path, capsys, monkeypatch +) -> None: + trace_path = tmp_path / "completed.jsonl" + output_path = tmp_path / "viewer.html" + opened = [] + monkeypatch.setattr( + "yada.traces.cli.webbrowser.open", + lambda url, new: opened.append((url, new)) or True, + ) + trace = TraceWriter(trace_path, level="debug", run_id="html-test") + trace.write( + "run_start", + { + "model": "deepseek-v4-pro", + "task": "Fix parser\n\nA long issue description that stays collapsed.", + "trace_level": "debug", + "model_config": {"thinking": True}, + "provenance": {"yada_version": "0.1.0", "case_id": "parser-1"}, + }, + ) + trace.write( + "model_request", + { + "step": 1, + "request_id": "request-1", + "payload": {"messages": [{"role": "user", "content": "fix it"}]}, + }, + ) + trace.write( + "assistant", + { + "step": 1, + "request_id": "request-1", + "duration_ms": 1250, + "usage": {"total_tokens": 42}, + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": "", + "reasoning_content": "The boundary is off by one.", + "tool_calls": [], + }, + }, + ) + trace.write( + "plan_decision", + {"step": 1, "action": "execute_tools", "tools": ["apply_patch"]}, + ) + trace.write( + "tool_call", + { + "step": 1, + "tool_call_id": "patch-1", + "tool": "apply_patch", + "arguments": {"patch": "diff --git a/a.py b/a.py"}, + }, + ) + trace.write( + "tool_result", + { + "step": 1, + "tool_call_id": "patch-1", + "tool": "apply_patch", + "duration_ms": 4, + "result": { + "ok": True, + "changed_files": [{"path": "a.py", "sha256": "abc"}], + }, + }, + ) + trace.write( + "run_end", + { + "finished": True, + "steps": 1, + "usage": {"total_tokens": 42}, + "final_state": { + "git_status": " M a.py", + "diff_stat": "a.py | 2 +-", + "diff": "-wrong\n+right", + }, + }, + ) + + assert run_cli([str(trace_path), "--html", str(output_path)]) == 0 + document = output_path.read_text(encoding="utf-8") + + assert "Wrote offline trace viewer" in capsys.readouterr().out + assert "Yada trace viewer · offline" in document + assert "

Fix parser

" in document + assert ( + '
Task' in document + ) + assert "A long issue description that stays collapsed." in document + assert "Resolved · 1 step" in document + assert "The boundary is off by one." in document + assert "Reasoning" not in document + assert '"reasoning_content"' not in document + assert ""reasoning_content"" in document + assert "message_field_presence" not in document + assert "Final diff" in document + assert "-wrong\n+right" in document + assert 'data-flags="file-change"' in document + assert "" + ) + trace = TraceWriter(path, level="debug") + trace.write("run_start", {"task": attack, "model": "fake"}) + trace.write( + "assistant", + {"step": 1, "message": {"content": attack, "reasoning_content": attack}}, + ) + + document = render_trace_html(path) + + assert attack not in document + assert "</script><script>" in document + assert " dict: + return { + "schema_version": TRACE_SCHEMA_VERSION, + "run_id": "html-test", + "sequence": sequence, + "elapsed_ms": sequence, + "event": event, + "data": data, + }