From b65a31f03b2decd979b4de641ebca88aecddeb43 Mon Sep 17 00:00:00 2001 From: mouxin <494624263qq@gmail.com> Date: Mon, 13 Apr 2026 16:20:57 +0800 Subject: [PATCH] Store trace detail markdowns under detail/trace subfolder --- .../.claude/skills/troubleshoot/SKILL.md | 27 +++++- .../references/report_templates.md | 7 +- .../troubleshoot/scripts/analyzers/cache.py | 97 +++++++++++-------- .../troubleshoot/scripts/analyzers/trace.py | 73 ++++++++++++-- .../troubleshoot/scripts/troubleshoot.py | 67 ++++++++----- 5 files changed, 194 insertions(+), 77 deletions(-) diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/SKILL.md b/fastdeploy/golang_router/.claude/skills/troubleshoot/SKILL.md index 2ea74156c82..00c94a2f487 100644 --- a/fastdeploy/golang_router/.claude/skills/troubleshoot/SKILL.md +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/SKILL.md @@ -11,14 +11,14 @@ description: > 关键词：troubleshoot、排查、router 问题、全量扫描、综合分析、error、502、latency、 health、load、cache、trace、/troubleshoot。 -IMPORTANT: 执行前务必先读取 references/log_patterns.md 了解日志格式和提取规则。 -错误分类时参考 references/error_catalog.md。涉及后端问题时参考 references/fastdeploy_cross_reference.md。 --- # Router Troubleshooting 综合排查 FastDeploy Go Router 问题，输出完整诊断报告。 +> IMPORTANT: 执行前务必先读取 `references/log_patterns.md` 了解日志格式和提取规则。错误分类时参考 `references/error_catalog.md`。涉及后端问题时参考 `references/fastdeploy_cross_reference.md`。 + ## 执行前交互 运行脚本前，Claude 必须按以下顺序向用户确认参数： @@ -51,6 +51,16 @@ IMPORTANT: 执行前务必先读取 references/log_patterns.md 了解日志格 缺失部分自动从日志首末行推断（缺年份取首行，缺日期取末行）。 `--start/--end` 与 `--tail` 互斥。 +当用户选择“指定时间段”时，必须再发起一次 **AskUserQuestion**（离散选项）引导时间输入： +- 选项 1: `当天（00:00:00 到当前）`（推荐） +- 选项 2: `最近半小时`（自动换算为 `--start now-30m --end now` 语义） + +用户若通过客户端默认 `Other` 输入时间，则将该输入直接作为时间范围参数解析。 +可补充一条简短示例引导： +- 示例 1：`16:00-16:30` +- 示例 2：`03/31 16:00 ~ 03/31 18:00` +- 示例 3：`2026/03/31 16:00:00`（仅起始） + ### 3. 分析模式 必须使用 **AskUserQuestion 的离散选项**（不要只发纯文本编号）： - 选项 1: `完整分析（默认）` — 运行所有维度（errors + latency + health + cache + load） @@ -59,8 +69,12 @@ IMPORTANT: 执行前务必先读取 references/log_patterns.md 了解日志格 如果用户未选择，默认使用完整分析。 -当用户选择“请求追踪”选项时，AskUserQuestion 的选项文案应直接提示可输入： -- `trace_id/request_id/session_id`（逗号分隔多 ID） +当用户选择“请求追踪”后，**不要再发 AskUserQuestion** 收集 trace ID。 +直接发一条提示并等待用户输入完成后再继续执行即可。 + +提示文案建议： +- `请输入要追踪的 ID（支持 trace_id / request_id / session_id，多个用逗号分隔；输入 all 可全量追踪）` +- 示例：`a1b2c3d4` / `trace-001,trace-002` / `session-abc-123` / `all` ### 4. 输出目录 诊断报告默认保存到 `skill_output/troubleshoot//`（自动按运行时间创建子目录）。 @@ -86,6 +100,7 @@ python3 $SCRIPTS/troubleshoot.py --load # 请求追踪（需指定 ID，支持逗号分隔多 ID） python3 $SCRIPTS/troubleshoot.py --trace python3 $SCRIPTS/troubleshoot.py --trace "id1,id2" +python3 $SCRIPTS/troubleshoot.py --trace all # 尾部分析 python3 $SCRIPTS/troubleshoot.py --tail 5000 @@ -109,7 +124,9 @@ python3 $SCRIPTS/troubleshoot.py --start "16:00" --end "17:00" --erro - **终端**：简洁三层汇总（Router / FD 后端 / 客户端），含状态码分布、错误 Top N、趋势图 - **文件**：详细报告导出到 `skill_output/troubleshoot//summary/troubleshoot_report.md` - 逐分钟事件详情拆分到 `detail/health_events.md` - - 请求追踪事件链拆分到 `detail/trace_.md` + - 请求追踪事件链拆分到 `detail/trace/trace_.md` +- **Cache 明细要求**：`cache_session_stickiness.md` / `cache_suboptimal.md` / `cache_eviction.md` / `cache_fallback.md` / `cache_cross.md` + 必须始终生成（即使无异常也写“未发现/样本不足”总结，避免链接缺失） - **状态行**：`STATUS: HEALTHY / DEGRADED / CRITICAL` ## 三层诊断框架 diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/references/report_templates.md b/fastdeploy/golang_router/.claude/skills/troubleshoot/references/report_templates.md index cd705d02816..61db59ec7e6 100644 --- a/fastdeploy/golang_router/.claude/skills/troubleshoot/references/report_templates.md +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/references/report_templates.md @@ -62,7 +62,7 @@ - `detail/latency_diagnoses.md` — 延迟诊断详情 - `detail/cache_diagnosis.md` — cache 六维诊断详情（session 粘性/非最优/驱逐/Fallback/冷启动/交叉诊断） - `detail/cache_session_stickiness.md` / `detail/cache_suboptimal.md` / `detail/cache_eviction.md` / `detail/cache_fallback.md` / `detail/cache_cross.md` — cache 分职责拆分明细 - - `detail/trace_.md` — 请求追踪事件链 + - `detail/trace/trace_.md` — 请求追踪事件链 --- @@ -102,15 +102,18 @@ Worker 可用性时间线 可用性统计 ``` -### Cache（调度诊断）— 待实现 +### Cache（调度诊断） ``` 调度策略分布 Session 粘性分析 非最优选择分析 Fallback 原因分类 +驱逐影响与交叉诊断 ``` +要求：即使某项计数为 0（例如“非最优选择”），也要输出该小节并给出“未发现/样本不足”总结，保证 detail 链接稳定存在。 + ### Load（负载分析）— 待实现 ``` diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/cache.py b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/cache.py index 57a1490d3fd..a12341967a0 100644 --- a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/cache.py +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/cache.py @@ -449,11 +449,11 @@ def format_cache_report(result): # Session 粘性 stickiness = result.get("session_stickiness", {}) + sections.append("### Session 粘性") + sections.append("") + sections.append(" Session 粘性详情见: [detail/cache_session_stickiness.md](../detail/cache_session_stickiness.md)") + sections.append("") if stickiness: - sections.append("### Session 粘性") - sections.append("") - sections.append(" Session 粘性详情见: [detail/cache_diagnosis.md](../detail/cache_diagnosis.md)") - sections.append("") table_data = [ { "Session": sid, @@ -473,14 +473,21 @@ def format_cache_report(result): ) ) detail_sections.append("") + else: + sections.append(" 未检测到可计算粘性的多请求 Session。") + sections.append("") + detail_sections.append("## Session 粘性") + detail_sections.append("") + detail_sections.append("- 无可用样本（需要同一 session 至少 2 次请求）。") + detail_sections.append("") # 非最优选择 - if result.get("suboptimal_selections"): - subs = result["suboptimal_selections"] - sections.append(f"### 非最优选择 ({len(subs)} 次)") - sections.append("") - sections.append(" 详情见: [detail/cache_diagnosis.md](../detail/cache_diagnosis.md)") - sections.append("") + subs = result.get("suboptimal_selections") or [] + sections.append(f"### 非最优选择 ({len(subs)} 次)") + sections.append("") + sections.append(" 详情见: [detail/cache_suboptimal.md](../detail/cache_suboptimal.md)") + sections.append("") + if subs: reason_counts = defaultdict(int) for s in subs: reason_counts[s["reason"]] += 1 @@ -494,15 +501,22 @@ def format_cache_report(result): f'- [{s.get("ts","")}] selected={s.get("selected","")}({s.get("selected_hr",0)}), best={s.get("best_hr_worker","")}({s.get("best_hr",0)}), reason={s.get("reason","")}' ) detail_sections.append("") + else: + sections.append(" 未发现非最优选择（selected_hitRatio 始终为当次最高）。") + sections.append("") + detail_sections.append("## 非最优选择") + detail_sections.append("") + detail_sections.append("- 未发现非最优选择。") + detail_sections.append("") # 驱逐影响 - if result.get("eviction_impact"): - evictions = result["eviction_impact"] - evicted = [e for e in evictions if e["evicted"]] - sections.append(f"### 驱逐影响 ({len(evictions)} 次超时, {len(evicted)} 次缓存失效)") - sections.append("") - sections.append(" 详情见: [detail/cache_diagnosis.md](../detail/cache_diagnosis.md)") - sections.append("") + evictions = result.get("eviction_impact") or [] + evicted = [e for e in evictions if e["evicted"]] + sections.append(f"### 驱逐影响 ({len(evictions)} 次超时, {len(evicted)} 次缓存失效)") + sections.append("") + sections.append(" 详情见: [detail/cache_eviction.md](../detail/cache_eviction.md)") + sections.append("") + if evictions: detail_sections.append("## 驱逐影响") detail_sections.append("") for e in evictions[:50]: @@ -510,6 +524,13 @@ def format_cache_report(result): f'- session={e.get("session_id","")[:24]} interval={e.get("interval_mins",0)}m hitRatio_after={e.get("hitRatio_after",0)} evicted={e.get("evicted",False)}' ) detail_sections.append("") + else: + sections.append(" 未检测到超时导致的潜在驱逐影响。") + sections.append("") + detail_sections.append("## 驱逐影响") + detail_sections.append("") + detail_sections.append("- 未检测到超时驱逐样本。") + detail_sections.append("") # 冷启动 if result.get("cold_starts", 0) > 0: @@ -520,11 +541,11 @@ def format_cache_report(result): detail_sections.append(f'- 冷启动次数: {result["cold_starts"]}') detail_sections.append("") + sections.append("### 交叉诊断") + sections.append("") + sections.append(" 详情见: [detail/cache_cross.md](../detail/cache_cross.md)") + sections.append("") if result.get("cross_diagnosis"): - sections.append("### 交叉诊断") - sections.append("") - sections.append(" 详情见: [detail/cache_diagnosis.md](../detail/cache_diagnosis.md)") - sections.append("") detail_sections.append("## 交叉诊断") detail_sections.append("") detail_sections.append( @@ -542,25 +563,23 @@ def format_cache_report(result): ) ) detail_sections.append("") - - # 只显示实际生成了文件的链接 - detail_links = [] - if result.get("session_stickiness"): - detail_links.append("[detail/cache_session_stickiness.md](../detail/cache_session_stickiness.md)") - if result.get("suboptimal_selections"): - detail_links.append("[detail/cache_suboptimal.md](../detail/cache_suboptimal.md)") - if result.get("eviction_impact"): - detail_links.append("[detail/cache_eviction.md](../detail/cache_eviction.md)") - if result.get("fallback_reasons"): - detail_links.append("[detail/cache_fallback.md](../detail/cache_fallback.md)") - if result.get("cross_diagnosis"): - detail_links.append("[detail/cache_cross.md](../detail/cache_cross.md)") - - if detail_links: - sections.append( - "> 详细诊断: [detail/cache_diagnosis.md](../detail/cache_diagnosis.md) | " + " | ".join(detail_links) - ) + else: + sections.append(" 样本不足，未生成交叉诊断。") sections.append("") + detail_sections.append("## 交叉诊断") + detail_sections.append("") + detail_sections.append("- 样本不足，未生成交叉诊断。") + detail_sections.append("") + + sections.append( + "> 详细诊断: [detail/cache_diagnosis.md](../detail/cache_diagnosis.md) | " + "[detail/cache_session_stickiness.md](../detail/cache_session_stickiness.md) | " + "[detail/cache_suboptimal.md](../detail/cache_suboptimal.md) | " + "[detail/cache_eviction.md](../detail/cache_eviction.md) | " + "[detail/cache_fallback.md](../detail/cache_fallback.md) | " + "[detail/cache_cross.md](../detail/cache_cross.md)" + ) + sections.append("") return "\n".join(sections), "\n".join(detail_sections) diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/trace.py b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/trace.py index 37006121994..d0dcbdca6d9 100644 --- a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/trace.py +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/trace.py @@ -62,8 +62,13 @@ def analyze_trace(log_file, trace_ids, tail=None): Returns: dict: {traces: {id: {events, lifecycle_complete, diagnoses}}, summary} """ + auto_discovery_summary = "" if isinstance(trace_ids, str): - trace_ids = [tid.strip() for tid in trace_ids.split(",") if tid.strip()] + normalized = trace_ids.strip().lower() + if normalized in ("all", "full", "all_ids", "全部", "全量"): + trace_ids, auto_discovery_summary = _discover_full_trace_targets(log_file, tail=tail) + else: + trace_ids = [tid.strip() for tid in trace_ids.split(",") if tid.strip()] if not trace_ids: return {"traces": {}, "summary": "未指定追踪 ID"} @@ -132,10 +137,64 @@ def analyze_trace(log_file, trace_ids, tail=None): total_traced = len(traces) complete = sum(1 for t in traces.values() if t["lifecycle_complete"]) - return { - "traces": traces, - "summary": f"{total_traced} ID(s) 追踪, {complete} 生命周期完整", - } + summary = f"{total_traced} ID(s) 追踪, {complete} 生命周期完整" + if auto_discovery_summary: + summary += f" | {auto_discovery_summary}" + + return {"traces": traces, "summary": summary} + + +def _discover_full_trace_targets(log_file, tail=None): + """全量追踪目标发现。 + + 规则： + 1) 有 session_id 的优先按 session_id 追踪 + 2) 无 session 但有 trace_id 的按 trace_id 追踪 + 3) 剩余“孤立”的 request_id/req_id 单独追踪 + """ + lines = _grep_lines(log_file, r"session_id:|trace_id:|request_id:|req_id:", tail=tail) + if not lines: + return [], "全量追踪未发现任何可用 ID" + + session_ids = set() + trace_ids = set() + all_request_ids = set() + request_ids_with_session_or_trace = set() + + for line in lines: + tags = extract_tags(line) + sid = tags.get("session_id") + tid = tags.get("trace_id") + rid = tags.get("request_id") or tags.get("req_id") + has_session = bool(sid) + has_trace = bool(tid) + has_request = bool(rid) + + if has_session: + session_ids.add(sid) + if has_trace: + trace_ids.add(tid) + if has_request: + all_request_ids.add(rid) + if has_session or has_trace: + request_ids_with_session_or_trace.add(rid) + + standalone_request_ids = all_request_ids - request_ids_with_session_or_trace + + targets = [] + chosen = set() + for bucket in (sorted(session_ids), sorted(trace_ids), sorted(standalone_request_ids)): + for _id in bucket: + if _id and _id not in chosen: + chosen.add(_id) + targets.append(_id) + + summary = ( + "全量ID发现: " + f"session={len(session_ids)}, trace={len(trace_ids)}, " + f"standalone_request={len(standalone_request_ids)}, total_targets={len(targets)}" + ) + return targets, summary def _parse_event_chain(lines): @@ -426,7 +485,9 @@ def format_trace_report(result): # 主报告中添加引用和摘要 safe_tid = tid.replace("/", "_") sections.append(f' 事件数: {len(trace["events"])}') - sections.append(f" > 完整事件链: [detail/trace_{safe_tid}.md](../detail/trace_{safe_tid}.md)") + sections.append( + f" > 完整事件链: [detail/trace/trace_{safe_tid}.md](../detail/trace/trace_{safe_tid}.md)" + ) sections.append("") return "\n".join(sections), detail_dict diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/troubleshoot.py b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/troubleshoot.py index d869f9c71cc..251a21c7e81 100644 --- a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/troubleshoot.py +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/troubleshoot.py @@ -11,7 +11,7 @@ --health 仅分析 Worker 健康 --cache 仅分析 Cache 调度 --load 仅分析负载与计数器 - --trace ID 追踪指定请求（支持逗号分隔多 ID） + --trace ID 追踪指定请求（支持逗号分隔多 ID；传 all 可全量追踪） --tail N 仅分析尾部 N 行（支持 N 或 Nm 格式如 30m） --start TIME 起始时间（如 "16:00:00"、"03/31 16:00"） --end TIME 结束时间（如 "17:00:00"、"2026/03/31 17:00:00"） @@ -191,7 +191,7 @@ def format_full_report(results, status, status_reason): details: dict 包含需要拆分到独立文件的详情数据 - 'health_events': str 或 None - 'load_select_release': str 或 None - - 'trace_files': {trace_id: text} 或 {} + - 'trace_files': {trace_id: text} 或 {}（写入 detail/trace/） """ parts = [] details = { @@ -292,44 +292,58 @@ def format_full_report(results, status, status_reason): if detail: details["cache_diagnosis"] = detail c = results["cache"] + lines = ["# Cache Session 粘性详情", ""] if c.get("session_stickiness"): - lines = ["# Cache Session 粘性详情", ""] for sid, s in c["session_stickiness"].items(): lines.append( f'- {sid}: req={s.get("total_requests",0)}, stickiness={s.get("stickiness_pct",0)}%, switches={s.get("switches",0)}' ) - lines.append("") - details["cache_session_stickiness"] = "\n".join(lines) + else: + lines.append("- 无可用样本（需要同一 session 至少 2 次请求）。") + lines.append("") + details["cache_session_stickiness"] = "\n".join(lines) + + lines = ["# Cache 非最优选择详情", ""] if c.get("suboptimal_selections"): - lines = ["# Cache 非最优选择详情", ""] for x in c["suboptimal_selections"][:200]: lines.append( f'- [{x.get("ts","")}] selected={x.get("selected","")} best={x.get("best_hr_worker","")} reason={x.get("reason","")}' ) - lines.append("") - details["cache_suboptimal"] = "\n".join(lines) + else: + lines.append("- 未发现非最优选择。") + lines.append("") + details["cache_suboptimal"] = "\n".join(lines) + + lines = ["# Cache 驱逐影响详情", ""] if c.get("eviction_impact"): - lines = ["# Cache 驱逐影响详情", ""] for x in c["eviction_impact"][:200]: lines.append( f'- session={x.get("session_id","")} interval={x.get("interval_mins",0)}m hitRatio_after={x.get("hitRatio_after",0)} evicted={x.get("evicted",False)}' ) - lines.append("") - details["cache_eviction"] = "\n".join(lines) + else: + lines.append("- 未检测到超时驱逐样本。") + lines.append("") + details["cache_eviction"] = "\n".join(lines) + + lines = ["# Cache Fallback 原因详情", ""] if c.get("fallback_reasons"): - lines = ["# Cache Fallback 原因详情", ""] for x in c["fallback_reasons"]: lines.append(f'- {x.get("value","")}: {x.get("count",0)} ({x.get("pct",0)}%)') - lines.append("") - details["cache_fallback"] = "\n".join(lines) + else: + lines.append("- 未出现 fallback 记录。") + lines.append("") + details["cache_fallback"] = "\n".join(lines) + + lines = ["# Cache 交叉诊断详情", ""] if c.get("cross_diagnosis"): - lines = ["# Cache 交叉诊断详情", ""] for x in c["cross_diagnosis"]: lines.append( f'- diagnosis={x.get("diagnosis","")}, action={x.get("action","")}, avg_stickiness={x.get("avg_stickiness_pct",0)}%' ) - lines.append("") - details["cache_cross"] = "\n".join(lines) + else: + lines.append("- 样本不足，未生成交叉诊断。") + lines.append("") + details["cache_cross"] = "\n".join(lines) if "trace" in results: summary, detail_dict = format_trace_report(results["trace"]) @@ -386,28 +400,31 @@ def save_detailed_report(report_text, output_dir, details=None): if details.get("load_counter_state"): with open(os.path.join(detail_dir, "load_counter_state.md"), "w", encoding="utf-8") as f: f.write(details["load_counter_state"]) - if details.get("cache_session_stickiness"): + if details.get("cache_session_stickiness") is not None: with open(os.path.join(detail_dir, "cache_session_stickiness.md"), "w", encoding="utf-8") as f: f.write(details["cache_session_stickiness"]) - if details.get("cache_suboptimal"): + if details.get("cache_suboptimal") is not None: with open(os.path.join(detail_dir, "cache_suboptimal.md"), "w", encoding="utf-8") as f: f.write(details["cache_suboptimal"]) - if details.get("cache_eviction"): + if details.get("cache_eviction") is not None: with open(os.path.join(detail_dir, "cache_eviction.md"), "w", encoding="utf-8") as f: f.write(details["cache_eviction"]) - if details.get("cache_fallback"): + if details.get("cache_fallback") is not None: with open(os.path.join(detail_dir, "cache_fallback.md"), "w", encoding="utf-8") as f: f.write(details["cache_fallback"]) - if details.get("cache_cross"): + if details.get("cache_cross") is not None: with open(os.path.join(detail_dir, "cache_cross.md"), "w", encoding="utf-8") as f: f.write(details["cache_cross"]) if details.get("errors_topn"): with open(os.path.join(detail_dir, "errors_topn.md"), "w", encoding="utf-8") as f: f.write(details["errors_topn"]) + trace_detail_dir = os.path.join(detail_dir, "trace") + if details.get("trace_files"): + os.makedirs(trace_detail_dir, exist_ok=True) for trace_id, trace_text in details.get("trace_files", {}).items(): safe_id = trace_id.replace("/", "_") - trace_path = os.path.join(detail_dir, f"trace_{safe_id}.md") + trace_path = os.path.join(trace_detail_dir, f"trace_{safe_id}.md") with open(trace_path, "w", encoding="utf-8") as f: f.write(trace_text) @@ -426,7 +443,7 @@ def main(): parser.add_argument("--health", action="store_true", help="仅分析 Worker 健康") parser.add_argument("--cache", action="store_true", help="仅分析 Cache 调度") parser.add_argument("--load", action="store_true", help="仅分析负载与计数器") - parser.add_argument("--trace", metavar="ID", help="追踪指定请求（逗号分隔多 ID）") + parser.add_argument("--trace", metavar="ID", help="追踪指定请求（逗号分隔多 ID；传 all 可全量追踪）") parser.add_argument("--tail", help="尾部行数或分钟数 (如 5000 或 30m)") parser.add_argument( "--start", default=None, help='起始时间（如 "16:00:00"、"03/31 16:00"、"2026/03/31 16:00:00"）' @@ -478,7 +495,7 @@ def main(): run_health = args.health or (not any_mode) run_load = args.load or (not any_mode) run_cache = args.cache or (not any_mode) - run_trace = bool(args.trace) # trace 需要指定 ID，全量扫描不自动调用 + run_trace = bool(args.trace) # trace 需要指定 ID（支持 all），全量扫描不自动调用 results = {} step = 0