diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/references/report_templates.md b/fastdeploy/golang_router/.claude/skills/troubleshoot/references/report_templates.md index c02f55c2d65..cd705d02816 100644 --- a/fastdeploy/golang_router/.claude/skills/troubleshoot/references/report_templates.md +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/references/report_templates.md @@ -55,6 +55,7 @@ - 主报告包含各维度总结 + 可视化图表（sparkline/柱状图/时间线等） - 详情拆分到 `details/` 子目录： - `detail/health_events.md` — Worker 逐分钟健康事件 + 健康诊断 + - `detail/errors_topn.md` — ERROR/WARN 模板明细（数量/级别/来源层/影响 + URLs） - `detail/load_select_release.md` — 负载诊断 + select/release 明细 - `detail/load_diagnoses.md` — load 诊断列表 - `detail/load_counter_state.md` — request/token counter 末状态 diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/cache.py b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/cache.py index 0d146c9b43c..3a5c19ad00b 100644 --- a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/cache.py +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/cache.py @@ -399,7 +399,7 @@ def format_cache_report(result): if result["diagnoses"]: sections.append("### 诊断") sections.append("") - sections.append(" 诊断见详情: [detail/cache_diagnosis.md](detail/cache_diagnosis.md)") + sections.append(" 诊断见详情: [detail/cache_diagnosis.md](../detail/cache_diagnosis.md)") sections.append("") detail_sections.append("## 诊断") detail_sections.append("") @@ -452,7 +452,7 @@ def format_cache_report(result): if stickiness: sections.append("### Session 粘性") sections.append("") - sections.append(" Session 粘性详情见: [detail/cache_diagnosis.md](detail/cache_diagnosis.md)") + sections.append(" Session 粘性详情见: [detail/cache_diagnosis.md](../detail/cache_diagnosis.md)") sections.append("") table_data = [ { @@ -479,7 +479,7 @@ def format_cache_report(result): subs = result["suboptimal_selections"] sections.append(f"### 非最优选择 ({len(subs)} 次)") sections.append("") - sections.append(" 详情见: [detail/cache_diagnosis.md](detail/cache_diagnosis.md)") + sections.append(" 详情见: [detail/cache_diagnosis.md](../detail/cache_diagnosis.md)") sections.append("") reason_counts = defaultdict(int) for s in subs: @@ -501,7 +501,7 @@ def format_cache_report(result): evicted = [e for e in evictions if e["evicted"]] sections.append(f"### 驱逐影响 ({len(evictions)} 次超时, {len(evicted)} 次缓存失效)") sections.append("") - sections.append(" 详情见: [detail/cache_diagnosis.md](detail/cache_diagnosis.md)") + sections.append(" 详情见: [detail/cache_diagnosis.md](../detail/cache_diagnosis.md)") sections.append("") detail_sections.append("## 驱逐影响") detail_sections.append("") @@ -523,7 +523,7 @@ def format_cache_report(result): if result.get("cross_diagnosis"): sections.append("### 交叉诊断") sections.append("") - sections.append(" 详情见: [detail/cache_diagnosis.md](detail/cache_diagnosis.md)") + sections.append(" 详情见: [detail/cache_diagnosis.md](../detail/cache_diagnosis.md)") sections.append("") detail_sections.append("## 交叉诊断") detail_sections.append("") @@ -546,12 +546,12 @@ def format_cache_report(result): ] ): sections.append( - "> 详细诊断: [detail/cache_diagnosis.md](detail/cache_diagnosis.md) | " - "[detail/cache_session_stickiness.md](detail/cache_session_stickiness.md) | " - "[detail/cache_suboptimal.md](detail/cache_suboptimal.md) | " - "[detail/cache_eviction.md](detail/cache_eviction.md) | " - "[detail/cache_fallback.md](detail/cache_fallback.md) | " - "[detail/cache_cross.md](detail/cache_cross.md)" + "> 详细诊断: [detail/cache_diagnosis.md](../detail/cache_diagnosis.md) | " + "[detail/cache_session_stickiness.md](../detail/cache_session_stickiness.md) | " + "[detail/cache_suboptimal.md](../detail/cache_suboptimal.md) | " + "[detail/cache_eviction.md](../detail/cache_eviction.md) | " + "[detail/cache_fallback.md](../detail/cache_fallback.md) | " + "[detail/cache_cross.md](../detail/cache_cross.md)" ) sections.append("") diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/errors.py b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/errors.py index 7f519806225..f0e4c352b6c 100644 --- a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/errors.py +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/errors.py @@ -298,27 +298,7 @@ def format_errors_report(result): sections.append(render_bar(bar_data, show_count=True)) sections.append("") - # 来源层表格 - table_data = [] - for e in result["error_top_n"][:10]: - table_data.append( - { - "模板": e["template"][:60], - "数量": e["count"], - "占比": f'{e["pct"]}%', - "级别": e["level"], - "来源层": e["source_layer"], - "影响": e.get("impact", "-"), - "URLs": ",".join(e.get("urls", [])[:2]) if e.get("urls") else "-", - } - ) - sections.append( - render_table( - table_data, - columns=["模板", "数量", "占比", "级别", "来源层", "影响", "URLs"], - right_align={"数量", "占比"}, - ) - ) + sections.append(" 具体模板表见: [../detail/errors_topn.md](../detail/errors_topn.md)") sections.append("") yaml_missing_count = sum( e["count"] for e in result["error_top_n"] if "Failed to read YAML file config/register.yaml" in e["template"] diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/health.py b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/health.py index 8fc4e88cc72..5d1994d9405 100644 --- a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/health.py +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/health.py @@ -335,7 +335,7 @@ def format_health_report(result): if result["diagnoses"]: sections.append("### 诊断") sections.append("") - sections.append(" 诊断见详情: [detail/health_events.md](detail/health_events.md)") + sections.append(" 诊断见详情: [detail/health_events.md](../detail/health_events.md)") sections.append("") # Worker 可用性表格 @@ -401,7 +401,7 @@ def format_health_report(result): # 主报告中添加引用 if has_events: - sections.append("> 完整事件详情: [detail/health_events.md](detail/health_events.md)") + sections.append("> 完整事件详情: [detail/health_events.md](../detail/health_events.md)") sections.append("") return "\n".join(sections), detail_text diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/latency.py b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/latency.py index 57af094661e..508cf3824d9 100644 --- a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/latency.py +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/latency.py @@ -335,7 +335,7 @@ def format_latency_report(result): # 诊断（仅在 detail 输出） if result["diagnoses"]: sections.append("### 诊断") - sections.append(" 诊断见详情: [detail/latency_diagnoses.md](detail/latency_diagnoses.md)") + sections.append(" 诊断见详情: [detail/latency_diagnoses.md](../detail/latency_diagnoses.md)") sections.append("") return "\n".join(sections) diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/load.py b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/load.py index 0d92ae56a32..2e03ba1ce63 100644 --- a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/load.py +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/load.py @@ -146,6 +146,7 @@ def analyze_load(log_file, tail=None): "per_worker": {}, "id_coverage": {}, "type_summary": {}, + "worker_type_profile": {}, } ) diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/load_report.py b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/load_report.py index 8c375e25e57..9d4e9b51496 100644 --- a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/load_report.py +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/load_report.py @@ -26,8 +26,8 @@ def format_load_report(result): sections.append("### 诊断") sections.append("") sections.append( - f' 共 {len(result["diagnoses"])} 条诊断，见详情: [detail/load_diagnoses.md](detail/load_diagnoses.md)；' - '匹配明细见 [detail/load_select_release.md](detail/load_select_release.md)' + f' 共 {len(result["diagnoses"])} 条诊断，见详情: [detail/load_diagnoses.md](../detail/load_diagnoses.md)；' + '匹配明细见 [detail/load_select_release.md](../detail/load_select_release.md)' ) sections.append("") detail_sections.append("## 诊断") @@ -111,7 +111,7 @@ def format_load_report(result): sections.append(render_table(type_rows, columns=["type", "counter(S/R)", "token(S/R)"])) sections.append("") sections.append(" 说明: prefill/mixed 的 token-select 同时表示 request counter + token counter 增加；decode 仅 request counter。") - sections.append(" 说明: token-release 由同 worker 邻近 select 推断到 prefill/mixed，不直接依赖 `release prefill tokens` 文本。") + sections.append(" 说明: `release prefill tokens` 会被识别为 token-release；worker type 按该 worker URL 在 select 中的类型映射（prefill/decode/mixed）。") if type_summary.get("unknown"): sections.append(" 说明: unknown 表示日志里缺少 worker type，且无法从邻近 select/release 关系推断。") sections.append("") @@ -184,6 +184,29 @@ def format_load_report(result): sections.append("") detail_sections.append("## Select/Release Per-Worker") detail_sections.append("") + + if sr.get("worker_type_profile"): + sections.append("### Worker URL 类型画像（基于 select）") + sections.append("") + rows = [] + for w, p in sorted(sr["worker_type_profile"].items()): + rows.append( + { + "Worker": _strip_scheme(w), + "Dominant": p.get("dominant_type", "unknown"), + "Prefill": p.get("prefill", 0), + "Decode": p.get("decode", 0), + "Mixed": p.get("mixed", 0), + } + ) + sections.append( + render_table( + rows, + columns=["Worker", "Dominant", "Prefill", "Decode", "Mixed"], + right_align={"Prefill", "Decode", "Mixed"}, + ) + ) + sections.append("") detail_sections.append( render_table( table_data, @@ -198,7 +221,7 @@ def format_load_report(result): sections.append(" 解释: 出现 request select，但在 request release 口径下找不到匹配。可能是请求卡住、日志缺失、或窗口外释放。") for u in sr["unmatched_selects"][:3]: sections.append(f' [{u.get("select_ts","")}] {_strip_scheme(u["worker"])} ({u["type"]})') - sections.append(" > 完整列表见: [detail/load_select_release.md](detail/load_select_release.md)") + sections.append(" > 完整列表见: [detail/load_select_release.md](../detail/load_select_release.md)") sections.append("") detail_sections.append("## 未匹配 select（完整）") detail_sections.append("") @@ -210,7 +233,7 @@ def format_load_report(result): if sr.get("unmatched_releases"): sections.append(f' ⚠ {len(sr["unmatched_releases"])} 个未匹配 release（已区分 req/token）') - sections.append(" > 完整列表见: [detail/load_select_release.md](detail/load_select_release.md)") + sections.append(" > 完整列表见: [detail/load_select_release.md](../detail/load_select_release.md)") sections.append("") detail_sections.append("## 未匹配 release（按 release_kind 分类）") detail_sections.append("") @@ -224,7 +247,7 @@ def format_load_report(result): sections.append(f' ℹ {len(sr["untracked_selects"])} 个 select 缺少可关联 ID，未参与卡住判定') for u in sr["untracked_selects"][:3]: sections.append(f' [{u.get("select_ts","")}] {_strip_scheme(u["worker"])} ({u["type"]})') - sections.append(" > 完整列表见: [detail/load_select_release.md](detail/load_select_release.md)") + sections.append(" > 完整列表见: [detail/load_select_release.md](../detail/load_select_release.md)") sections.append("") detail_sections.append("## Untracked selects（缺少可关联 ID）") detail_sections.append("") @@ -260,7 +283,7 @@ def format_load_report(result): if result.get("counter_last_state"): sections.append("### 计数器末状态") sections.append("") - sections.append(" 末状态详情见: [detail/load_counter_state.md](detail/load_counter_state.md)") + sections.append(" 末状态详情见: [detail/load_counter_state.md](../detail/load_counter_state.md)") sections.append("") detail_sections.append("## Counter / Token Counter 末状态（最后一条计数日志）") detail_sections.append("") diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/trace.py b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/trace.py index a792da8002f..24af9a23500 100644 --- a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/trace.py +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/trace.py @@ -386,7 +386,7 @@ def format_trace_report(result): # 主报告中添加引用和摘要 safe_tid = tid.replace("/", "_") sections.append(f' 事件数: {len(trace["events"])}') - sections.append(f" > 完整事件链: [detail/trace_{safe_tid}.md](detail/trace_{safe_tid}.md)") + sections.append(f" > 完整事件链: [detail/trace_{safe_tid}.md](../detail/trace_{safe_tid}.md)") sections.append("") return "\n".join(sections), detail_dict diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/log_parser.py b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/log_parser.py index b99e75c37c5..548c29ebc29 100644 --- a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/log_parser.py +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/log_parser.py @@ -612,8 +612,8 @@ def match_select_release(lines, fallback_window_s=120): { "ts": ts, "worker": trm.group(2), - # 不直接信任日志里的 token type 文本（"release prefill tokens" 也可能来自 mixed） - "type": "unknown_tokens", + # 文本默认按 prefill 记，再结合同 worker 邻近 select 做纠偏（mixed 场景） + "type": f'{_normalize_worker_type(token_type or "prefill")}_tokens', "raw_token_type": token_type or "", "tags": tags, "tokens": int(trm.group(3)), @@ -789,15 +789,24 @@ def match_select_release(lines, fallback_window_s=120): "token_releases": counts["token_releases"], } + # 基于 select 构建 worker URL -> dominant type 映射 + per_worker_type_counts = defaultdict(lambda: defaultdict(int)) + for s in selects: + per_worker_type_counts[s["worker"]][_normalize_worker_type(s.get("type"))] += 1 + worker_dominant_type = {} + for w, counts in per_worker_type_counts.items(): + worker_dominant_type[w] = sorted(counts.items(), key=lambda kv: -kv[1])[0][0] if counts else "unknown" + # 为未显式标注 type 的 release 推断 worker type（避免大量 unknown） inferred_release_types = {} for i, r in enumerate(releases): r_type_raw = str(r.get("type", "")) if r_type_raw.endswith("_tokens"): base_t = _normalize_worker_type(r_type_raw.replace("_tokens", "")) - if base_t == "unknown": - # token release 的 worker type 由同 worker 邻近 select 推断（prefill/mixed） - base_t = _infer_token_release_worker_type(r, selects, fallback_window_s=fallback_window_s) + # token release 按 worker URL 对应的 select 类型映射，不做邻近时间纠偏 + mapped_t = worker_dominant_type.get(r.get("worker", ""), "unknown") + if mapped_t in ("prefill", "decode", "mixed"): + base_t = mapped_t inferred_release_types[i] = f"{base_t}_tokens" continue base_t = _normalize_worker_type(r_type_raw) @@ -828,6 +837,20 @@ def match_select_release(lines, fallback_window_s=120): else: type_summary[r_type]["counter_releases"] += 1 + # 每个 worker URL 的类型画像（基于 select） + worker_type_profile = {} + for w, counts in per_worker_type_counts.items(): + dominant = "unknown" + if counts: + dominant = sorted(counts.items(), key=lambda kv: -kv[1])[0][0] + worker_type_profile[w] = { + "dominant_type": dominant, + "prefill": counts.get("prefill", 0), + "decode": counts.get("decode", 0), + "mixed": counts.get("mixed", 0), + "unknown": counts.get("unknown", 0), + } + unmatched_releases = [] for i, r in enumerate(releases): if str(r.get("type", "")).endswith("_tokens"): @@ -868,6 +891,7 @@ def match_select_release(lines, fallback_window_s=120): "without_any_id": without_any_id, }, "type_summary": dict(type_summary), + "worker_type_profile": worker_type_profile, } diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/troubleshoot.py b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/troubleshoot.py index 7dd2e153a64..641c5106bee 100644 --- a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/troubleshoot.py +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/troubleshoot.py @@ -128,20 +128,14 @@ def determine_status(results): reasons.append(d["message"]) if reasons: - # 去重并限制长度，避免状态行过长难读 + # 去重但保留完整信息 deduped = [] seen = set() for r in reasons: if r not in seen: deduped.append(r) seen.add(r) - max_reasons = 4 - shown = deduped[:max_reasons] - extra = len(deduped) - len(shown) - summary = "；".join(shown) - if extra > 0: - summary += f"；另有 {extra} 项诊断见各维度 detail 报告" - return "DEGRADED", summary + return "DEGRADED", "；".join(deduped) if not results: return "HEALTHY", "无分析数据" @@ -173,6 +167,7 @@ def format_full_report(results, status, status_reason): "cache_eviction": None, "cache_fallback": None, "cache_cross": None, + "errors_topn": None, "trace_files": {}, } @@ -187,6 +182,29 @@ def format_full_report(results, status, status_reason): # 各维度报告 if "errors" in results: parts.append(format_errors_report(results["errors"])) + if results["errors"].get("error_top_n"): + lines = [ + "# Errors TopN 详情", + "", + "| 模板 | 数量 | 级别 | 来源层 | 影响 |", + "|:--|--:|:--|:--|:--|", + ] + for e in results["errors"]["error_top_n"]: + lines.append( + f'| {e.get("template","")} | {e.get("count",0)} | {e.get("level","")} | {e.get("source_layer","")} | {e.get("impact","-")} |' + ) + lines.append("") + lines.append("## 涉及 URLs") + lines.append("") + for e in results["errors"]["error_top_n"]: + urls = e.get("urls") or [] + if not urls: + continue + lines.append(f'- 模板: {e.get("template","")}') + for u in urls: + lines.append(f' - {u}') + lines.append("") + details["errors_topn"] = "\n".join(lines) if "latency" in results: parts.append(format_latency_report(results["latency"])) @@ -331,6 +349,9 @@ def save_detailed_report(report_text, output_dir, details=None): if details.get("cache_cross"): with open(os.path.join(detail_dir, "cache_cross.md"), "w", encoding="utf-8") as f: f.write(details["cache_cross"]) + if details.get("errors_topn"): + with open(os.path.join(detail_dir, "errors_topn.md"), "w", encoding="utf-8") as f: + f.write(details["errors_topn"]) for trace_id, trace_text in details.get("trace_files", {}).items(): safe_id = trace_id.replace("/", "_")