troubleshoot: enrich log analysis and reporting (select/release inference, cache cross-diagnosis, detailed outputs)

fastdeploy/golang_router/.claude/skills/troubleshoot/references/report_templates.md

@@ -55,6 +55,7 @@
 - 主报告包含各维度总结 + 可视化图表（sparkline/柱状图/时间线等）
 - 详情拆分到 `details/` 子目录：
   - `detail/health_events.md` — Worker 逐分钟健康事件 + 健康诊断
+  - `detail/errors_topn.md` — ERROR/WARN 模板明细（数量/级别/来源层/影响 + URLs）
   - `detail/load_select_release.md` — 负载诊断 + select/release 明细
   - `detail/load_diagnoses.md` — load 诊断列表
   - `detail/load_counter_state.md` — request/token counter 末状态

fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/cache.py

@@ -399,7 +399,7 @@ def format_cache_report(result):
     if result["diagnoses"]:
         sections.append("### 诊断")
         sections.append("")
-        sections.append("  诊断见详情: [detail/cache_diagnosis.md](detail/cache_diagnosis.md)")
+        sections.append("  诊断见详情: [detail/cache_diagnosis.md](../detail/cache_diagnosis.md)")
         sections.append("")
         detail_sections.append("## 诊断")
         detail_sections.append("")
@@ -452,7 +452,7 @@ def format_cache_report(result):
     if stickiness:
         sections.append("### Session 粘性")
         sections.append("")
-        sections.append("  Session 粘性详情见: [detail/cache_diagnosis.md](detail/cache_diagnosis.md)")
+        sections.append("  Session 粘性详情见: [detail/cache_diagnosis.md](../detail/cache_diagnosis.md)")
         sections.append("")
         table_data = [
             {
@@ -479,7 +479,7 @@ def format_cache_report(result):
         subs = result["suboptimal_selections"]
         sections.append(f"### 非最优选择 ({len(subs)} 次)")
         sections.append("")
-        sections.append("  详情见: [detail/cache_diagnosis.md](detail/cache_diagnosis.md)")
+        sections.append("  详情见: [detail/cache_diagnosis.md](../detail/cache_diagnosis.md)")
         sections.append("")
         reason_counts = defaultdict(int)
         for s in subs:
@@ -501,7 +501,7 @@ def format_cache_report(result):
         evicted = [e for e in evictions if e["evicted"]]
         sections.append(f"### 驱逐影响 ({len(evictions)} 次超时, {len(evicted)} 次缓存失效)")
         sections.append("")
-        sections.append("  详情见: [detail/cache_diagnosis.md](detail/cache_diagnosis.md)")
+        sections.append("  详情见: [detail/cache_diagnosis.md](../detail/cache_diagnosis.md)")
         sections.append("")
         detail_sections.append("## 驱逐影响")
         detail_sections.append("")
@@ -523,7 +523,7 @@ def format_cache_report(result):
     if result.get("cross_diagnosis"):
         sections.append("### 交叉诊断")
         sections.append("")
-        sections.append("  详情见: [detail/cache_diagnosis.md](detail/cache_diagnosis.md)")
+        sections.append("  详情见: [detail/cache_diagnosis.md](../detail/cache_diagnosis.md)")
         sections.append("")
         detail_sections.append("## 交叉诊断")
         detail_sections.append("")
@@ -546,12 +546,12 @@ def format_cache_report(result):
         ]
     ):
         sections.append(
-            "> 详细诊断: [detail/cache_diagnosis.md](detail/cache_diagnosis.md) | "
-            "[detail/cache_session_stickiness.md](detail/cache_session_stickiness.md) | "
-            "[detail/cache_suboptimal.md](detail/cache_suboptimal.md) | "
-            "[detail/cache_eviction.md](detail/cache_eviction.md) | "
-            "[detail/cache_fallback.md](detail/cache_fallback.md) | "
-            "[detail/cache_cross.md](detail/cache_cross.md)"
+            "> 详细诊断: [detail/cache_diagnosis.md](../detail/cache_diagnosis.md) | "
+            "[detail/cache_session_stickiness.md](../detail/cache_session_stickiness.md) | "
+            "[detail/cache_suboptimal.md](../detail/cache_suboptimal.md) | "
+            "[detail/cache_eviction.md](../detail/cache_eviction.md) | "
+            "[detail/cache_fallback.md](../detail/cache_fallback.md) | "
+            "[detail/cache_cross.md](../detail/cache_cross.md)"
         )
         sections.append("")
 

fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/errors.py

@@ -298,27 +298,7 @@ def format_errors_report(result):
         sections.append(render_bar(bar_data, show_count=True))
         sections.append("")
 
-        # 来源层表格
-        table_data = []
-        for e in result["error_top_n"][:10]:
-            table_data.append(
-                {
-                    "模板": e["template"][:60],
-                    "数量": e["count"],
-                    "占比": f'{e["pct"]}%',
-                    "级别": e["level"],
-                    "来源层": e["source_layer"],
-                    "影响": e.get("impact", "-"),
-                    "URLs": ",".join(e.get("urls", [])[:2]) if e.get("urls") else "-",
-                }
-            )
-        sections.append(
-            render_table(
-                table_data,
-                columns=["模板", "数量", "占比", "级别", "来源层", "影响", "URLs"],
-                right_align={"数量", "占比"},
-            )
-        )
+        sections.append("  具体模板表见: [../detail/errors_topn.md](../detail/errors_topn.md)")
         sections.append("")
         yaml_missing_count = sum(
             e["count"] for e in result["error_top_n"] if "Failed to read YAML file config/register.yaml" in e["template"]

fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/health.py

@@ -335,7 +335,7 @@ def format_health_report(result):
     if result["diagnoses"]:
         sections.append("### 诊断")
         sections.append("")
-        sections.append("  诊断见详情: [detail/health_events.md](detail/health_events.md)")
+        sections.append("  诊断见详情: [detail/health_events.md](../detail/health_events.md)")
         sections.append("")
 
     # Worker 可用性表格
@@ -401,7 +401,7 @@ def format_health_report(result):
 
     # 主报告中添加引用
     if has_events:
-        sections.append("> 完整事件详情: [detail/health_events.md](detail/health_events.md)")
+        sections.append("> 完整事件详情: [detail/health_events.md](../detail/health_events.md)")
         sections.append("")
 
     return "\n".join(sections), detail_text

fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/latency.py

@@ -335,7 +335,7 @@ def format_latency_report(result):
     # 诊断（仅在 detail 输出）
     if result["diagnoses"]:
         sections.append("### 诊断")
-        sections.append("  诊断见详情: [detail/latency_diagnoses.md](detail/latency_diagnoses.md)")
+        sections.append("  诊断见详情: [detail/latency_diagnoses.md](../detail/latency_diagnoses.md)")
         sections.append("")
 
     return "\n".join(sections)

fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/load.py

@@ -146,6 +146,7 @@ def analyze_load(log_file, tail=None):
             "per_worker": {},
             "id_coverage": {},
             "type_summary": {},
+            "worker_type_profile": {},
         }
     )
 

fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/load_report.py

@@ -26,8 +26,8 @@ def format_load_report(result):
         sections.append("### 诊断")
         sections.append("")
         sections.append(
-            f'  共 {len(result["diagnoses"])} 条诊断，见详情: [detail/load_diagnoses.md](detail/load_diagnoses.md)；'
-            '匹配明细见 [detail/load_select_release.md](detail/load_select_release.md)'
+            f'  共 {len(result["diagnoses"])} 条诊断，见详情: [detail/load_diagnoses.md](../detail/load_diagnoses.md)；'
+            '匹配明细见 [detail/load_select_release.md](../detail/load_select_release.md)'
         )
         sections.append("")
         detail_sections.append("## 诊断")
@@ -111,7 +111,7 @@ def format_load_report(result):
             sections.append(render_table(type_rows, columns=["type", "counter(S/R)", "token(S/R)"]))
             sections.append("")
             sections.append("  说明: prefill/mixed 的 token-select 同时表示 request counter + token counter 增加；decode 仅 request counter。")
-            sections.append("  说明: token-release 由同 worker 邻近 select 推断到 prefill/mixed，不直接依赖 `release prefill tokens` 文本。")
+            sections.append("  说明: `release prefill tokens` 会被识别为 token-release；worker type 按该 worker URL 在 select 中的类型映射（prefill/decode/mixed）。")
             if type_summary.get("unknown"):
                 sections.append("  说明: unknown 表示日志里缺少 worker type，且无法从邻近 select/release 关系推断。")
             sections.append("")
@@ -184,6 +184,29 @@ def format_load_report(result):
         sections.append("")
         detail_sections.append("## Select/Release Per-Worker")
         detail_sections.append("")
+
+    if sr.get("worker_type_profile"):
+        sections.append("### Worker URL 类型画像（基于 select）")
+        sections.append("")
+        rows = []
+        for w, p in sorted(sr["worker_type_profile"].items()):
+            rows.append(
+                {
+                    "Worker": _strip_scheme(w),
+                    "Dominant": p.get("dominant_type", "unknown"),
+                    "Prefill": p.get("prefill", 0),
+                    "Decode": p.get("decode", 0),
+                    "Mixed": p.get("mixed", 0),
+                }
+            )
+        sections.append(
+            render_table(
+                rows,
+                columns=["Worker", "Dominant", "Prefill", "Decode", "Mixed"],
+                right_align={"Prefill", "Decode", "Mixed"},
+            )
+        )
+        sections.append("")
         detail_sections.append(
             render_table(
                 table_data,
@@ -198,7 +221,7 @@ def format_load_report(result):
         sections.append("  解释: 出现 request select，但在 request release 口径下找不到匹配。可能是请求卡住、日志缺失、或窗口外释放。")
         for u in sr["unmatched_selects"][:3]:
             sections.append(f'    [{u.get("select_ts","")}] {_strip_scheme(u["worker"])} ({u["type"]})')
-        sections.append("  > 完整列表见: [detail/load_select_release.md](detail/load_select_release.md)")
+        sections.append("  > 完整列表见: [detail/load_select_release.md](../detail/load_select_release.md)")
         sections.append("")
         detail_sections.append("## 未匹配 select（完整）")
         detail_sections.append("")
@@ -210,7 +233,7 @@ def format_load_report(result):
 
     if sr.get("unmatched_releases"):
         sections.append(f'  ⚠ {len(sr["unmatched_releases"])} 个未匹配 release（已区分 req/token）')
-        sections.append("  > 完整列表见: [detail/load_select_release.md](detail/load_select_release.md)")
+        sections.append("  > 完整列表见: [detail/load_select_release.md](../detail/load_select_release.md)")
         sections.append("")
         detail_sections.append("## 未匹配 release（按 release_kind 分类）")
         detail_sections.append("")
@@ -224,7 +247,7 @@ def format_load_report(result):
         sections.append(f'  ℹ {len(sr["untracked_selects"])} 个 select 缺少可关联 ID，未参与卡住判定')
         for u in sr["untracked_selects"][:3]:
             sections.append(f'    [{u.get("select_ts","")}] {_strip_scheme(u["worker"])} ({u["type"]})')
-        sections.append("  > 完整列表见: [detail/load_select_release.md](detail/load_select_release.md)")
+        sections.append("  > 完整列表见: [detail/load_select_release.md](../detail/load_select_release.md)")
         sections.append("")
         detail_sections.append("## Untracked selects（缺少可关联 ID）")
         detail_sections.append("")
@@ -260,7 +283,7 @@ def format_load_report(result):
     if result.get("counter_last_state"):
         sections.append("### 计数器末状态")
         sections.append("")
-        sections.append("  末状态详情见: [detail/load_counter_state.md](detail/load_counter_state.md)")
+        sections.append("  末状态详情见: [detail/load_counter_state.md](../detail/load_counter_state.md)")
         sections.append("")
         detail_sections.append("## Counter / Token Counter 末状态（最后一条计数日志）")
         detail_sections.append("")

fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/trace.py

@@ -386,7 +386,7 @@ def format_trace_report(result):
             # 主报告中添加引用和摘要
             safe_tid = tid.replace("/", "_")
             sections.append(f'  事件数: {len(trace["events"])}')
-            sections.append(f"  > 完整事件链: [detail/trace_{safe_tid}.md](detail/trace_{safe_tid}.md)")
+            sections.append(f"  > 完整事件链: [detail/trace_{safe_tid}.md](../detail/trace_{safe_tid}.md)")
             sections.append("")
 
     return "\n".join(sections), detail_dict

fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/log_parser.py

@@ -612,8 +612,8 @@ def match_select_release(lines, fallback_window_s=120):
                 {
                     "ts": ts,
                     "worker": trm.group(2),
-                    # 不直接信任日志里的 token type 文本（"release prefill tokens" 也可能来自 mixed）
-                    "type": "unknown_tokens",
+                    # 文本默认按 prefill 记，再结合同 worker 邻近 select 做纠偏（mixed 场景）
+                    "type": f'{_normalize_worker_type(token_type or "prefill")}_tokens',
                     "raw_token_type": token_type or "",
                     "tags": tags,
                     "tokens": int(trm.group(3)),
@@ -789,15 +789,24 @@ def match_select_release(lines, fallback_window_s=120):
             "token_releases": counts["token_releases"],
         }
 
+    # 基于 select 构建 worker URL -> dominant type 映射
+    per_worker_type_counts = defaultdict(lambda: defaultdict(int))
+    for s in selects:
+        per_worker_type_counts[s["worker"]][_normalize_worker_type(s.get("type"))] += 1
+    worker_dominant_type = {}
+    for w, counts in per_worker_type_counts.items():
+        worker_dominant_type[w] = sorted(counts.items(), key=lambda kv: -kv[1])[0][0] if counts else "unknown"
+
     # 为未显式标注 type 的 release 推断 worker type（避免大量 unknown）
     inferred_release_types = {}
     for i, r in enumerate(releases):
         r_type_raw = str(r.get("type", ""))
         if r_type_raw.endswith("_tokens"):
             base_t = _normalize_worker_type(r_type_raw.replace("_tokens", ""))
-            if base_t == "unknown":
-                # token release 的 worker type 由同 worker 邻近 select 推断（prefill/mixed）
-                base_t = _infer_token_release_worker_type(r, selects, fallback_window_s=fallback_window_s)
+            # token release 按 worker URL 对应的 select 类型映射，不做邻近时间纠偏
+            mapped_t = worker_dominant_type.get(r.get("worker", ""), "unknown")
+            if mapped_t in ("prefill", "decode", "mixed"):
+                base_t = mapped_t
             inferred_release_types[i] = f"{base_t}_tokens"
             continue
         base_t = _normalize_worker_type(r_type_raw)
@@ -828,6 +837,20 @@ def match_select_release(lines, fallback_window_s=120):
         else:
             type_summary[r_type]["counter_releases"] += 1
 
+    # 每个 worker URL 的类型画像（基于 select）
+    worker_type_profile = {}
+    for w, counts in per_worker_type_counts.items():
+        dominant = "unknown"
+        if counts:
+            dominant = sorted(counts.items(), key=lambda kv: -kv[1])[0][0]
+        worker_type_profile[w] = {
+            "dominant_type": dominant,
+            "prefill": counts.get("prefill", 0),
+            "decode": counts.get("decode", 0),
+            "mixed": counts.get("mixed", 0),
+            "unknown": counts.get("unknown", 0),
+        }
+
     unmatched_releases = []
     for i, r in enumerate(releases):
         if str(r.get("type", "")).endswith("_tokens"):
@@ -868,6 +891,7 @@ def match_select_release(lines, fallback_window_s=120):
             "without_any_id": without_any_id,
         },
         "type_summary": dict(type_summary),
+        "worker_type_profile": worker_type_profile,
     }
 
 

fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/troubleshoot.py

@@ -128,20 +128,14 @@ def determine_status(results):
                 reasons.append(d["message"])
 
     if reasons:
-        # 去重并限制长度，避免状态行过长难读
+        # 去重但保留完整信息
         deduped = []
         seen = set()
         for r in reasons:
             if r not in seen:
                 deduped.append(r)
                 seen.add(r)
-        max_reasons = 4
-        shown = deduped[:max_reasons]
-        extra = len(deduped) - len(shown)
-        summary = "；".join(shown)
-        if extra > 0:
-            summary += f"；另有 {extra} 项诊断见各维度 detail 报告"
-        return "DEGRADED", summary
+        return "DEGRADED", "；".join(deduped)
 
     if not results:
         return "HEALTHY", "无分析数据"
@@ -173,6 +167,7 @@ def format_full_report(results, status, status_reason):
         "cache_eviction": None,
         "cache_fallback": None,
         "cache_cross": None,
+        "errors_topn": None,
         "trace_files": {},
     }
 
@@ -187,6 +182,29 @@ def format_full_report(results, status, status_reason):
     # 各维度报告
     if "errors" in results:
         parts.append(format_errors_report(results["errors"]))
+        if results["errors"].get("error_top_n"):
+            lines = [
+                "# Errors TopN 详情",
+                "",
+                "| 模板 | 数量 | 级别 | 来源层 | 影响 |",
+                "|:--|--:|:--|:--|:--|",
+            ]
+            for e in results["errors"]["error_top_n"]:
+                lines.append(
+                    f'| {e.get("template","")} | {e.get("count",0)} | {e.get("level","")} | {e.get("source_layer","")} | {e.get("impact","-")} |'
+                )
+            lines.append("")
+            lines.append("## 涉及 URLs")
+            lines.append("")
+            for e in results["errors"]["error_top_n"]:
+                urls = e.get("urls") or []
+                if not urls:
+                    continue
+                lines.append(f'- 模板: {e.get("template","")}')
+                for u in urls:
+                    lines.append(f'  - {u}')
+            lines.append("")
+            details["errors_topn"] = "\n".join(lines)
 
     if "latency" in results:
         parts.append(format_latency_report(results["latency"]))
@@ -331,6 +349,9 @@ def save_detailed_report(report_text, output_dir, details=None):
         if details.get("cache_cross"):
             with open(os.path.join(detail_dir, "cache_cross.md"), "w", encoding="utf-8") as f:
                 f.write(details["cache_cross"])
+        if details.get("errors_topn"):
+            with open(os.path.join(detail_dir, "errors_topn.md"), "w", encoding="utf-8") as f:
+                f.write(details["errors_topn"])
 
         for trace_id, trace_text in details.get("trace_files", {}).items():
             safe_id = trace_id.replace("/", "_")