From 5ac7cb3afd399b6372884dad17f5714b07ea96e4 Mon Sep 17 00:00:00 2001 From: mouxin <494624263qq@gmail.com> Date: Mon, 13 Apr 2026 11:21:35 +0800 Subject: [PATCH] troubleshoot: map token-release type by worker URL instead of time-neighbor inference --- .../troubleshoot/references/error_catalog.md | 1 + .../troubleshoot/references/log_patterns.md | 11 ++ .../references/report_templates.md | 12 +- .../troubleshoot/scripts/analyzers/cache.py | 131 +++++++++++++- .../troubleshoot/scripts/analyzers/errors.py | 56 ++++-- .../troubleshoot/scripts/analyzers/health.py | 38 +++- .../troubleshoot/scripts/analyzers/latency.py | 8 +- .../troubleshoot/scripts/analyzers/load.py | 65 ++++++- .../scripts/analyzers/load_report.py | 65 ++++++- .../troubleshoot/scripts/analyzers/trace.py | 25 ++- .../skills/troubleshoot/scripts/chart.py | 3 +- .../skills/troubleshoot/scripts/log_parser.py | 163 +++++++++++++++++- .../troubleshoot/scripts/troubleshoot.py | 142 ++++++++++++++- 13 files changed, 669 insertions(+), 51 deletions(-) diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/references/error_catalog.md b/fastdeploy/golang_router/.claude/skills/troubleshoot/references/error_catalog.md index ba48297d9c9..60b4931b546 100644 --- a/fastdeploy/golang_router/.claude/skills/troubleshoot/references/error_catalog.md +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/references/error_catalog.md @@ -61,6 +61,7 @@ | `Failed to select worker pair: {err}` | HIGH | FD 后端 | 请求返回 502 | | `Failed to build disaggregate_info: {err}` | HIGH | Router | 请求返回 500 | | `Failed to encode modified request: {err}` | HIGH | Router | 请求返回 500 | +| `Failed to read YAML file config/register.yaml: {err}` | LOW | Router | 启动时未找到可选配置文件（若未使用 register.yaml 可忽略） | | `Failed to select worker: {err}` | HIGH | FD 后端 | 请求返回 502 | | `Failed to connect to backend service: {err}` | HIGH | FD 后端 | 请求返回 502 | | `Request failed (attempt {n}/{max}): {err}` | MEDIUM | FD 后端 | 重试中 | diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/references/log_patterns.md b/fastdeploy/golang_router/.claude/skills/troubleshoot/references/log_patterns.md index cf33b41f723..4322909c01d 100644 --- a/fastdeploy/golang_router/.claude/skills/troubleshoot/references/log_patterns.md +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/references/log_patterns.md @@ -233,6 +233,17 @@ PD（Prefill/Decode 分离）模式下，`completions.go` 产生的 `[prefill]` --- +## Select/Release 日志细节（与代码一致） + +- `select worker (prefill): , tokens: ` +- `select worker (decode|mixed): , count: ` +- `release worker: , count: `（request counter 释放） +- `release prefill tokens: , tokens: `（token counter 释放；可能来自 prefill 或 mixed 请求路径） + +重点：release 只有上面这两种。`release worker` 不带 worker type，`release prefill tokens` 的文本也不能直接断定是 prefill（mixed 也可能调用）。因此按 `prefill/decode/mixed` 统计时，需要从 select 侧做归类；确实无法归类时才记为 `unknown`。 + +--- + ## 使用脚本工具 各 skill 的脚本位于各自的 `scripts/` 目录下，自动处理上述所有日志解析和计算。 diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/references/report_templates.md b/fastdeploy/golang_router/.claude/skills/troubleshoot/references/report_templates.md index ba9e40e9869..cd705d02816 100644 --- a/fastdeploy/golang_router/.claude/skills/troubleshoot/references/report_templates.md +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/references/report_templates.md @@ -44,6 +44,7 @@ ### 简洁版（终端输出） - 第一行：`STATUS: HEALTHY / DEGRADED / CRITICAL — 简要说明` +- 状态定义：`HEALTHY`=无明显异常；`DEGRADED`=服务可用但性能/稳定性下降（需关注）；`CRITICAL`=服务不可用或高风险故障 - 按三层分类（Router / FD 后端 / 客户端） - 每个问题一行摘要 + 关键指标 - 末尾提示详细版文件路径 @@ -53,8 +54,15 @@ - 路径：`skill_output/troubleshoot//troubleshoot_report_.md` - 主报告包含各维度总结 + 可视化图表（sparkline/柱状图/时间线等） - 详情拆分到 `details/` 子目录： - - `details/health_events.md` — Worker 逐分钟健康事件 - - `details/trace_.md` — 请求追踪事件链 + - `detail/health_events.md` — Worker 逐分钟健康事件 + 健康诊断 + - `detail/errors_topn.md` — ERROR/WARN 模板明细（数量/级别/来源层/影响 + URLs） + - `detail/load_select_release.md` — 负载诊断 + select/release 明细 + - `detail/load_diagnoses.md` — load 诊断列表 + - `detail/load_counter_state.md` — request/token counter 末状态 + - `detail/latency_diagnoses.md` — 延迟诊断详情 + - `detail/cache_diagnosis.md` — cache 六维诊断详情（session 粘性/非最优/驱逐/Fallback/冷启动/交叉诊断） + - `detail/cache_session_stickiness.md` / `detail/cache_suboptimal.md` / `detail/cache_eviction.md` / `detail/cache_fallback.md` / `detail/cache_cross.md` — cache 分职责拆分明细 + - `detail/trace_.md` — 请求追踪事件链 --- diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/cache.py b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/cache.py index 3fca296f4d6..3a5c19ad00b 100644 --- a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/cache.py +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/cache.py @@ -136,6 +136,12 @@ def analyze_cache(log_file, tail=None, eviction_duration_mins=30, hit_ratio_weig "cold_starts": cold_starts, "hitratio_stats": hitratio_stats, "tokenizer_degraded_count": tokenizer_degraded_count, + "cross_diagnosis": _analyze_cross_diagnosis( + session_stickiness=session_stickiness, + hitratio_stats=hitratio_stats, + strategy_dist=strategy_dist, + eviction_impact=eviction_impact, + ), "diagnoses": diagnoses, "summary": f"{total} 策略决策, cache_aware {cache_aware_count}, fallback {fallback_count}, " f"冷启动 {cold_starts}", @@ -339,6 +345,45 @@ def _diagnose( return diagnoses +def _analyze_cross_diagnosis(session_stickiness, hitratio_stats, strategy_dist, eviction_impact): + """交叉诊断：基于粘性/命中率/fallback/驱逐给出简表。""" + if not session_stickiness: + return [] + avg_stickiness = sum(v["stickiness_pct"] for v in session_stickiness.values()) / max(len(session_stickiness), 1) + mean_hr = hitratio_stats.get("mean", 0) + fallback_pct = 0 + for s in strategy_dist: + if s.get("value") == "process_tokens": + fallback_pct = s.get("pct", 0) + break + evicted_cnt = sum(1 for e in eviction_impact if e.get("evicted")) + + diagnosis = "运行良好" + action = "-" + if avg_stickiness >= 70 and mean_hr >= 40 and fallback_pct < 10: + diagnosis = "运行良好" + elif avg_stickiness >= 70 and mean_hr < 20 and evicted_cnt > 0: + diagnosis = "疑似驱逐导致命中率低" + action = "考虑增大 eviction-duration-mins" + elif avg_stickiness < 40 and fallback_pct >= 20: + diagnosis = "低粘性 + 高 fallback" + action = "检查负载阈值与 cache-aware 参数" + elif avg_stickiness < 40 and mean_hr < 20: + diagnosis = "低粘性 + 低命中" + action = "检查缓存预热与 prompt 稳定性" + + return [ + { + "avg_stickiness_pct": round(avg_stickiness, 1), + "mean_hitRatio_pct": round(mean_hr, 1), + "fallback_pct": round(fallback_pct, 1), + "evicted_after_timeout": evicted_cnt, + "diagnosis": diagnosis, + "action": action, + } + ] + + # ════════════════════════════════════════════════════════════════ # 报告格式化 # ════════════════════════════════════════════════════════════════ @@ -349,13 +394,18 @@ def format_cache_report(result): sections = ["## Cache 调度诊断", ""] sections.append(f' {result["summary"]}') sections.append("") + detail_sections = ["# Cache 调度详情", "", f'总结: {result["summary"]}', ""] if result["diagnoses"]: sections.append("### 诊断") sections.append("") - for d in result["diagnoses"]: - sections.append(f' [{d["severity"]}] [{d["source_layer"]}] {d["message"]}') + sections.append(" 诊断见详情: [detail/cache_diagnosis.md](../detail/cache_diagnosis.md)") sections.append("") + detail_sections.append("## 诊断") + detail_sections.append("") + for d in result["diagnoses"]: + detail_sections.append(f'[{d["severity"]}] [{d["source_layer"]}] {d["message"]}') + detail_sections.append("") # 策略分布 if result["strategy_dist"]: @@ -364,6 +414,10 @@ def format_cache_report(result): bar_data = [{"label": s["value"], "value": s["pct"], "count": s["count"]} for s in result["strategy_dist"]] sections.append(render_bar(bar_data, show_count=True)) sections.append("") + detail_sections.append("## 策略分布") + detail_sections.append("") + detail_sections.append(render_bar(bar_data, show_count=True)) + detail_sections.append("") # hitRatio 统计 hs = result.get("hitratio_stats", {}) @@ -383,6 +437,10 @@ def format_cache_report(result): bar_data = [{"label": f["value"], "value": f["pct"], "count": f["count"]} for f in result["fallback_reasons"]] sections.append(render_bar(bar_data, show_count=True)) sections.append("") + detail_sections.append("## Fallback 原因分布") + detail_sections.append("") + detail_sections.append(render_bar(bar_data, show_count=True)) + detail_sections.append("") # Tokenizer 退化 if result.get("tokenizer_degraded_count", 0) > 0: @@ -394,6 +452,8 @@ def format_cache_report(result): if stickiness: sections.append("### Session 粘性") sections.append("") + sections.append(" Session 粘性详情见: [detail/cache_diagnosis.md](../detail/cache_diagnosis.md)") + sections.append("") table_data = [ { "Session": sid[:16], @@ -403,26 +463,37 @@ def format_cache_report(result): } for sid, s in sorted(stickiness.items(), key=lambda x: x[1]["stickiness_pct"]) ] - sections.append( + detail_sections.append("## Session 粘性") + detail_sections.append("") + detail_sections.append( render_table( - table_data[:10], + table_data, columns=["Session", "请求数", "粘性率", "切换次数"], right_align={"请求数", "粘性率", "切换次数"}, ) ) - sections.append("") + detail_sections.append("") # 非最优选择 if result.get("suboptimal_selections"): subs = result["suboptimal_selections"] sections.append(f"### 非最优选择 ({len(subs)} 次)") sections.append("") + sections.append(" 详情见: [detail/cache_diagnosis.md](../detail/cache_diagnosis.md)") + sections.append("") reason_counts = defaultdict(int) for s in subs: reason_counts[s["reason"]] += 1 for reason, count in sorted(reason_counts.items(), key=lambda x: -x[1]): sections.append(f" {reason}: {count} 次") sections.append("") + detail_sections.append("## 非最优选择（Top 20）") + detail_sections.append("") + for s in subs[:20]: + detail_sections.append( + f'- [{s.get("ts","")}] selected={s.get("selected","")}({s.get("selected_hr",0)}), best={s.get("best_hr_worker","")}({s.get("best_hr",0)}), reason={s.get("reason","")}' + ) + detail_sections.append("") # 驱逐影响 if result.get("eviction_impact"): @@ -430,13 +501,61 @@ def format_cache_report(result): evicted = [e for e in evictions if e["evicted"]] sections.append(f"### 驱逐影响 ({len(evictions)} 次超时, {len(evicted)} 次缓存失效)") sections.append("") + sections.append(" 详情见: [detail/cache_diagnosis.md](../detail/cache_diagnosis.md)") + sections.append("") + detail_sections.append("## 驱逐影响") + detail_sections.append("") + for e in evictions[:50]: + detail_sections.append( + f'- session={e.get("session_id","")[:24]} interval={e.get("interval_mins",0)}m hitRatio_after={e.get("hitRatio_after",0)} evicted={e.get("evicted",False)}' + ) + detail_sections.append("") # 冷启动 if result.get("cold_starts", 0) > 0: sections.append(f' 冷启动: {result["cold_starts"]} 次（hitRatios=map[]）') sections.append("") + detail_sections.append("## 冷启动识别") + detail_sections.append("") + detail_sections.append(f'- 冷启动次数: {result["cold_starts"]}') + detail_sections.append("") + + if result.get("cross_diagnosis"): + sections.append("### 交叉诊断") + sections.append("") + sections.append(" 详情见: [detail/cache_diagnosis.md](../detail/cache_diagnosis.md)") + sections.append("") + detail_sections.append("## 交叉诊断") + detail_sections.append("") + detail_sections.append( + render_table( + result["cross_diagnosis"], + columns=["avg_stickiness_pct", "mean_hitRatio_pct", "fallback_pct", "evicted_after_timeout", "diagnosis", "action"], + right_align={"avg_stickiness_pct", "mean_hitRatio_pct", "fallback_pct", "evicted_after_timeout"}, + ) + ) + detail_sections.append("") + + if any( + [ + result.get("session_stickiness"), + result.get("suboptimal_selections"), + result.get("eviction_impact"), + result.get("cross_diagnosis"), + result.get("diagnoses"), + ] + ): + sections.append( + "> 详细诊断: [detail/cache_diagnosis.md](../detail/cache_diagnosis.md) | " + "[detail/cache_session_stickiness.md](../detail/cache_session_stickiness.md) | " + "[detail/cache_suboptimal.md](../detail/cache_suboptimal.md) | " + "[detail/cache_eviction.md](../detail/cache_eviction.md) | " + "[detail/cache_fallback.md](../detail/cache_fallback.md) | " + "[detail/cache_cross.md](../detail/cache_cross.md)" + ) + sections.append("") - return "\n".join(sections) + return "\n".join(sections), "\n".join(detail_sections) # ════════════════════════════════════════════════════════════════ diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/errors.py b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/errors.py index b8217a5ffa4..f0e4c352b6c 100644 --- a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/errors.py +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/errors.py @@ -33,6 +33,7 @@ ("counter already zero", "Router"), ("tokenizer failed", "Router"), ("Instance {url} role is unknown", "Router"), + ("Failed to read YAML file config/register.yaml", "Router"), # 客户端 ("Invalid request body", "客户端"), ("Invalid JSON format", "客户端"), @@ -55,6 +56,15 @@ ("GetRemoteMetrics failed", "FD 后端"), ] +IMPACT_RULES = [ + ("Failed to select", "请求可能返回 502/503"), + ("Failed to connect to backend", "后端不可达，请求失败"), + ("Panic recovered", "Router 代码异常，可能影响稳定性"), + ("scanner error", "流式响应中断"), + ("copy error", "非流式响应中断"), + ("Failed to read YAML file config/register.yaml", "可选配置未加载（若未启用可忽略）"), +] + # scanner error / copy error 特殊处理：context canceled → 客户端，其他 → FD 后端 SCANNER_COPY_PATTERNS = ("scanner error", "copy error") @@ -75,6 +85,13 @@ def classify_source_layer(template, original=""): return "未知" +def classify_impact(template): + for pattern, impact in IMPACT_RULES: + if pattern in template: + return impact + return "-" + + # ════════════════════════════════════════════════════════════════ # 主分析函数 # ════════════════════════════════════════════════════════════════ @@ -182,7 +199,9 @@ def _compute_error_top_n(records, top_n): "count": g["count"], "pct": round(g["count"] / total * 100, 1) if total else 0, "source_layer": source_layer, + "impact": classify_impact(g["template"]), "level": g["level"], + "urls": _extract_urls(g["originals"]), "sample_originals": g["originals"], } ) @@ -192,6 +211,16 @@ def _compute_error_top_n(records, top_n): return result +def _extract_urls(originals): + import re + + urls = set() + for line in originals: + for m in re.findall(r"https?://[A-Za-z0-9_.:-]+", line): + urls.add(m) + return sorted(urls) + + def _grep_lines(log_file, pattern, tail=None): """用 grep 从日志文件提取匹配行。""" try: @@ -240,6 +269,9 @@ def format_errors_report(result): f'请求总数: {result["total_requests"]} | ' f'错误率: {result["error_rate"]}%' ) + sections.append(" 指标口径: ERROR/WARN=日志级别计数；请求总数=HTTP 请求行数；错误率=非200请求数/请求总数×100%。") + if result["error_rate"] == 0 and (result["total_errors"] > 0 or result["total_warns"] > 0): + sections.append(" ℹ 错误率为 0.0% 仅表示 HTTP 状态码均为 200；并不代表没有 ERROR/WARN 日志。") sections.append("") # Panic @@ -266,22 +298,16 @@ def format_errors_report(result): sections.append(render_bar(bar_data, show_count=True)) sections.append("") - # 来源层表格 - table_data = [] - for e in result["error_top_n"][:10]: - table_data.append( - { - "模板": e["template"][:60], - "数量": e["count"], - "占比": f'{e["pct"]}%', - "级别": e["level"], - "来源层": e["source_layer"], - } - ) - sections.append( - render_table(table_data, columns=["模板", "数量", "占比", "级别", "来源层"], right_align={"数量", "占比"}) - ) + sections.append(" 具体模板表见: [../detail/errors_topn.md](../detail/errors_topn.md)") sections.append("") + yaml_missing_count = sum( + e["count"] for e in result["error_top_n"] if "Failed to read YAML file config/register.yaml" in e["template"] + ) + if yaml_missing_count > 0: + sections.append( + f" ℹ `Failed to read YAML file config/register.yaml` 出现 {yaml_missing_count} 次：若未启用该配置文件，可忽略。" + ) + sections.append("") # 状态码分布 if result["status_code_dist"]: diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/health.py b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/health.py index ca01d718dbc..5d1994d9405 100644 --- a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/health.py +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/health.py @@ -150,12 +150,15 @@ def _build_worker_timelines(health_events, counter_events, register_events): break all_events = [{"ts": e["ts"], "type": e["event_type"]} for e in events] + for reg in register_by_ip.get(worker_ip, []): + all_events.append({"ts": reg["ts"], "type": "REGISTERED"}) all_events.extend(recovery_events) all_events.sort(key=lambda e: e["ts"] or "") down_periods = _compute_down_periods(all_events) down_count = len(down_periods) avg_down_s = (sum(p["duration_s"] for p in down_periods) / len(down_periods)) if down_periods else 0.0 + detect_latency = _compute_detect_latency(all_events) workers[url] = { "events": all_events, @@ -165,6 +168,7 @@ def _build_worker_timelines(health_events, counter_events, register_events): "recovered": recovered, "inflight_preserved": counter_counts.get(url, 0), "down_periods": down_periods, + "avg_detect_latency_s": detect_latency, } return workers @@ -191,6 +195,24 @@ def _compute_down_periods(events): return down_periods +def _compute_detect_latency(events): + """计算 NOT_HEALTHY -> REMOVED 平均检测延迟（秒）。""" + last_unhealthy = None + latencies = [] + for evt in events: + if evt["type"] == "NOT_HEALTHY" and evt.get("ts"): + last_unhealthy = evt["ts"] + elif evt["type"] == "REMOVED" and last_unhealthy and evt.get("ts"): + try: + latencies.append((parse_ts(evt["ts"]) - parse_ts(last_unhealthy)).total_seconds()) + except ValueError: + pass + last_unhealthy = None + if not latencies: + return "-" + return round(sum(latencies) / len(latencies), 1) + + def _compute_uptime_pct(events): """计算 Worker 可用性百分比。""" if not events: @@ -313,8 +335,7 @@ def format_health_report(result): if result["diagnoses"]: sections.append("### 诊断") sections.append("") - for d in result["diagnoses"]: - sections.append(f' [{d["severity"]}] [{d["source_layer"]}] {d["message"]}') + sections.append(" 诊断见详情: [detail/health_events.md](../detail/health_events.md)") sections.append("") # Worker 可用性表格 @@ -335,6 +356,7 @@ def format_health_report(result): "在线率": f'{w["uptime_pct"]}%', "下线次数": str(w["down_count"]), "平均下线时长": avg_down or "-", + "检测延迟": (f'{w["avg_detect_latency_s"]}s' if w["avg_detect_latency_s"] != "-" else "-"), "恢复": "是" if w["recovered"] else ("否" if w["down_count"] > 0 else "-"), "inflight保留": str(w["inflight_preserved"]) if w["inflight_preserved"] > 0 else "-", } @@ -342,8 +364,8 @@ def format_health_report(result): sections.append( render_table( table_data, - columns=["Worker", "在线率", "下线次数", "平均下线时长", "恢复", "inflight保留"], - right_align={"在线率", "下线次数", "平均下线时长", "inflight保留"}, + columns=["Worker", "在线率", "下线次数", "平均下线时长", "检测延迟", "恢复", "inflight保留"], + right_align={"在线率", "下线次数", "平均下线时长", "检测延迟", "inflight保留"}, ) ) sections.append("") @@ -360,6 +382,12 @@ def format_health_report(result): # 事件详情 → 拆分到 detail_text detail_parts = ["# Worker 健康事件详情", ""] has_events = False + if result.get("diagnoses"): + detail_parts.append("## 诊断") + detail_parts.append("") + for d in result["diagnoses"]: + detail_parts.append(f'[{d["severity"]}] [{d["source_layer"]}] {d["message"]}') + detail_parts.append("") for url, w in sorted(result["workers"].items()): if w["events"]: has_events = True @@ -373,7 +401,7 @@ def format_health_report(result): # 主报告中添加引用 if has_events: - sections.append("> 完整事件详情: [details/health_events.md](details/health_events.md)") + sections.append("> 完整事件详情: [detail/health_events.md](../detail/health_events.md)") sections.append("") return "\n".join(sections), detail_text diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/latency.py b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/latency.py index eec862910e8..508cf3824d9 100644 --- a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/latency.py +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/latency.py @@ -255,6 +255,7 @@ def format_latency_report(result): f'p95={_fmt_ms(stats["p95"])} p99={_fmt_ms(stats["p99"])} ' f'max={_fmt_ms(stats["max"])}' ) + sections.append(" 指标口径: pXX=延迟分位数；吞吐量=每个时间桶内请求数(count)；调度耗时=同 request_id 的 ts_ms(max-min)。") sections.append("") # 延迟分布 @@ -331,13 +332,10 @@ def format_latency_report(result): ) sections.append("") - # 诊断 + # 诊断（仅在 detail 输出） if result["diagnoses"]: sections.append("### 诊断") - for d in result["diagnoses"]: - severity_mark = {"CRITICAL": "!!", "HIGH": "!", "MEDIUM": "~", "LOW": "-", "INFO": " "} - mark = severity_mark.get(d["severity"], " ") - sections.append(f' [{mark}] {d["message"]}') + sections.append(" 诊断见详情: [detail/latency_diagnoses.md](../detail/latency_diagnoses.md)") sections.append("") return "\n".join(sections) diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/load.py b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/load.py index c38b0b80953..2e03ba1ce63 100644 --- a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/load.py +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/load.py @@ -30,6 +30,8 @@ # Token 事件 SELECT_TOKENS_RE = re.compile(rf"select worker \((\w+)\):\s*{URL_RE},\s*tokens:\s*(\d+)") RELEASE_TOKENS_RE = re.compile(rf"release (?:([a-zA-Z_]+)\s+)?tokens:\s*{URL_RE},\s*tokens:\s*(\d+)") +SELECT_REQ_COUNT_RE = re.compile(rf"select worker \((\w+)\):\s*{URL_RE},\s*count:\s*(\d+)") +RELEASE_REQ_COUNT_RE = re.compile(rf"release worker:\s*{URL_RE},\s*count:\s*(\d+)") def _strip_scheme(url): @@ -135,11 +137,22 @@ def analyze_load(log_file, tail=None): sr_result = ( match_select_release(h3_lines + h11_lines) if h3_lines - else {"matched": [], "unmatched_selects": [], "untracked_selects": [], "failed_selects": [], "per_worker": {}} + else { + "matched": [], + "unmatched_selects": [], + "unmatched_releases": [], + "untracked_selects": [], + "failed_selects": [], + "per_worker": {}, + "id_coverage": {}, + "type_summary": {}, + "worker_type_profile": {}, + } ) # Token 统计 token_stats = _analyze_tokens(h3_lines, h11_lines) + counter_last_state = _analyze_counter_last_state(h3_lines + h11_lines) # 请求堆积检测 pileup = _detect_pileup(stats_records) @@ -154,6 +167,7 @@ def analyze_load(log_file, tail=None): "counter_anomalies": anomaly_summary, "select_release": sr_result, "token_stats": token_stats, + "counter_last_state": counter_last_state, "pileup_detected": pileup, "diagnoses": diagnoses, "summary": f"{len(stats_records)} stats 采样, {len(worker_running)} Worker(s)", @@ -191,6 +205,55 @@ def _analyze_tokens(h3_lines, h11_lines): return result +def _analyze_counter_last_state(lines): + """统计每个 worker 的 request/token counter 最后一条计数日志值与动作类型。""" + state = defaultdict( + lambda: { + "req_last_action": "-", + "req_last_value": "-", + "token_last_action": "-", + "token_last_value": "-", + "last_ts": "", + } + ) + for line in lines: + ts = extract_ts(line) or "" + m = SELECT_REQ_COUNT_RE.search(line) + if m: + w = m.group(2) + state[w]["req_last_action"] = "select" + state[w]["req_last_value"] = m.group(3) + state[w]["last_ts"] = ts + continue + m = RELEASE_REQ_COUNT_RE.search(line) + if m: + w = m.group(1) + state[w]["req_last_action"] = "release" + state[w]["req_last_value"] = m.group(2) + state[w]["last_ts"] = ts + continue + m = SELECT_TOKENS_RE.search(line) + if m: + w = m.group(2) + state[w]["token_last_action"] = "select" + state[w]["token_last_value"] = m.group(3) + state[w]["last_ts"] = ts + continue + m = RELEASE_TOKENS_RE.search(line) + if m: + w = m.group(2) + state[w]["token_last_action"] = "release" + state[w]["token_last_value"] = m.group(3) + state[w]["last_ts"] = ts + continue + + result = [] + for w in sorted(state.keys()): + s = state[w] + result.append({"worker": _strip_scheme(w), **s}) + return result + + def _detect_pileup(stats_records): """检测请求堆积：total_running 连续上升 >5 个采样点。""" if len(stats_records) < 5: diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/load_report.py b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/load_report.py index 86ba1f0d94f..9d4e9b51496 100644 --- a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/load_report.py +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/load_report.py @@ -25,8 +25,10 @@ def format_load_report(result): if result["diagnoses"]: sections.append("### 诊断") sections.append("") - for d in result["diagnoses"]: - sections.append(f' [{d["severity"]}] [{d["source_layer"]}] {d["message"]}') + sections.append( + f' 共 {len(result["diagnoses"])} 条诊断，见详情: [detail/load_diagnoses.md](../detail/load_diagnoses.md)；' + '匹配明细见 [detail/load_select_release.md](../detail/load_select_release.md)' + ) sections.append("") detail_sections.append("## 诊断") detail_sections.append("") @@ -39,6 +41,7 @@ def format_load_report(result): if ls: sections.append("### 负载概览 (total_running)") sections.append("") + sections.append(" 说明: stats 采样来自 `[stats]` 周期日志（通常每 5s 一条），用于观察当前并发与负载变化趋势。") sections.append( f' mean={ls.get("mean",0)} p50={ls.get("p50",0)} p90={ls.get("p90",0)} ' f'p99={ls.get("p99",0)} max={ls.get("max",0)} stddev={ls.get("stddev",0)}' @@ -108,6 +111,9 @@ def format_load_report(result): sections.append(render_table(type_rows, columns=["type", "counter(S/R)", "token(S/R)"])) sections.append("") sections.append(" 说明: prefill/mixed 的 token-select 同时表示 request counter + token counter 增加；decode 仅 request counter。") + sections.append(" 说明: `release prefill tokens` 会被识别为 token-release；worker type 按该 worker URL 在 select 中的类型映射（prefill/decode/mixed）。") + if type_summary.get("unknown"): + sections.append(" 说明: unknown 表示日志里缺少 worker type，且无法从邻近 select/release 关系推断。") sections.append("") detail_sections.append("## 按类型统计") detail_sections.append("") @@ -178,6 +184,29 @@ def format_load_report(result): sections.append("") detail_sections.append("## Select/Release Per-Worker") detail_sections.append("") + + if sr.get("worker_type_profile"): + sections.append("### Worker URL 类型画像（基于 select）") + sections.append("") + rows = [] + for w, p in sorted(sr["worker_type_profile"].items()): + rows.append( + { + "Worker": _strip_scheme(w), + "Dominant": p.get("dominant_type", "unknown"), + "Prefill": p.get("prefill", 0), + "Decode": p.get("decode", 0), + "Mixed": p.get("mixed", 0), + } + ) + sections.append( + render_table( + rows, + columns=["Worker", "Dominant", "Prefill", "Decode", "Mixed"], + right_align={"Prefill", "Decode", "Mixed"}, + ) + ) + sections.append("") detail_sections.append( render_table( table_data, @@ -192,7 +221,7 @@ def format_load_report(result): sections.append(" 解释: 出现 request select，但在 request release 口径下找不到匹配。可能是请求卡住、日志缺失、或窗口外释放。") for u in sr["unmatched_selects"][:3]: sections.append(f' [{u.get("select_ts","")}] {_strip_scheme(u["worker"])} ({u["type"]})') - sections.append(" > 完整列表见: [details/load_select_release.md](details/load_select_release.md)") + sections.append(" > 完整列表见: [detail/load_select_release.md](../detail/load_select_release.md)") sections.append("") detail_sections.append("## 未匹配 select（完整）") detail_sections.append("") @@ -202,11 +231,23 @@ def format_load_report(result): ) detail_sections.append("") + if sr.get("unmatched_releases"): + sections.append(f' ⚠ {len(sr["unmatched_releases"])} 个未匹配 release（已区分 req/token）') + sections.append(" > 完整列表见: [detail/load_select_release.md](../detail/load_select_release.md)") + sections.append("") + detail_sections.append("## 未匹配 release（按 release_kind 分类）") + detail_sections.append("") + for r in sr["unmatched_releases"]: + detail_sections.append( + f'- [{r.get("release_ts","")}] worker={_strip_scheme(r["worker"])} release_kind={r.get("release_kind","")} type={r.get("type","")}' + ) + detail_sections.append("") + if sr.get("untracked_selects"): sections.append(f' ℹ {len(sr["untracked_selects"])} 个 select 缺少可关联 ID，未参与卡住判定') for u in sr["untracked_selects"][:3]: sections.append(f' [{u.get("select_ts","")}] {_strip_scheme(u["worker"])} ({u["type"]})') - sections.append(" > 完整列表见: [details/load_select_release.md](details/load_select_release.md)") + sections.append(" > 完整列表见: [detail/load_select_release.md](../detail/load_select_release.md)") sections.append("") detail_sections.append("## Untracked selects（缺少可关联 ID）") detail_sections.append("") @@ -239,4 +280,20 @@ def format_load_report(result): ) sections.append("") + if result.get("counter_last_state"): + sections.append("### 计数器末状态") + sections.append("") + sections.append(" 末状态详情见: [detail/load_counter_state.md](../detail/load_counter_state.md)") + sections.append("") + detail_sections.append("## Counter / Token Counter 末状态（最后一条计数日志）") + detail_sections.append("") + detail_sections.append( + render_table( + result["counter_last_state"], + columns=["worker", "req_last_action", "req_last_value", "token_last_action", "token_last_value", "last_ts"], + right_align={"req_last_value", "token_last_value"}, + ) + ) + detail_sections.append("") + return "\n".join(sections), "\n".join(detail_sections) diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/trace.py b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/trace.py index 6c9a0323724..24af9a23500 100644 --- a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/trace.py +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/trace.py @@ -16,6 +16,7 @@ from log_parser import ( extract_tags, extract_ts, + match_select_release, parse_cache_strategy_line, parse_http_line, ) @@ -108,12 +109,14 @@ def analyze_trace(log_file, trace_ids, tail=None): # 解析事件链 events = _parse_event_chain(all_lines) lifecycle_complete = _check_lifecycle_complete(events) - diagnoses = _diagnose_trace(events, lifecycle_complete) + sr_check = match_select_release(all_lines) + diagnoses = _diagnose_trace(events, lifecycle_complete, sr_check) traces[tid] = { "events": events, "lifecycle_complete": lifecycle_complete, "diagnoses": diagnoses, + "sr_check": sr_check, "matched_tag": "session_id" if is_session else "request_id/trace_id", "related_ids": { "request_ids": sorted(related_request_ids) if is_session else [], @@ -271,7 +274,7 @@ def _check_lifecycle_complete(events): return has_entry and has_exit and (not has_select or has_release) -def _diagnose_trace(events, lifecycle_complete): +def _diagnose_trace(events, lifecycle_complete, sr_check=None): """生成追踪诊断。""" diagnoses = [] types = [e["type"] for e in events] @@ -294,6 +297,22 @@ def _diagnose_trace(events, lifecycle_complete): if "FAILED_SELECT" in types: diagnoses.append({"severity": "HIGH", "message": "Failed to select worker — 无可用 Worker"}) + if sr_check: + if sr_check.get("unmatched_selects"): + diagnoses.append( + { + "severity": "HIGH", + "message": f'match-select-release 检测到 {len(sr_check["unmatched_selects"])} 个 unmatched select', + } + ) + if sr_check.get("unmatched_releases"): + diagnoses.append( + { + "severity": "MEDIUM", + "message": f'match-select-release 检测到 {len(sr_check["unmatched_releases"])} 个 unmatched release', + } + ) + return diagnoses @@ -367,7 +386,7 @@ def format_trace_report(result): # 主报告中添加引用和摘要 safe_tid = tid.replace("/", "_") sections.append(f' 事件数: {len(trace["events"])}') - sections.append(f" > 完整事件链: [details/trace_{safe_tid}.md](details/trace_{safe_tid}.md)") + sections.append(f" > 完整事件链: [detail/trace_{safe_tid}.md](../detail/trace_{safe_tid}.md)") sections.append("") return "\n".join(sections), detail_dict diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/chart.py b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/chart.py index 83bb0203432..1eaea1369f8 100644 --- a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/chart.py +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/chart.py @@ -227,9 +227,10 @@ def render_table(data, columns=None, right_align=None): w = col_widths[col] if col in right_align: header_parts.append(f" {col:>{w}} ") + sep_parts.append("-" * (w + 1) + ":") else: header_parts.append(f" {col:<{w}} ") - sep_parts.append("-" * (w + 2)) + sep_parts.append(":" + "-" * (w + 1)) lines = [] lines.append("|" + "|".join(header_parts) + "|") diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/log_parser.py b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/log_parser.py index 200f976f2ff..548c29ebc29 100644 --- a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/log_parser.py +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/log_parser.py @@ -501,6 +501,77 @@ def _normalize_worker_type(worker_type): return "unknown" +def _infer_release_worker_type(release, selects, fallback_window_s=120): + """为未显式标注 type 的 release 近似推断 worker type。 + + 优先级： + 1) 同 worker、时间上最近且不晚于 release 的 select type + 2) 若无可解析时间戳，则使用同 worker 的最后一个 select type + 3) 推断失败返回 unknown + """ + worker = release.get("worker") + if not worker: + return "unknown" + + r_ts = _parse_ts_safe(release.get("ts")) + candidates = [s for s in selects if s.get("worker") == worker] + if not candidates: + return "unknown" + + if r_ts: + best = None + best_delta = None + for s in candidates: + s_ts = _parse_ts_safe(s.get("ts")) + if not s_ts: + continue + delta = (r_ts - s_ts).total_seconds() + if delta < 0 or delta > fallback_window_s: + continue + if best_delta is None or delta < best_delta: + best = s + best_delta = delta + if best is not None: + return _normalize_worker_type(best.get("type")) + + # 回退：按出现顺序取同 worker 的最近 select + return _normalize_worker_type(candidates[-1].get("type")) + + +def _infer_token_release_worker_type(release, selects, fallback_window_s=120): + """为 token release 推断 worker type（prefill/mixed）。 + + 注意：日志文本通常固定为 `release prefill tokens`，即使 mixed 也可能走这条日志。 + 因此 token release 的类型优先依据同 worker 的邻近 select 推断。 + """ + worker = release.get("worker") + if not worker: + return "unknown" + + r_ts = _parse_ts_safe(release.get("ts")) + candidates = [s for s in selects if s.get("worker") == worker and _normalize_worker_type(s.get("type")) in ("prefill", "mixed")] + if not candidates: + return "unknown" + + if r_ts: + best = None + best_delta = None + for s in candidates: + s_ts = _parse_ts_safe(s.get("ts")) + if not s_ts: + continue + delta = (r_ts - s_ts).total_seconds() + if delta < 0 or delta > fallback_window_s: + continue + if best_delta is None or delta < best_delta: + best = s + best_delta = delta + if best is not None: + return _normalize_worker_type(best.get("type")) + + return _normalize_worker_type(candidates[-1].get("type")) + + def match_select_release(lines, fallback_window_s=120): """匹配 select/release worker 事件对。 @@ -536,12 +607,14 @@ def match_select_release(lines, fallback_window_s=120): # Token-bearing release trm = RELEASE_TOKENS_RE.search(line) if trm: - token_type = trm.group(1) or "prefill" + token_type = trm.group(1) releases.append( { "ts": ts, "worker": trm.group(2), - "type": f'{_normalize_worker_type(token_type)}_tokens', + # 文本默认按 prefill 记，再结合同 worker 邻近 select 做纠偏（mixed 场景） + "type": f'{_normalize_worker_type(token_type or "prefill")}_tokens', + "raw_token_type": token_type or "", "tags": tags, "tokens": int(trm.group(3)), "line": line_no, @@ -716,7 +789,33 @@ def match_select_release(lines, fallback_window_s=120): "token_releases": counts["token_releases"], } - # 按 worker type 分类统计（prefill/decode/mixed） + # 基于 select 构建 worker URL -> dominant type 映射 + per_worker_type_counts = defaultdict(lambda: defaultdict(int)) + for s in selects: + per_worker_type_counts[s["worker"]][_normalize_worker_type(s.get("type"))] += 1 + worker_dominant_type = {} + for w, counts in per_worker_type_counts.items(): + worker_dominant_type[w] = sorted(counts.items(), key=lambda kv: -kv[1])[0][0] if counts else "unknown" + + # 为未显式标注 type 的 release 推断 worker type（避免大量 unknown） + inferred_release_types = {} + for i, r in enumerate(releases): + r_type_raw = str(r.get("type", "")) + if r_type_raw.endswith("_tokens"): + base_t = _normalize_worker_type(r_type_raw.replace("_tokens", "")) + # token release 按 worker URL 对应的 select 类型映射，不做邻近时间纠偏 + mapped_t = worker_dominant_type.get(r.get("worker", ""), "unknown") + if mapped_t in ("prefill", "decode", "mixed"): + base_t = mapped_t + inferred_release_types[i] = f"{base_t}_tokens" + continue + base_t = _normalize_worker_type(r_type_raw) + if base_t != "unknown": + inferred_release_types[i] = base_t + continue + inferred_release_types[i] = _infer_release_worker_type(r, selects, fallback_window_s=fallback_window_s) + + # 按 worker type 分类统计（prefill/decode/mixed，必要时保留 unknown） type_summary = defaultdict( lambda: { "counter_selects": 0, @@ -730,16 +829,57 @@ def match_select_release(lines, fallback_window_s=120): type_summary[s_type]["counter_selects"] += 1 if s_type in ("prefill", "mixed"): type_summary[s_type]["token_selects"] += 1 - for r in releases: - r_type = _normalize_worker_type(str(r.get("type", "")).replace("_tokens", "")) - if str(r.get("type", "")).endswith("_tokens"): + for i, r in enumerate(releases): + inferred = inferred_release_types.get(i, _normalize_worker_type(str(r.get("type", "")))) + r_type = _normalize_worker_type(str(inferred).replace("_tokens", "")) + if str(inferred).endswith("_tokens"): type_summary[r_type]["token_releases"] += 1 else: type_summary[r_type]["counter_releases"] += 1 + # 每个 worker URL 的类型画像（基于 select） + worker_type_profile = {} + for w, counts in per_worker_type_counts.items(): + dominant = "unknown" + if counts: + dominant = sorted(counts.items(), key=lambda kv: -kv[1])[0][0] + worker_type_profile[w] = { + "dominant_type": dominant, + "prefill": counts.get("prefill", 0), + "decode": counts.get("decode", 0), + "mixed": counts.get("mixed", 0), + "unknown": counts.get("unknown", 0), + } + + unmatched_releases = [] + for i, r in enumerate(releases): + if str(r.get("type", "")).endswith("_tokens"): + # token release: 近邻存在 prefill/mixed select 则视为可解释，不计入 unmatched + inferred_token_type = _normalize_worker_type(str(inferred_release_types.get(i, "unknown_tokens")).replace("_tokens", "")) + if inferred_token_type == "unknown": + unmatched_releases.append( + { + "worker": r.get("worker", ""), + "release_ts": r.get("ts", ""), + "type": inferred_token_type, + "release_kind": "token_release", + } + ) + continue + if i not in release_used: + unmatched_releases.append( + { + "worker": r.get("worker", ""), + "release_ts": r.get("ts", ""), + "type": _normalize_worker_type(inferred_release_types.get(i, "unknown")), + "release_kind": "request_release", + } + ) + return { "matched": matched, "unmatched_selects": unmatched_selects, + "unmatched_releases": unmatched_releases, "untracked_selects": untracked_selects, "failed_selects": failed_selects, "per_worker": pw_result, @@ -751,6 +891,7 @@ def match_select_release(lines, fallback_window_s=120): "without_any_id": without_any_id, }, "type_summary": dict(type_summary), + "worker_type_profile": worker_type_profile, } @@ -949,6 +1090,16 @@ def check(name, got, expected): "dial tcp {ip:port}: connection refused", ) + print("\n=== Testing match_select_release (token release type inference) ===") + sample_lines = [ + "[INFO] 2026/04/12 10:00:00 logger.go:1: [request_id:r1] select worker (mixed): http://10.0.0.1:9965, count: 1", + "[INFO] 2026/04/12 10:00:01 logger.go:1: [request_id:r1] release prefill tokens: http://10.0.0.1:9965, tokens: 10", + "[INFO] 2026/04/12 10:00:02 logger.go:1: [request_id:r1] release worker: http://10.0.0.1:9965, count: 0", + ] + msr = match_select_release(sample_lines) + check("mixed token_releases inferred", msr["type_summary"].get("mixed", {}).get("token_releases", 0), 1) + check("prefill token_releases remains 0", msr["type_summary"].get("prefill", {}).get("token_releases", 0), 0) + print(f'\n{"=" * 40}') print(f"Results: {passed} passed, {failed} failed") if failed: diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/troubleshoot.py b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/troubleshoot.py index a818d31150f..641c5106bee 100644 --- a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/troubleshoot.py +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/troubleshoot.py @@ -128,7 +128,14 @@ def determine_status(results): reasons.append(d["message"]) if reasons: - return "DEGRADED", ", ".join(reasons) + # 去重但保留完整信息 + deduped = [] + seen = set() + for r in reasons: + if r not in seen: + deduped.append(r) + seen.add(r) + return "DEGRADED", "；".join(deduped) if not results: return "HEALTHY", "无分析数据" @@ -148,19 +155,65 @@ def format_full_report(results, status, status_reason): - 'trace_files': {trace_id: text} 或 {} """ parts = [] - details = {"health_events": None, "load_select_release": None, "trace_files": {}} + details = { + "health_events": None, + "load_select_release": None, + "latency_diagnoses": None, + "cache_diagnosis": None, + "load_diagnoses": None, + "load_counter_state": None, + "cache_session_stickiness": None, + "cache_suboptimal": None, + "cache_eviction": None, + "cache_fallback": None, + "cache_cross": None, + "errors_topn": None, + "trace_files": {}, + } # 状态行 parts.append(f"STATUS: {status} — {status_reason}") + parts.append( + "状态定义: HEALTHY=无明显异常；DEGRADED=服务可用但存在性能/稳定性问题（需关注）；CRITICAL=服务不可用或高风险故障。" + ) parts.append("=" * 60) parts.append("") # 各维度报告 if "errors" in results: parts.append(format_errors_report(results["errors"])) + if results["errors"].get("error_top_n"): + lines = [ + "# Errors TopN 详情", + "", + "| 模板 | 数量 | 级别 | 来源层 | 影响 |", + "|:--|--:|:--|:--|:--|", + ] + for e in results["errors"]["error_top_n"]: + lines.append( + f'| {e.get("template","")} | {e.get("count",0)} | {e.get("level","")} | {e.get("source_layer","")} | {e.get("impact","-")} |' + ) + lines.append("") + lines.append("## 涉及 URLs") + lines.append("") + for e in results["errors"]["error_top_n"]: + urls = e.get("urls") or [] + if not urls: + continue + lines.append(f'- 模板: {e.get("template","")}') + for u in urls: + lines.append(f' - {u}') + lines.append("") + details["errors_topn"] = "\n".join(lines) if "latency" in results: parts.append(format_latency_report(results["latency"])) + if results["latency"].get("diagnoses"): + lines = ["# 延迟诊断详情", ""] + for d in results["latency"]["diagnoses"]: + lines.append(f'[{d.get("severity","")}] {d.get("message","")}') + lines.append("") + details["latency_diagnoses"] = "\n".join(lines) if "health" in results: summary, detail = format_health_report(results["health"]) @@ -173,9 +226,58 @@ def format_full_report(results, status, status_reason): parts.append(summary) if detail: details["load_select_release"] = detail + if results["load"].get("diagnoses"): + lines = ["# Load 诊断详情", ""] + for d in results["load"]["diagnoses"]: + lines.append(f'[{d.get("severity","")}] [{d.get("source_layer","")}] {d.get("message","")}') + lines.append("") + details["load_diagnoses"] = "\n".join(lines) + if results["load"].get("counter_last_state"): + rows = results["load"]["counter_last_state"] + lines = ["# Load Counter 末状态", "", "| worker | req_last_action | req_last_value | token_last_action | token_last_value | last_ts |", "|:--|:--|--:|:--|--:|:--|"] + for r in rows: + lines.append( + f'| {r.get("worker","")} | {r.get("req_last_action","-")} | {r.get("req_last_value","-")} | {r.get("token_last_action","-")} | {r.get("token_last_value","-")} | {r.get("last_ts","")} |' + ) + lines.append("") + details["load_counter_state"] = "\n".join(lines) if "cache" in results: - parts.append(format_cache_report(results["cache"])) + summary, detail = format_cache_report(results["cache"]) + parts.append(summary) + if detail: + details["cache_diagnosis"] = detail + c = results["cache"] + if c.get("session_stickiness"): + lines = ["# Cache Session 粘性详情", ""] + for sid, s in c["session_stickiness"].items(): + lines.append(f'- {sid}: req={s.get("total_requests",0)}, stickiness={s.get("stickiness_pct",0)}%, switches={s.get("switches",0)}') + lines.append("") + details["cache_session_stickiness"] = "\n".join(lines) + if c.get("suboptimal_selections"): + lines = ["# Cache 非最优选择详情", ""] + for x in c["suboptimal_selections"][:200]: + lines.append(f'- [{x.get("ts","")}] selected={x.get("selected","")} best={x.get("best_hr_worker","")} reason={x.get("reason","")}') + lines.append("") + details["cache_suboptimal"] = "\n".join(lines) + if c.get("eviction_impact"): + lines = ["# Cache 驱逐影响详情", ""] + for x in c["eviction_impact"][:200]: + lines.append(f'- session={x.get("session_id","")} interval={x.get("interval_mins",0)}m hitRatio_after={x.get("hitRatio_after",0)} evicted={x.get("evicted",False)}') + lines.append("") + details["cache_eviction"] = "\n".join(lines) + if c.get("fallback_reasons"): + lines = ["# Cache Fallback 原因详情", ""] + for x in c["fallback_reasons"]: + lines.append(f'- {x.get("value","")}: {x.get("count",0)} ({x.get("pct",0)}%)') + lines.append("") + details["cache_fallback"] = "\n".join(lines) + if c.get("cross_diagnosis"): + lines = ["# Cache 交叉诊断详情", ""] + for x in c["cross_diagnosis"]: + lines.append(f'- diagnosis={x.get("diagnosis","")}, action={x.get("action","")}, avg_stickiness={x.get("avg_stickiness_pct",0)}%') + lines.append("") + details["cache_cross"] = "\n".join(lines) if "trace" in results: summary, detail_dict = format_trace_report(results["trace"]) @@ -217,6 +319,40 @@ def save_detailed_report(report_text, output_dir, details=None): with open(load_path, "w", encoding="utf-8") as f: f.write(details["load_select_release"]) + if details.get("latency_diagnoses"): + latency_path = os.path.join(detail_dir, "latency_diagnoses.md") + with open(latency_path, "w", encoding="utf-8") as f: + f.write(details["latency_diagnoses"]) + + if details.get("cache_diagnosis"): + cache_path = os.path.join(detail_dir, "cache_diagnosis.md") + with open(cache_path, "w", encoding="utf-8") as f: + f.write(details["cache_diagnosis"]) + if details.get("load_diagnoses"): + with open(os.path.join(detail_dir, "load_diagnoses.md"), "w", encoding="utf-8") as f: + f.write(details["load_diagnoses"]) + if details.get("load_counter_state"): + with open(os.path.join(detail_dir, "load_counter_state.md"), "w", encoding="utf-8") as f: + f.write(details["load_counter_state"]) + if details.get("cache_session_stickiness"): + with open(os.path.join(detail_dir, "cache_session_stickiness.md"), "w", encoding="utf-8") as f: + f.write(details["cache_session_stickiness"]) + if details.get("cache_suboptimal"): + with open(os.path.join(detail_dir, "cache_suboptimal.md"), "w", encoding="utf-8") as f: + f.write(details["cache_suboptimal"]) + if details.get("cache_eviction"): + with open(os.path.join(detail_dir, "cache_eviction.md"), "w", encoding="utf-8") as f: + f.write(details["cache_eviction"]) + if details.get("cache_fallback"): + with open(os.path.join(detail_dir, "cache_fallback.md"), "w", encoding="utf-8") as f: + f.write(details["cache_fallback"]) + if details.get("cache_cross"): + with open(os.path.join(detail_dir, "cache_cross.md"), "w", encoding="utf-8") as f: + f.write(details["cache_cross"]) + if details.get("errors_topn"): + with open(os.path.join(detail_dir, "errors_topn.md"), "w", encoding="utf-8") as f: + f.write(details["errors_topn"]) + for trace_id, trace_text in details.get("trace_files", {}).items(): safe_id = trace_id.replace("/", "_") trace_path = os.path.join(detail_dir, f"trace_{safe_id}.md")