From 83279b7001264114508c64138766410ce25c1234 Mon Sep 17 00:00:00 2001 From: mouxin <494624263qq@gmail.com> Date: Mon, 13 Apr 2026 00:33:09 +0800 Subject: [PATCH] troubleshoot: clarify DEGRADED meaning in report header --- .../troubleshoot/references/error_catalog.md | 1 + .../troubleshoot/references/log_patterns.md | 11 ++ .../references/report_templates.md | 1 + .../troubleshoot/scripts/analyzers/errors.py | 9 ++ .../scripts/analyzers/load_report.py | 9 +- .../skills/troubleshoot/scripts/chart.py | 3 +- .../skills/troubleshoot/scripts/log_parser.py | 113 +++++++++++++++++- .../troubleshoot/scripts/troubleshoot.py | 18 ++- 8 files changed, 156 insertions(+), 9 deletions(-) diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/references/error_catalog.md b/fastdeploy/golang_router/.claude/skills/troubleshoot/references/error_catalog.md index ba48297d9c9..60b4931b546 100644 --- a/fastdeploy/golang_router/.claude/skills/troubleshoot/references/error_catalog.md +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/references/error_catalog.md @@ -61,6 +61,7 @@ | `Failed to select worker pair: {err}` | HIGH | FD 后端 | 请求返回 502 | | `Failed to build disaggregate_info: {err}` | HIGH | Router | 请求返回 500 | | `Failed to encode modified request: {err}` | HIGH | Router | 请求返回 500 | +| `Failed to read YAML file config/register.yaml: {err}` | LOW | Router | 启动时未找到可选配置文件（若未使用 register.yaml 可忽略） | | `Failed to select worker: {err}` | HIGH | FD 后端 | 请求返回 502 | | `Failed to connect to backend service: {err}` | HIGH | FD 后端 | 请求返回 502 | | `Request failed (attempt {n}/{max}): {err}` | MEDIUM | FD 后端 | 重试中 | diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/references/log_patterns.md b/fastdeploy/golang_router/.claude/skills/troubleshoot/references/log_patterns.md index cf33b41f723..4322909c01d 100644 --- a/fastdeploy/golang_router/.claude/skills/troubleshoot/references/log_patterns.md +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/references/log_patterns.md @@ -233,6 +233,17 @@ PD（Prefill/Decode 分离）模式下，`completions.go` 产生的 `[prefill]` --- +## Select/Release 日志细节（与代码一致） + +- `select worker (prefill): , tokens: ` +- `select worker (decode|mixed): , count: ` +- `release worker: , count: `（request counter 释放） +- `release prefill tokens: , tokens: `（token counter 释放；可能来自 prefill 或 mixed 请求路径） + +重点：release 只有上面这两种。`release worker` 不带 worker type，`release prefill tokens` 的文本也不能直接断定是 prefill（mixed 也可能调用）。因此按 `prefill/decode/mixed` 统计时，需要从 select 侧做归类；确实无法归类时才记为 `unknown`。 + +--- + ## 使用脚本工具 各 skill 的脚本位于各自的 `scripts/` 目录下，自动处理上述所有日志解析和计算。 diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/references/report_templates.md b/fastdeploy/golang_router/.claude/skills/troubleshoot/references/report_templates.md index ba9e40e9869..5eec70d1514 100644 --- a/fastdeploy/golang_router/.claude/skills/troubleshoot/references/report_templates.md +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/references/report_templates.md @@ -44,6 +44,7 @@ ### 简洁版（终端输出） - 第一行：`STATUS: HEALTHY / DEGRADED / CRITICAL — 简要说明` +- 状态定义：`HEALTHY`=无明显异常；`DEGRADED`=服务可用但性能/稳定性下降（需关注）；`CRITICAL`=服务不可用或高风险故障 - 按三层分类（Router / FD 后端 / 客户端） - 每个问题一行摘要 + 关键指标 - 末尾提示详细版文件路径 diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/errors.py b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/errors.py index b8217a5ffa4..1f3f63fcc45 100644 --- a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/errors.py +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/errors.py @@ -33,6 +33,7 @@ ("counter already zero", "Router"), ("tokenizer failed", "Router"), ("Instance {url} role is unknown", "Router"), + ("Failed to read YAML file config/register.yaml", "Router"), # 客户端 ("Invalid request body", "客户端"), ("Invalid JSON format", "客户端"), @@ -282,6 +283,14 @@ def format_errors_report(result): render_table(table_data, columns=["模板", "数量", "占比", "级别", "来源层"], right_align={"数量", "占比"}) ) sections.append("") + yaml_missing_count = sum( + e["count"] for e in result["error_top_n"] if "Failed to read YAML file config/register.yaml" in e["template"] + ) + if yaml_missing_count > 0: + sections.append( + f" ℹ `Failed to read YAML file config/register.yaml` 出现 {yaml_missing_count} 次：若未启用该配置文件，可忽略。" + ) + sections.append("") # 状态码分布 if result["status_code_dist"]: diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/load_report.py b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/load_report.py index 86ba1f0d94f..b06d26883aa 100644 --- a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/load_report.py +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/load_report.py @@ -25,8 +25,11 @@ def format_load_report(result): if result["diagnoses"]: sections.append("### 诊断") sections.append("") - for d in result["diagnoses"]: + max_diag_in_summary = 8 + for d in result["diagnoses"][:max_diag_in_summary]: sections.append(f' [{d["severity"]}] [{d["source_layer"]}] {d["message"]}') + if len(result["diagnoses"]) > max_diag_in_summary: + sections.append(f' ... 其余 {len(result["diagnoses"]) - max_diag_in_summary} 项见 detail 报告') sections.append("") detail_sections.append("## 诊断") detail_sections.append("") @@ -39,6 +42,7 @@ def format_load_report(result): if ls: sections.append("### 负载概览 (total_running)") sections.append("") + sections.append(" 说明: stats 采样来自 `[stats]` 周期日志（通常每 5s 一条），用于观察当前并发与负载变化趋势。") sections.append( f' mean={ls.get("mean",0)} p50={ls.get("p50",0)} p90={ls.get("p90",0)} ' f'p99={ls.get("p99",0)} max={ls.get("max",0)} stddev={ls.get("stddev",0)}' @@ -108,6 +112,9 @@ def format_load_report(result): sections.append(render_table(type_rows, columns=["type", "counter(S/R)", "token(S/R)"])) sections.append("") sections.append(" 说明: prefill/mixed 的 token-select 同时表示 request counter + token counter 增加；decode 仅 request counter。") + sections.append(" 说明: token-release 由同 worker 邻近 select 推断到 prefill/mixed，不直接依赖 `release prefill tokens` 文本。") + if type_summary.get("unknown"): + sections.append(" 说明: unknown 表示日志里缺少 worker type，且无法从邻近 select/release 关系推断。") sections.append("") detail_sections.append("## 按类型统计") detail_sections.append("") diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/chart.py b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/chart.py index 83bb0203432..1eaea1369f8 100644 --- a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/chart.py +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/chart.py @@ -227,9 +227,10 @@ def render_table(data, columns=None, right_align=None): w = col_widths[col] if col in right_align: header_parts.append(f" {col:>{w}} ") + sep_parts.append("-" * (w + 1) + ":") else: header_parts.append(f" {col:<{w}} ") - sep_parts.append("-" * (w + 2)) + sep_parts.append(":" + "-" * (w + 1)) lines = [] lines.append("|" + "|".join(header_parts) + "|") diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/log_parser.py b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/log_parser.py index 200f976f2ff..a5d646dc029 100644 --- a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/log_parser.py +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/log_parser.py @@ -501,6 +501,77 @@ def _normalize_worker_type(worker_type): return "unknown" +def _infer_release_worker_type(release, selects, fallback_window_s=120): + """为未显式标注 type 的 release 近似推断 worker type。 + + 优先级： + 1) 同 worker、时间上最近且不晚于 release 的 select type + 2) 若无可解析时间戳，则使用同 worker 的最后一个 select type + 3) 推断失败返回 unknown + """ + worker = release.get("worker") + if not worker: + return "unknown" + + r_ts = _parse_ts_safe(release.get("ts")) + candidates = [s for s in selects if s.get("worker") == worker] + if not candidates: + return "unknown" + + if r_ts: + best = None + best_delta = None + for s in candidates: + s_ts = _parse_ts_safe(s.get("ts")) + if not s_ts: + continue + delta = (r_ts - s_ts).total_seconds() + if delta < 0 or delta > fallback_window_s: + continue + if best_delta is None or delta < best_delta: + best = s + best_delta = delta + if best is not None: + return _normalize_worker_type(best.get("type")) + + # 回退：按出现顺序取同 worker 的最近 select + return _normalize_worker_type(candidates[-1].get("type")) + + +def _infer_token_release_worker_type(release, selects, fallback_window_s=120): + """为 token release 推断 worker type（prefill/mixed）。 + + 注意：日志文本通常固定为 `release prefill tokens`，即使 mixed 也可能走这条日志。 + 因此 token release 的类型优先依据同 worker 的邻近 select 推断。 + """ + worker = release.get("worker") + if not worker: + return "unknown" + + r_ts = _parse_ts_safe(release.get("ts")) + candidates = [s for s in selects if s.get("worker") == worker and _normalize_worker_type(s.get("type")) in ("prefill", "mixed")] + if not candidates: + return "unknown" + + if r_ts: + best = None + best_delta = None + for s in candidates: + s_ts = _parse_ts_safe(s.get("ts")) + if not s_ts: + continue + delta = (r_ts - s_ts).total_seconds() + if delta < 0 or delta > fallback_window_s: + continue + if best_delta is None or delta < best_delta: + best = s + best_delta = delta + if best is not None: + return _normalize_worker_type(best.get("type")) + + return _normalize_worker_type(candidates[-1].get("type")) + + def match_select_release(lines, fallback_window_s=120): """匹配 select/release worker 事件对。 @@ -536,12 +607,14 @@ def match_select_release(lines, fallback_window_s=120): # Token-bearing release trm = RELEASE_TOKENS_RE.search(line) if trm: - token_type = trm.group(1) or "prefill" + token_type = trm.group(1) releases.append( { "ts": ts, "worker": trm.group(2), - "type": f'{_normalize_worker_type(token_type)}_tokens', + # 不直接信任日志里的 token type 文本（"release prefill tokens" 也可能来自 mixed） + "type": "unknown_tokens", + "raw_token_type": token_type or "", "tags": tags, "tokens": int(trm.group(3)), "line": line_no, @@ -716,7 +789,24 @@ def match_select_release(lines, fallback_window_s=120): "token_releases": counts["token_releases"], } - # 按 worker type 分类统计（prefill/decode/mixed） + # 为未显式标注 type 的 release 推断 worker type（避免大量 unknown） + inferred_release_types = {} + for i, r in enumerate(releases): + r_type_raw = str(r.get("type", "")) + if r_type_raw.endswith("_tokens"): + base_t = _normalize_worker_type(r_type_raw.replace("_tokens", "")) + if base_t == "unknown": + # token release 的 worker type 由同 worker 邻近 select 推断（prefill/mixed） + base_t = _infer_token_release_worker_type(r, selects, fallback_window_s=fallback_window_s) + inferred_release_types[i] = f"{base_t}_tokens" + continue + base_t = _normalize_worker_type(r_type_raw) + if base_t != "unknown": + inferred_release_types[i] = base_t + continue + inferred_release_types[i] = _infer_release_worker_type(r, selects, fallback_window_s=fallback_window_s) + + # 按 worker type 分类统计（prefill/decode/mixed，必要时保留 unknown） type_summary = defaultdict( lambda: { "counter_selects": 0, @@ -730,9 +820,10 @@ def match_select_release(lines, fallback_window_s=120): type_summary[s_type]["counter_selects"] += 1 if s_type in ("prefill", "mixed"): type_summary[s_type]["token_selects"] += 1 - for r in releases: - r_type = _normalize_worker_type(str(r.get("type", "")).replace("_tokens", "")) - if str(r.get("type", "")).endswith("_tokens"): + for i, r in enumerate(releases): + inferred = inferred_release_types.get(i, _normalize_worker_type(str(r.get("type", "")))) + r_type = _normalize_worker_type(str(inferred).replace("_tokens", "")) + if str(inferred).endswith("_tokens"): type_summary[r_type]["token_releases"] += 1 else: type_summary[r_type]["counter_releases"] += 1 @@ -949,6 +1040,16 @@ def check(name, got, expected): "dial tcp {ip:port}: connection refused", ) + print("\n=== Testing match_select_release (token release type inference) ===") + sample_lines = [ + "[INFO] 2026/04/12 10:00:00 logger.go:1: [request_id:r1] select worker (mixed): http://10.0.0.1:9965, count: 1", + "[INFO] 2026/04/12 10:00:01 logger.go:1: [request_id:r1] release prefill tokens: http://10.0.0.1:9965, tokens: 10", + "[INFO] 2026/04/12 10:00:02 logger.go:1: [request_id:r1] release worker: http://10.0.0.1:9965, count: 0", + ] + msr = match_select_release(sample_lines) + check("mixed token_releases inferred", msr["type_summary"].get("mixed", {}).get("token_releases", 0), 1) + check("prefill token_releases remains 0", msr["type_summary"].get("prefill", {}).get("token_releases", 0), 0) + print(f'\n{"=" * 40}') print(f"Results: {passed} passed, {failed} failed") if failed: diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/troubleshoot.py b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/troubleshoot.py index a818d31150f..3b80cd45c5e 100644 --- a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/troubleshoot.py +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/troubleshoot.py @@ -128,7 +128,20 @@ def determine_status(results): reasons.append(d["message"]) if reasons: - return "DEGRADED", ", ".join(reasons) + # 去重并限制长度，避免状态行过长难读 + deduped = [] + seen = set() + for r in reasons: + if r not in seen: + deduped.append(r) + seen.add(r) + max_reasons = 4 + shown = deduped[:max_reasons] + extra = len(deduped) - len(shown) + summary = "；".join(shown) + if extra > 0: + summary += f"；另有 {extra} 项诊断见各维度 detail 报告" + return "DEGRADED", summary if not results: return "HEALTHY", "无分析数据" @@ -152,6 +165,9 @@ def format_full_report(results, status, status_reason): # 状态行 parts.append(f"STATUS: {status} — {status_reason}") + parts.append( + "状态定义: HEALTHY=无明显异常；DEGRADED=服务可用但存在性能/稳定性问题（需关注）；CRITICAL=服务不可用或高风险故障。" + ) parts.append("=" * 60) parts.append("")