diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/SKILL.md b/fastdeploy/golang_router/.claude/skills/troubleshoot/SKILL.md index 43ee91a46b1..7f7a5793e91 100644 --- a/fastdeploy/golang_router/.claude/skills/troubleshoot/SKILL.md +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/SKILL.md @@ -24,10 +24,9 @@ IMPORTANT: 执行前务必先读取 references/log_patterns.md 了解日志格 运行脚本前,Claude 必须按以下顺序向用户确认参数: ### 1. 日志文件路径 -使用 AskUserQuestion 工具向用户询问日志文件路径。提供常见的默认选项,同时允许用户直接输入自定义路径(支持绝对路径和相对路径): +使用 AskUserQuestion 工具向用户询问日志文件路径。提供两个常用快捷选项(客户端会自动提供 Other 自定义输入): - 选项 1: `logs/router.log`(默认) - 选项 2: `fd-router.log`(golang_router 根目录) -- 选项 3: 用户通过 Other 输入自定义路径 **重要规则**: - 如果用户已经在消息中明确指定了日志路径,直接使用该路径,跳过询问步骤 @@ -37,11 +36,10 @@ IMPORTANT: 执行前务必先读取 references/log_patterns.md 了解日志格 如果用户直接确认或未指定路径,使用脚本的自动发现逻辑。 ### 2. 分析范围 -向用户询问分析范围: -> "请选择分析范围: -> 1. **全量分析**(默认)— 分析整个日志文件 -> 2. **尾部分析** — 只分析最近数据(可指定行数或时间如 `--tail 5000` 或 `--tail 30m`) -> 3. **指定时间段** — 分析特定时间范围内的日志" +必须使用 **AskUserQuestion 的离散选项**(不要只发纯文本编号): +- 选项 1: `全量分析(默认)` — 分析整个日志文件 +- 选项 2: `尾部分析` — 只分析最近数据(可指定行数或时间如 `--tail 5000` 或 `--tail 30m`) +- 选项 3: `指定时间段` — 分析特定时间范围内的日志 如果用户未选择,默认使用全量分析。 @@ -54,11 +52,10 @@ IMPORTANT: 执行前务必先读取 references/log_patterns.md 了解日志格 `--start/--end` 与 `--tail` 互斥。 ### 3. 分析模式 -向用户询问分析模式: -> "请选择分析模式: -> 1. **完整分析**(默认)— 运行所有维度(errors + latency + health + cache + load) -> 2. **单维度/多维度分析** — 选择特定维度(errors / latency / health / cache / load),可选多个 -> 3. **请求追踪** — 追踪特定请求 ID(需提供 ID)" +必须使用 **AskUserQuestion 的离散选项**(不要只发纯文本编号): +- 选项 1: `完整分析(默认)` — 运行所有维度(errors + latency + health + cache + load) +- 选项 2: `单维度/多维度分析` — 选择特定维度(errors / latency / health / cache / load),可选多个 +- 选项 3: `请求追踪` — 追踪特定请求 ID(需提供 ID) 如果用户未选择,默认使用完整分析。 diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/load.py b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/load.py index 9be82357494..c38b0b80953 100644 --- a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/load.py +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/load.py @@ -13,9 +13,9 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from chart import render_bar, render_sparkline, render_table from log_parser import extract_ts, match_select_release, parse_stats_line from stats import compute_statistics, time_bucket +from analyzers.load_report import format_load_report # ════════════════════════════════════════════════════════════════ # Counter 异常检测正则 @@ -28,14 +28,21 @@ TOKEN_PRESERVED_RE = re.compile(rf"token counter preserved.*?{URL_RE}") # Token 事件 -SELECT_TOKENS_RE = re.compile(rf"select worker \(prefill\):\s*{URL_RE},\s*tokens:\s*(\d+)") -RELEASE_TOKENS_RE = re.compile(rf"release prefill tokens:\s*{URL_RE},\s*tokens:\s*(\d+)") +SELECT_TOKENS_RE = re.compile(rf"select worker \((\w+)\):\s*{URL_RE},\s*tokens:\s*(\d+)") +RELEASE_TOKENS_RE = re.compile(rf"release (?:([a-zA-Z_]+)\s+)?tokens:\s*{URL_RE},\s*tokens:\s*(\d+)") def _strip_scheme(url): return re.sub(r"^https?://", "", url) +def _normalize_worker_type(worker_type): + t = (worker_type or "unknown").lower() + if t in ("prefill", "decode", "mixed"): + return t + return "unknown" + + def parse_counter_anomaly(line): """解析 H5 counter 异常行。""" ts = extract_ts(line) @@ -73,7 +80,7 @@ def analyze_load(log_file, tail=None): r"counter preserved|cleanup unhealthy|removed counters|counter already|double-release|preserved counters", tail, ) - h11_lines = _grep_lines(log_file, r"release prefill tokens", tail) + h11_lines = _grep_lines(log_file, r"release (?:[a-zA-Z_]+\s+)?tokens", tail) # 解析 stats 行 stats_records = [r for line in h7_lines for r in [parse_stats_line(line)] if r] @@ -161,12 +168,12 @@ def _analyze_tokens(h3_lines, h11_lines): for line in h3_lines: m = SELECT_TOKENS_RE.search(line) if m: - token_alloc[m.group(1)].append(int(m.group(2))) + token_alloc[m.group(2)].append(int(m.group(3))) for line in h11_lines: m = RELEASE_TOKENS_RE.search(line) if m: - token_release[m.group(1)].append(int(m.group(2))) + token_release[m.group(2)].append(int(m.group(3))) result = [] all_workers = set(token_alloc.keys()) | set(token_release.keys()) @@ -285,135 +292,6 @@ def _diagnose(load_stats, worker_load, anomaly_summary, sr_result, token_stats, # ════════════════════════════════════════════════════════════════ -def format_load_report(result): - """将分析结果格式化为终端报告。""" - sections = ["## 负载与计数器分析", ""] - sections.append(f' {result["summary"]}') - sections.append("") - - if result["diagnoses"]: - sections.append("### 诊断") - sections.append("") - for d in result["diagnoses"]: - sections.append(f' [{d["severity"]}] [{d["source_layer"]}] {d["message"]}') - sections.append("") - - # 负载概览 - ls = result.get("load_stats", {}) - if ls: - sections.append("### 负载概览 (total_running)") - sections.append("") - sections.append( - f' mean={ls.get("mean",0)} p50={ls.get("p50",0)} p90={ls.get("p90",0)} ' - f'p99={ls.get("p99",0)} max={ls.get("max",0)} stddev={ls.get("stddev",0)}' - ) - sections.append("") - - # Per-Worker 负载 - if result["worker_load"]: - sections.append("### Per-Worker 负载") - sections.append("") - bar_data = [ - {"label": w["worker"][:25], "value": min(100, w["avg_running"] * 5), "count": w["avg_running"]} - for w in result["worker_load"] - ] - sections.append(render_bar(bar_data, show_count=True)) - sections.append("") - - # 负载趋势 - if result["load_trend"] and len(result["load_trend"]) > 1: - sections.append("### 负载趋势") - sections.append("") - sections.append( - render_sparkline( - result["load_trend"], value_field="total_running_mean", title="Total Running", y_label="req" - ) - ) - sections.append("") - - # Counter 异常 - if result["counter_anomalies"]: - sections.append("### 计数器异常") - sections.append("") - for a in result["counter_anomalies"]: - workers_str = ", ".join(f'{_strip_scheme(w)}({c})' for w, c in a["workers"].items()) - sections.append(f' {a["type"]}: {a["total"]} 次 [{workers_str}]') - sections.append("") - - id_cov = result.get("select_release", {}).get("id_coverage", {}) - if id_cov: - sections.append("### 请求标识覆盖(基于 select 近似请求数)") - sections.append("") - sections.append( - " total={total} | with_request_id={with_rid} | without_request_id={without_rid} | " - "with_alt_id={with_alt} | without_any_id={without_any}".format( - total=id_cov.get("total_requests_estimated", 0), - with_rid=id_cov.get("with_request_id", 0), - without_rid=id_cov.get("without_request_id", 0), - with_alt=id_cov.get("with_alt_id", 0), - without_any=id_cov.get("without_any_id", 0), - ) - ) - if id_cov.get("without_any_id", 0) > 0: - sections.append(" ℹ 无 request/session/trace/req_id 时,不做退化匹配,仅统计为 untracked。") - sections.append("") - - # Select/Release 匹配 - sr = result.get("select_release", {}) - if sr.get("per_worker"): - sections.append("### Select/Release 匹配") - sections.append("") - id_cov = sr.get("id_coverage", {}) - no_correlatable_id = (id_cov.get("with_request_id", 0) + id_cov.get("with_alt_id", 0)) == 0 - table_data = [] - for w_url, pw in sorted(sr["per_worker"].items()): - delta_display = "N/A" if no_correlatable_id else str(pw["delta"]) - table_data.append( - { - "Worker": _strip_scheme(w_url), - "Select": str(pw["selects"]), - "Release": str(pw["releases"]), - "Delta": delta_display, - } - ) - sections.append( - render_table( - table_data, - columns=["Worker", "Select", "Release", "Delta"], - right_align={"Select", "Release", "Delta"}, - ) - ) - sections.append("") - if no_correlatable_id: - sections.append(" ℹ 当前样本无可关联 ID,Delta 不用于请求泄漏结论。") - sections.append("") - - if sr.get("unmatched_selects"): - sections.append(f' ⚠ {len(sr["unmatched_selects"])} 个未匹配 select(疑似请求卡住)') - for u in sr["unmatched_selects"][:5]: - sections.append(f' [{u.get("select_ts","")}] {_strip_scheme(u["worker"])} ({u["type"]})') - sections.append("") - - if sr.get("untracked_selects"): - sections.append(f' ℹ {len(sr["untracked_selects"])} 个 select 缺少可关联 ID,未参与卡住判定') - for u in sr["untracked_selects"][:5]: - sections.append(f' [{u.get("select_ts","")}] {_strip_scheme(u["worker"])} ({u["type"]})') - sections.append("") - - # Token 统计 - if result.get("token_stats"): - sections.append("### Token 计数器") - sections.append("") - sections.append( - render_table( - result["token_stats"], - columns=["worker", "alloc_count", "alloc_avg", "release_count"], - right_align={"alloc_count", "alloc_avg", "release_count"}, - ) - ) - sections.append("") - - return "\n".join(sections) # ════════════════════════════════════════════════════════════════ diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/load_report.py b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/load_report.py new file mode 100644 index 00000000000..86ba1f0d94f --- /dev/null +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/load_report.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python3 +"""Load report formatter.""" + +from chart import render_bar, render_sparkline, render_table + + +def _strip_scheme(url): + import re + return re.sub(r"^https?://", "", url) + + +def format_load_report(result): + """将分析结果格式化为终端报告。 + + Returns: + tuple: (summary_text, detail_text) + """ + sections = ["## 负载与计数器分析", ""] + sections.append(f' {result["summary"]}') + sections.append("") + detail_sections = ["# 负载与计数器详情", ""] + detail_sections.append(f'总结: {result["summary"]}') + detail_sections.append("") + + if result["diagnoses"]: + sections.append("### 诊断") + sections.append("") + for d in result["diagnoses"]: + sections.append(f' [{d["severity"]}] [{d["source_layer"]}] {d["message"]}') + sections.append("") + detail_sections.append("## 诊断") + detail_sections.append("") + for d in result["diagnoses"]: + detail_sections.append(f'[{d["severity"]}] [{d["source_layer"]}] {d["message"]}') + detail_sections.append("") + + # 负载概览 + ls = result.get("load_stats", {}) + if ls: + sections.append("### 负载概览 (total_running)") + sections.append("") + sections.append( + f' mean={ls.get("mean",0)} p50={ls.get("p50",0)} p90={ls.get("p90",0)} ' + f'p99={ls.get("p99",0)} max={ls.get("max",0)} stddev={ls.get("stddev",0)}' + ) + sections.append("") + + # Per-Worker 负载 + if result["worker_load"]: + sections.append("### Per-Worker 负载") + sections.append("") + bar_data = [ + {"label": w["worker"][:25], "value": min(100, w["avg_running"] * 5), "count": w["avg_running"]} + for w in result["worker_load"] + ] + sections.append(render_bar(bar_data, show_count=True)) + sections.append("") + + # 负载趋势 + if result["load_trend"] and len(result["load_trend"]) > 1: + sections.append("### 负载趋势") + sections.append("") + sections.append( + render_sparkline( + result["load_trend"], value_field="total_running_mean", title="Total Running", y_label="req" + ) + ) + sections.append("") + + # Counter 异常 + if result["counter_anomalies"]: + sections.append("### 计数器异常") + sections.append("") + for a in result["counter_anomalies"]: + workers_str = ", ".join(f'{_strip_scheme(w)}({c})' for w, c in a["workers"].items()) + sections.append(f' {a["type"]}: {a["total"]} 次 [{workers_str}]') + sections.append("") + detail_sections.append("## 计数器异常") + detail_sections.append("") + for a in result["counter_anomalies"]: + workers_str = ", ".join(f'{_strip_scheme(w)}({c})' for w, c in a["workers"].items()) + detail_sections.append(f'- {a["type"]}: {a["total"]} 次 [{workers_str}]') + detail_sections.append("") + + # 按 prefill / decode / mixed 分类统计 + type_summary = result.get("select_release", {}).get("type_summary", {}) + if type_summary: + sections.append("### 按类型统计(prefill / decode / mixed)") + sections.append("") + type_rows = [] + for t in ("prefill", "decode", "mixed", "unknown"): + s = type_summary.get(t) + if not s: + continue + token_display = "-" + if t == "prefill": + token_display = f'{s.get("token_selects",0)}/{s.get("token_releases",0)}' + elif t == "mixed" and (s.get("token_selects", 0) > 0 or s.get("token_releases", 0) > 0): + token_display = f'{s.get("token_selects",0)}/{s.get("token_releases",0)}' + type_rows.append( + { + "type": t, + "counter(S/R)": f'{s.get("counter_selects",0)}/{s.get("counter_releases",0)}', + "token(S/R)": token_display, + } + ) + if type_rows: + sections.append(render_table(type_rows, columns=["type", "counter(S/R)", "token(S/R)"])) + sections.append("") + sections.append(" 说明: prefill/mixed 的 token-select 同时表示 request counter + token counter 增加;decode 仅 request counter。") + sections.append("") + detail_sections.append("## 按类型统计") + detail_sections.append("") + detail_sections.append(render_table(type_rows, columns=["type", "counter(S/R)", "token(S/R)"])) + detail_sections.append("") + + id_cov = result.get("select_release", {}).get("id_coverage", {}) + if id_cov: + sections.append("### 请求标识覆盖(基于 select 近似请求数)") + sections.append("") + sections.append( + " total={total} | with_request_id={with_rid} | without_request_id={without_rid} | " + "with_alt_id={with_alt} | without_any_id={without_any}".format( + total=id_cov.get("total_requests_estimated", 0), + with_rid=id_cov.get("with_request_id", 0), + without_rid=id_cov.get("without_request_id", 0), + with_alt=id_cov.get("with_alt_id", 0), + without_any=id_cov.get("without_any_id", 0), + ) + ) + if id_cov.get("without_any_id", 0) > 0: + sections.append(" ℹ 无 request/session/trace/req_id 时,不做退化匹配,仅统计为 untracked。") + sections.append(" 字段说明: total=select 事件总数估算;with_request_id=含 request_id;without_request_id=不含 request_id;with_alt_id=含 req_id/trace_id/session_id;without_any_id=四类 ID 都缺失。") + sections.append("") + detail_sections.append("## 请求标识覆盖字段说明") + detail_sections.append("") + detail_sections.append( + "- total: select 事件总数(近似请求数)\n" + "- with_request_id: 携带 request_id 的 select 数\n" + "- without_request_id: 未携带 request_id 的 select 数\n" + "- with_alt_id: 无 request_id 但携带 req_id/trace_id/session_id 的 select 数\n" + "- without_any_id: 四类 ID 都没有,无法做请求级关联" + ) + detail_sections.append("") + + # Select/Release 匹配 + sr = result.get("select_release", {}) + if sr.get("per_worker"): + sections.append("### Select/Release 匹配") + sections.append("") + id_cov = sr.get("id_coverage", {}) + no_correlatable_id = (id_cov.get("with_request_id", 0) + id_cov.get("with_alt_id", 0)) == 0 + table_data = [] + for w_url, pw in sorted(sr["per_worker"].items()): + delta_display = "N/A" if no_correlatable_id else str(pw["delta"]) + table_data.append( + { + "Worker": _strip_scheme(w_url), + "ReqSelect": str(pw["selects"]), + "ReqRelease": str(pw["releases"]), + "ReqDelta": delta_display, + "TokenSelect": str(pw.get("token_selects", 0)), + "TokenRelease": str(pw.get("token_releases", 0)), + } + ) + sections.append( + render_table( + table_data, + columns=["Worker", "ReqSelect", "ReqRelease", "ReqDelta", "TokenSelect", "TokenRelease"], + right_align={"ReqSelect", "ReqRelease", "ReqDelta", "TokenSelect", "TokenRelease"}, + ) + ) + sections.append("") + if no_correlatable_id: + sections.append(" ℹ 当前样本无可关联 ID,Delta 不用于请求泄漏结论。") + sections.append("") + sections.append(" 说明: TokenSelect 按 worker type 统计(prefill + mixed 的 select 都计入),不依赖日志里是否出现 tokens 字段。") + sections.append("") + detail_sections.append("## Select/Release Per-Worker") + detail_sections.append("") + detail_sections.append( + render_table( + table_data, + columns=["Worker", "ReqSelect", "ReqRelease", "ReqDelta", "TokenSelect", "TokenRelease"], + right_align={"ReqSelect", "ReqRelease", "ReqDelta", "TokenSelect", "TokenRelease"}, + ) + ) + detail_sections.append("") + + if sr.get("unmatched_selects"): + sections.append(f' ⚠ {len(sr["unmatched_selects"])} 个未匹配 select(疑似请求卡住)') + sections.append(" 解释: 出现 request select,但在 request release 口径下找不到匹配。可能是请求卡住、日志缺失、或窗口外释放。") + for u in sr["unmatched_selects"][:3]: + sections.append(f' [{u.get("select_ts","")}] {_strip_scheme(u["worker"])} ({u["type"]})') + sections.append(" > 完整列表见: [details/load_select_release.md](details/load_select_release.md)") + sections.append("") + detail_sections.append("## 未匹配 select(完整)") + detail_sections.append("") + for u in sr["unmatched_selects"]: + detail_sections.append( + f'- [{u.get("select_ts","")}] worker={_strip_scheme(u["worker"])} type={u["type"]} note={u.get("note","")}' + ) + detail_sections.append("") + + if sr.get("untracked_selects"): + sections.append(f' ℹ {len(sr["untracked_selects"])} 个 select 缺少可关联 ID,未参与卡住判定') + for u in sr["untracked_selects"][:3]: + sections.append(f' [{u.get("select_ts","")}] {_strip_scheme(u["worker"])} ({u["type"]})') + sections.append(" > 完整列表见: [details/load_select_release.md](details/load_select_release.md)") + sections.append("") + detail_sections.append("## Untracked selects(缺少可关联 ID)") + detail_sections.append("") + for u in sr["untracked_selects"]: + detail_sections.append( + f'- [{u.get("select_ts","")}] worker={_strip_scheme(u["worker"])} type={u["type"]} note={u.get("note","")}' + ) + detail_sections.append("") + + if sr.get("failed_selects"): + sections.append(f' ⚠ Failed to select: {len(sr["failed_selects"])} 次') + sections.append(" 解释: 路由在该时刻未能选出可用 worker,通常意味着可用池不足或健康状态异常。") + sections.append("") + detail_sections.append("## Failed to select") + detail_sections.append("") + for f in sr["failed_selects"]: + detail_sections.append(f'- [{f.get("ts","")}] line={f.get("line","")}') + detail_sections.append("") + + # Token 统计 + if result.get("token_stats"): + sections.append("### Token 计数器") + sections.append("") + sections.append( + render_table( + result["token_stats"], + columns=["worker", "alloc_count", "alloc_avg", "release_count"], + right_align={"alloc_count", "alloc_avg", "release_count"}, + ) + ) + sections.append("") + + return "\n".join(sections), "\n".join(detail_sections) diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/log_parser.py b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/log_parser.py index 44f5cdebd94..200f976f2ff 100644 --- a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/log_parser.py +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/log_parser.py @@ -466,8 +466,8 @@ def parse_error_line(line): SELECT_RE = re.compile(rf"select worker\s*(?:\((\w+)\))?:\s*({URL_RE})") RELEASE_RE = re.compile(rf"release worker\s*(?:\((\w+)\))?:\s*({URL_RE})") FAILED_SELECT_RE = re.compile(r"Failed to select") -SELECT_TOKENS_RE = re.compile(rf"select worker \(prefill\):\s*({URL_RE}),\s*tokens:\s*(\d+)") -RELEASE_TOKENS_RE = re.compile(rf"release prefill tokens:\s*({URL_RE}),\s*tokens:\s*(\d+)") +SELECT_TOKENS_RE = re.compile(rf"select worker \((\w+)\):\s*({URL_RE}),\s*tokens:\s*(\d+)") +RELEASE_TOKENS_RE = re.compile(rf"release (?:([a-zA-Z_]+)\s+)?tokens:\s*({URL_RE}),\s*tokens:\s*(\d+)") def _parse_ts_safe(ts): @@ -493,6 +493,14 @@ def _select_match_key(tags): return (None, None) +def _normalize_worker_type(worker_type): + """归一化 worker type。""" + t = (worker_type or "unknown").lower() + if t in ("prefill", "decode", "mixed"): + return t + return "unknown" + + def match_select_release(lines, fallback_window_s=120): """匹配 select/release worker 事件对。 @@ -516,10 +524,10 @@ def match_select_release(lines, fallback_window_s=120): selects.append( { "ts": ts, - "worker": tm.group(1), - "type": "prefill", + "worker": tm.group(2), + "type": _normalize_worker_type(tm.group(1)), "tags": tags, - "tokens": int(tm.group(2)), + "tokens": int(tm.group(3)), "line": line_no, } ) @@ -528,13 +536,14 @@ def match_select_release(lines, fallback_window_s=120): # Token-bearing release trm = RELEASE_TOKENS_RE.search(line) if trm: + token_type = trm.group(1) or "prefill" releases.append( { "ts": ts, - "worker": trm.group(1), - "type": "prefill_tokens", + "worker": trm.group(2), + "type": f'{_normalize_worker_type(token_type)}_tokens', "tags": tags, - "tokens": int(trm.group(2)), + "tokens": int(trm.group(3)), "line": line_no, } ) @@ -546,7 +555,7 @@ def match_select_release(lines, fallback_window_s=120): { "ts": ts, "worker": sm.group(2), - "type": sm.group(1) or "unknown", + "type": _normalize_worker_type(sm.group(1)), "tags": tags, "tokens": None, "line": line_no, @@ -560,7 +569,7 @@ def match_select_release(lines, fallback_window_s=120): { "ts": ts, "worker": rm.group(2), - "type": rm.group(1) or "unknown", + "type": _normalize_worker_type(rm.group(1)), "tags": tags, "tokens": None, "line": line_no, @@ -576,8 +585,11 @@ def match_select_release(lines, fallback_window_s=120): unmatched_selects = [] release_used = set() + # 请求生命周期匹配只使用 request counter release(排除 token release) + counter_release_indexes = [i for i, r in enumerate(releases) if not str(r.get("type", "")).endswith("_tokens")] release_by_key = defaultdict(list) - for i, r in enumerate(releases): + for i in counter_release_indexes: + r = releases[i] _, key = _select_match_key(r.get("tags", {})) if key: release_by_key[key].append(i) @@ -639,7 +651,8 @@ def match_select_release(lines, fallback_window_s=120): sdt = _parse_ts_safe(s["ts"]) best_idx = None best_delta = None - for ri, r in enumerate(releases): + for ri in counter_release_indexes: + r = releases[ri] if ri in release_used: continue if r.get("worker") != s.get("worker"): @@ -679,12 +692,19 @@ def match_select_release(lines, fallback_window_s=120): } ) - # Per-worker summary - per_worker = defaultdict(lambda: {"selects": 0, "releases": 0}) + # Per-worker summary(按 worker type 统计,不依赖日志中的 tokens 字段) + # 规则:prefill/mixed 的 select 均计入 token_selects。 + per_worker = defaultdict(lambda: {"selects": 0, "releases": 0, "token_selects": 0, "token_releases": 0}) for s in selects: + s_type = _normalize_worker_type(s.get("type")) per_worker[s["worker"]]["selects"] += 1 + if s_type in ("prefill", "mixed"): + per_worker[s["worker"]]["token_selects"] += 1 for r in releases: - per_worker[r["worker"]]["releases"] += 1 + if str(r.get("type", "")).endswith("_tokens"): + per_worker[r["worker"]]["token_releases"] += 1 + else: + per_worker[r["worker"]]["releases"] += 1 pw_result = {} for w, counts in per_worker.items(): @@ -692,7 +712,30 @@ def match_select_release(lines, fallback_window_s=120): "selects": counts["selects"], "releases": counts["releases"], "delta": counts["selects"] - counts["releases"], + "token_selects": counts["token_selects"], + "token_releases": counts["token_releases"], + } + + # 按 worker type 分类统计(prefill/decode/mixed) + type_summary = defaultdict( + lambda: { + "counter_selects": 0, + "counter_releases": 0, + "token_selects": 0, + "token_releases": 0, } + ) + for s in selects: + s_type = _normalize_worker_type(s.get("type")) + type_summary[s_type]["counter_selects"] += 1 + if s_type in ("prefill", "mixed"): + type_summary[s_type]["token_selects"] += 1 + for r in releases: + r_type = _normalize_worker_type(str(r.get("type", "")).replace("_tokens", "")) + if str(r.get("type", "")).endswith("_tokens"): + type_summary[r_type]["token_releases"] += 1 + else: + type_summary[r_type]["counter_releases"] += 1 return { "matched": matched, @@ -707,6 +750,7 @@ def match_select_release(lines, fallback_window_s=120): "with_alt_id": with_alt_id, "without_any_id": without_any_id, }, + "type_summary": dict(type_summary), } diff --git a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/troubleshoot.py b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/troubleshoot.py index 30b9df0f443..a818d31150f 100644 --- a/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/troubleshoot.py +++ b/fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/troubleshoot.py @@ -144,10 +144,11 @@ def format_full_report(results, status, status_reason): report_text: 主报告文本(总结 + 可视化) details: dict 包含需要拆分到独立文件的详情数据 - 'health_events': str 或 None + - 'load_select_release': str 或 None - 'trace_files': {trace_id: text} 或 {} """ parts = [] - details = {"health_events": None, "trace_files": {}} + details = {"health_events": None, "load_select_release": None, "trace_files": {}} # 状态行 parts.append(f"STATUS: {status} — {status_reason}") @@ -168,7 +169,10 @@ def format_full_report(results, status, status_reason): details["health_events"] = detail if "load" in results: - parts.append(format_load_report(results["load"])) + summary, detail = format_load_report(results["load"]) + parts.append(summary) + if detail: + details["load_select_release"] = detail if "cache" in results: parts.append(format_cache_report(results["cache"])) @@ -208,6 +212,11 @@ def save_detailed_report(report_text, output_dir, details=None): with open(health_path, "w", encoding="utf-8") as f: f.write(details["health_events"]) + if details.get("load_select_release"): + load_path = os.path.join(detail_dir, "load_select_release.md") + with open(load_path, "w", encoding="utf-8") as f: + f.write(details["load_select_release"]) + for trace_id, trace_text in details.get("trace_files", {}).items(): safe_id = trace_id.replace("/", "_") trace_path = os.path.join(detail_dir, f"trace_{safe_id}.md")