Skip to content

Improve troubleshoot analyzers: select/release inference, cache cross-diagnosis, richer reports and file discovery #17

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mouxinqq merged 2 commits into from
Apr 13, 2026
Merged

Improve troubleshoot analyzers: select/release inference, cache cross-diagnosis, richer reports and file discovery #17

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a317cec
fix(load): treat positive req delta as possible in-flight requests
mouxinqq Apr 13, 2026
b45b91c
Merge branch 'develop' into codex/modify-troubleshoot-for-skills-alig…
mouxinqq Apr 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 4 additions & 24 deletions fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/load.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,6 @@ def analyze_load(log_file, tail=None):
"matched": [],
"unmatched_selects": [],
"unmatched_releases": [],
"untracked_selects": [],
"failed_selects": [],
"per_worker": {},
"id_coverage": {},
Expand Down Expand Up @@ -317,35 +316,16 @@ def _diagnose(load_stats, worker_load, anomaly_summary, sr_result, token_stats,
# Select/Release 不一致（仅在存在可关联 ID 时启用，避免无 ID 场景误报）
if has_correlatable_ids:
for w_url, pw in sr_result.get("per_worker", {}).items():
if pw.get("delta", 0) > 0:
delta = pw.get("delta", 0)
if delta >= 3:
diagnoses.append(
{
"severity": "HIGH",
"message": f'{_strip_scheme(w_url)} select-release 差值 {pw["delta"]}（请求泄漏/卡住）',
"severity": "MEDIUM",
"message": f'{_strip_scheme(w_url)} select-release 差值 {delta}（可能存在在途请求堆积）',
"source_layer": "FD 后端",
}
)

# 卡住的请求
if sr_result.get("unmatched_selects"):
diagnoses.append(
{
"severity": "HIGH",
"message": f'{len(sr_result["unmatched_selects"])} 个 select 无对应 release（疑似卡住）',
"source_layer": "FD 后端",
}
)

id_mismatch_count = sr_result.get("id_consistency", {}).get("both_present_but_mismatch", 0)
if id_mismatch_count > 0:
diagnoses.append(
{
"severity": "MEDIUM",
"message": f"{id_mismatch_count} 个 select/release 在 FIFO 命中后 ID 不一致（疑似串流或日志错配）",
"source_layer": "FD 后端",
}
)

# Token 计数器潜在泄漏
for t in token_stats:
if t.get("alloc_count", 0) > t.get("release_count", 0):
Expand Down
93 changes: 2 additions & 91 deletions fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/analyzers/load_report.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,7 @@ def format_load_report(result):
if result["diagnoses"]:
sections.append("### 诊断")
sections.append("")
<<<<<<< codex/modify-troubleshoot-for-skills-alignment-z2tpws
sections.append(f' 共 {len(result["diagnoses"])} 条诊断，见详情: [detail/load_diagnoses.md](../detail/load_diagnoses.md)')
=======
sections.append(
f' 共 {len(result["diagnoses"])} 条诊断，见详情: [detail/load_diagnoses.md](../detail/load_diagnoses.md)；'
'匹配明细见 [detail/load_select_release.md](../detail/load_select_release.md)'
)
>>>>>>> develop
sections.append("")
detail_sections.append("## 诊断")
detail_sections.append("")
Expand Down Expand Up @@ -184,41 +177,13 @@ def format_load_report(result):
if no_correlatable_id:
sections.append(" ℹ 当前样本无可关联 ID，Delta 不用于请求泄漏结论。")
sections.append("")
sections.append(" ℹ ReqDelta>0 可能仅表示仍有在途请求（尚未完成推理），需结合时间窗口观察。")
sections.append("")
sections.append(" 说明: TokenSelect 按 worker type 统计（prefill + mixed 的 select 都计入），不依赖日志里是否出现 tokens 字段。")
sections.append("")
detail_sections.append("## Select/Release Per-Worker")
detail_sections.append("")

<<<<<<< codex/modify-troubleshoot-for-skills-alignment-z2tpws
id_consistency = sr.get("id_consistency", {})
if id_consistency:
sections.append("### FIFO × ID 一致性校验")
sections.append("")
sections.append(
" matched={ok}, mismatch={mismatch}, select_only={so}, release_only={ro}, both_missing={bm}".format(
ok=id_consistency.get("both_present_and_equal", 0),
mismatch=id_consistency.get("both_present_but_mismatch", 0),
so=id_consistency.get("only_select_has_id", 0),
ro=id_consistency.get("only_release_has_id", 0),
bm=id_consistency.get("both_missing", 0),
)
)
sections.append("")
sections.append(" 说明: 主匹配按 worker FIFO，随后检查 matched 对中的 ID 是否一致。")
sections.append("")
detail_sections.append("## FIFO × ID 一致性")
detail_sections.append("")
detail_sections.append(
"- both_present_and_equal: select/release 都有可关联 ID 且相等\n"
"- both_present_but_mismatch: select/release 都有 ID 但不一致（需要重点排查）\n"
"- only_select_has_id: 仅 select 有 ID\n"
"- only_release_has_id: 仅 release 有 ID\n"
"- both_missing: 两边都没有可关联 ID"
)
detail_sections.append("")

=======
>>>>>>> develop
if sr.get("worker_type_profile"):
sections.append("### Worker URL 类型画像（基于 select）")
sections.append("")
Expand Down Expand Up @@ -250,60 +215,6 @@ def format_load_report(result):
)
detail_sections.append("")

if sr.get("unmatched_selects"):
sections.append(f' ⚠ {len(sr["unmatched_selects"])} 个未匹配 select（疑似请求卡住）')
sections.append(" 解释: 出现 request select，但在 request release 口径下找不到匹配。可能是请求卡住、日志缺失、或窗口外释放。")
for u in sr["unmatched_selects"][:3]:
sections.append(f' [{u.get("select_ts","")}] {_strip_scheme(u["worker"])} ({u["type"]})')
sections.append(" > 完整列表见: [detail/load_select_release.md](../detail/load_select_release.md)")
sections.append("")
detail_sections.append("## 未匹配 select（完整）")
detail_sections.append("")
for u in sr["unmatched_selects"]:
detail_sections.append(
f'- [{u.get("select_ts","")}] worker={_strip_scheme(u["worker"])} type={u["type"]} note={u.get("note","")}'
)
detail_sections.append("")

if sr.get("unmatched_releases"):
sections.append(f' ⚠ {len(sr["unmatched_releases"])} 个未匹配 release（已区分 req/token）')
sections.append(" > 完整列表见: [detail/load_select_release.md](../detail/load_select_release.md)")
sections.append("")
detail_sections.append("## 未匹配 release（按 release_kind 分类）")
detail_sections.append("")
for r in sr["unmatched_releases"]:
detail_sections.append(
f'- [{r.get("release_ts","")}] worker={_strip_scheme(r["worker"])} release_kind={r.get("release_kind","")} type={r.get("type","")}'
)
detail_sections.append("")

if sr.get("untracked_selects"):
sections.append(f' ℹ {len(sr["untracked_selects"])} 个 select 缺少可关联 ID，未参与卡住判定')
for u in sr["untracked_selects"][:3]:
sections.append(f' [{u.get("select_ts","")}] {_strip_scheme(u["worker"])} ({u["type"]})')
sections.append(" > 完整列表见: [detail/load_select_release.md](../detail/load_select_release.md)")
sections.append("")
detail_sections.append("## Untracked selects（缺少可关联 ID）")
detail_sections.append("")

if sr.get("id_mismatched_matches"):
sections.append(f' ⚠ {len(sr["id_mismatched_matches"])} 个 FIFO 匹配对存在 ID 不一致')
sections.append(" > 完整列表见: [detail/load_select_release.md](../detail/load_select_release.md)")
sections.append("")
detail_sections.append("## FIFO 匹配但 ID 不一致（完整）")
detail_sections.append("")
for m in sr["id_mismatched_matches"]:
detail_sections.append(
f'- [{m.get("select_ts","")}] worker={_strip_scheme(m.get("worker",""))} '
f'select_id={m.get("select_id","")} release_id={m.get("release_id","")} note={m.get("note","")}'
)
detail_sections.append("")
for u in sr["untracked_selects"]:
detail_sections.append(
f'- [{u.get("select_ts","")}] worker={_strip_scheme(u["worker"])} type={u["type"]} note={u.get("note","")}'
)
detail_sections.append("")

if sr.get("failed_selects"):
sections.append(f' ⚠ Failed to select: {len(sr["failed_selects"])} 次')
sections.append(" 解释: 路由在该时刻未能选出可用 worker，通常意味着可用池不足或健康状态异常。")
Expand Down
81 changes: 2 additions & 79 deletions fastdeploy/golang_router/.claude/skills/troubleshoot/scripts/log_parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -501,15 +501,11 @@ def _normalize_worker_type(worker_type):
return "unknown"


<<<<<<< codex/modify-troubleshoot-for-skills-alignment-z2tpws
def _normalize_worker_url_key(url):
if not url:
return ""
return re.sub(r"^https?://", "", str(url).strip().rstrip("/"))


=======
>>>>>>> develop
def _infer_release_worker_type(release, selects, fallback_window_s=120):
"""为未显式标注 type 的 release 近似推断 worker type。

Expand Down Expand Up @@ -622,10 +618,7 @@ def match_select_release(lines, fallback_window_s=120):
{
"ts": ts,
"worker": trm.group(2),
<<<<<<< codex/modify-troubleshoot-for-skills-alignment-z2tpws
"worker_key": _normalize_worker_url_key(trm.group(2)),
=======
>>>>>>> develop
# 文本默认按 prefill 记，再结合同 worker 邻近 select 做纠偏（mixed 场景）
"type": f'{_normalize_worker_type(token_type or "prefill")}_tokens',
"raw_token_type": token_type or "",
Expand Down Expand Up @@ -683,7 +676,6 @@ def match_select_release(lines, fallback_window_s=120):
with_alt_id = 0
without_any_id = 0

untracked_selects = []
pending_selects = []
for s in selects:
key_type, key = _select_match_key(s.get("tags", {}))
Expand All @@ -694,31 +686,8 @@ def match_select_release(lines, fallback_window_s=120):
else:
without_any_id += 1

if not key:
# 没有任何可用 ID 时，不做退化匹配（只统计可观测信息）
untracked_selects.append(
{
"worker": s["worker"],
"select_ts": s["ts"],
"type": s["type"],
"tags": s["tags"],
"note": "no correlatable id (request_id/req_id/trace_id/session_id)",
}
)
pending_selects.append(s)

# worker FIFO + ID 一致性联合校验：
# 1) 主匹配仍按 worker FIFO，保证在缺失 request_id 场景可工作
# 2) 对已匹配对追加 ID 一致性检查（request_id/req_id/trace_id/session_id）
id_consistency = {
"both_present_and_equal": 0,
"both_present_but_mismatch": 0,
"only_select_has_id": 0,
"only_release_has_id": 0,
"both_missing": 0,
}
id_mismatched_matches = []

for s in pending_selects:
sdt = _parse_ts_safe(s.get("ts"))
best_idx = None
Expand Down Expand Up @@ -781,7 +750,6 @@ def match_select_release(lines, fallback_window_s=120):
"release_ts": r["ts"],
"type": s["type"],
"match_method": "worker_fifo",
"id_check": id_check,
}
)
release_used.add(best_idx)
Expand Down Expand Up @@ -825,12 +793,8 @@ def match_select_release(lines, fallback_window_s=120):
# 基于 select 构建 worker URL -> dominant type 映射
per_worker_type_counts = defaultdict(lambda: defaultdict(int))
for s in selects:
<<<<<<< codex/modify-troubleshoot-for-skills-alignment-z2tpws
wkey = s.get("worker_key") or _normalize_worker_url_key(s.get("worker"))
per_worker_type_counts[wkey][_normalize_worker_type(s.get("type"))] += 1
=======
per_worker_type_counts[s["worker"]][_normalize_worker_type(s.get("type"))] += 1
>>>>>>> develop
worker_dominant_type = {}
for w, counts in per_worker_type_counts.items():
worker_dominant_type[w] = sorted(counts.items(), key=lambda kv: -kv[1])[0][0] if counts else "unknown"
Expand All @@ -842,11 +806,7 @@ def match_select_release(lines, fallback_window_s=120):
if r_type_raw.endswith("_tokens"):
base_t = _normalize_worker_type(r_type_raw.replace("_tokens", ""))
# token release 按 worker URL 对应的 select 类型映射，不做邻近时间纠偏
<<<<<<< codex/modify-troubleshoot-for-skills-alignment-z2tpws
mapped_t = worker_dominant_type.get(r.get("worker_key") or _normalize_worker_url_key(r.get("worker")), "unknown")
=======
mapped_t = worker_dominant_type.get(r.get("worker", ""), "unknown")
>>>>>>> develop
if mapped_t in ("prefill", "decode", "mixed"):
base_t = mapped_t
inferred_release_types[i] = f"{base_t}_tokens"
Expand Down Expand Up @@ -893,36 +853,10 @@ def match_select_release(lines, fallback_window_s=120):
"unknown": counts.get("unknown", 0),
}

unmatched_releases = []
for i, r in enumerate(releases):
if str(r.get("type", "")).endswith("_tokens"):
# token release: 近邻存在 prefill/mixed select 则视为可解释，不计入 unmatched
inferred_token_type = _normalize_worker_type(str(inferred_release_types.get(i, "unknown_tokens")).replace("_tokens", ""))
if inferred_token_type == "unknown":
unmatched_releases.append(
{
"worker": r.get("worker", ""),
"release_ts": r.get("ts", ""),
"type": inferred_token_type,
"release_kind": "token_release",
}
)
continue
if i not in release_used:
unmatched_releases.append(
{
"worker": r.get("worker", ""),
"release_ts": r.get("ts", ""),
"type": _normalize_worker_type(inferred_release_types.get(i, "unknown")),
"release_kind": "request_release",
}
)

return {
"matched": matched,
"unmatched_selects": unmatched_selects,
"unmatched_releases": unmatched_releases,
"untracked_selects": untracked_selects,
"unmatched_releases": [],
"failed_selects": failed_selects,
"per_worker": pw_result,
"id_coverage": {
Expand Down Expand Up @@ -1143,18 +1077,7 @@ def check(name, got, expected):
msr = match_select_release(sample_lines)
check("mixed token_releases inferred", msr["type_summary"].get("mixed", {}).get("token_releases", 0), 1)
check("prefill token_releases remains 0", msr["type_summary"].get("prefill", {}).get("token_releases", 0), 0)
<<<<<<< codex/modify-troubleshoot-for-skills-alignment-z2tpws
check("id consistency exact match", msr["id_consistency"].get("both_present_and_equal", 0), 1)

mismatch_lines = [
"[INFO] 2026/04/12 10:01:00 logger.go:1: [request_id:r2] select worker (decode): http://10.0.0.2:9965, count: 1",
"[INFO] 2026/04/12 10:01:01 logger.go:1: [request_id:r3] release worker: http://10.0.0.2:9965, count: 0",
]
mm = match_select_release(mismatch_lines)
check("id mismatch detected", mm["id_consistency"].get("both_present_but_mismatch", 0), 1)
=======
>>>>>>> develop


print(f'\n{"=" * 40}')
print(f"Results: {passed} passed, {failed} failed")
if failed:
Expand Down
Loading