Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@
| `Failed to select worker pair: {err}` | HIGH | FD 后端 | 请求返回 502 |
| `Failed to build disaggregate_info: {err}` | HIGH | Router | 请求返回 500 |
| `Failed to encode modified request: {err}` | HIGH | Router | 请求返回 500 |
| `Failed to read YAML file config/register.yaml: {err}` | LOW | Router | 启动时未找到可选配置文件(若未使用 register.yaml 可忽略) |
| `Failed to select worker: {err}` | HIGH | FD 后端 | 请求返回 502 |
| `Failed to connect to backend service: {err}` | HIGH | FD 后端 | 请求返回 502 |
| `Request failed (attempt {n}/{max}): {err}` | MEDIUM | FD 后端 | 重试中 |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -233,6 +233,17 @@ PD(Prefill/Decode 分离)模式下,`completions.go` 产生的 `[prefill]`

---

## Select/Release 日志细节(与代码一致)

- `select worker (prefill): <url>, tokens: <n>`
- `select worker (decode|mixed): <url>, count: <n>`
- `release worker: <url>, count: <n>`(request counter 释放)
- `release prefill tokens: <url>, tokens: <n>`(token counter 释放;可能来自 prefill 或 mixed 请求路径)

重点:release 只有上面这两种。`release worker` 不带 worker type,`release prefill tokens` 的文本也不能直接断定是 prefill(mixed 也可能调用)。因此按 `prefill/decode/mixed` 统计时,需要从 select 侧做归类;确实无法归类时才记为 `unknown`。

---

## 使用脚本工具

各 skill 的脚本位于各自的 `scripts/` 目录下,自动处理上述所有日志解析和计算。
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
### 简洁版(终端输出)

- 第一行:`STATUS: HEALTHY / DEGRADED / CRITICAL — 简要说明`
- 状态定义:`HEALTHY`=无明显异常;`DEGRADED`=服务可用但性能/稳定性下降(需关注);`CRITICAL`=服务不可用或高风险故障
- 按三层分类(Router / FD 后端 / 客户端)
- 每个问题一行摘要 + 关键指标
- 末尾提示详细版文件路径
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
("counter already zero", "Router"),
("tokenizer failed", "Router"),
("Instance {url} role is unknown", "Router"),
("Failed to read YAML file config/register.yaml", "Router"),
# 客户端
("Invalid request body", "客户端"),
("Invalid JSON format", "客户端"),
Expand Down Expand Up @@ -282,6 +283,14 @@ def format_errors_report(result):
render_table(table_data, columns=["模板", "数量", "占比", "级别", "来源层"], right_align={"数量", "占比"})
)
sections.append("")
yaml_missing_count = sum(
e["count"] for e in result["error_top_n"] if "Failed to read YAML file config/register.yaml" in e["template"]
)
if yaml_missing_count > 0:
sections.append(
f" ℹ `Failed to read YAML file config/register.yaml` 出现 {yaml_missing_count} 次:若未启用该配置文件,可忽略。"
)
sections.append("")

# 状态码分布
if result["status_code_dist"]:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,11 @@ def format_load_report(result):
if result["diagnoses"]:
sections.append("### 诊断")
sections.append("")
for d in result["diagnoses"]:
max_diag_in_summary = 8
for d in result["diagnoses"][:max_diag_in_summary]:
sections.append(f' [{d["severity"]}] [{d["source_layer"]}] {d["message"]}')
if len(result["diagnoses"]) > max_diag_in_summary:
sections.append(f' ... 其余 {len(result["diagnoses"]) - max_diag_in_summary} 项见 detail 报告')
sections.append("")
detail_sections.append("## 诊断")
detail_sections.append("")
Expand All @@ -39,6 +42,7 @@ def format_load_report(result):
if ls:
sections.append("### 负载概览 (total_running)")
sections.append("")
sections.append(" 说明: stats 采样来自 `[stats]` 周期日志(通常每 5s 一条),用于观察当前并发与负载变化趋势。")
sections.append(
f' mean={ls.get("mean",0)} p50={ls.get("p50",0)} p90={ls.get("p90",0)} '
f'p99={ls.get("p99",0)} max={ls.get("max",0)} stddev={ls.get("stddev",0)}'
Expand Down Expand Up @@ -108,6 +112,9 @@ def format_load_report(result):
sections.append(render_table(type_rows, columns=["type", "counter(S/R)", "token(S/R)"]))
sections.append("")
sections.append(" 说明: prefill/mixed 的 token-select 同时表示 request counter + token counter 增加;decode 仅 request counter。")
sections.append(" 说明: token-release 由同 worker 邻近 select 推断到 prefill/mixed,不直接依赖 `release prefill tokens` 文本。")
if type_summary.get("unknown"):
sections.append(" 说明: unknown 表示日志里缺少 worker type,且无法从邻近 select/release 关系推断。")
sections.append("")
detail_sections.append("## 按类型统计")
detail_sections.append("")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -227,9 +227,10 @@ def render_table(data, columns=None, right_align=None):
w = col_widths[col]
if col in right_align:
header_parts.append(f" {col:>{w}} ")
sep_parts.append("-" * (w + 1) + ":")
else:
header_parts.append(f" {col:<{w}} ")
sep_parts.append("-" * (w + 2))
sep_parts.append(":" + "-" * (w + 1))

lines = []
lines.append("|" + "|".join(header_parts) + "|")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -501,6 +501,77 @@ def _normalize_worker_type(worker_type):
return "unknown"


def _infer_release_worker_type(release, selects, fallback_window_s=120):
"""为未显式标注 type 的 release 近似推断 worker type。

优先级:
1) 同 worker、时间上最近且不晚于 release 的 select type
2) 若无可解析时间戳,则使用同 worker 的最后一个 select type
3) 推断失败返回 unknown
"""
worker = release.get("worker")
if not worker:
return "unknown"

r_ts = _parse_ts_safe(release.get("ts"))
candidates = [s for s in selects if s.get("worker") == worker]
if not candidates:
return "unknown"

if r_ts:
best = None
best_delta = None
for s in candidates:
s_ts = _parse_ts_safe(s.get("ts"))
if not s_ts:
continue
delta = (r_ts - s_ts).total_seconds()
if delta < 0 or delta > fallback_window_s:
continue
if best_delta is None or delta < best_delta:
best = s
best_delta = delta
if best is not None:
return _normalize_worker_type(best.get("type"))

# 回退:按出现顺序取同 worker 的最近 select
return _normalize_worker_type(candidates[-1].get("type"))


def _infer_token_release_worker_type(release, selects, fallback_window_s=120):
"""为 token release 推断 worker type(prefill/mixed)。

注意:日志文本通常固定为 `release prefill tokens`,即使 mixed 也可能走这条日志。
因此 token release 的类型优先依据同 worker 的邻近 select 推断。
"""
worker = release.get("worker")
if not worker:
return "unknown"

r_ts = _parse_ts_safe(release.get("ts"))
candidates = [s for s in selects if s.get("worker") == worker and _normalize_worker_type(s.get("type")) in ("prefill", "mixed")]
if not candidates:
return "unknown"

if r_ts:
best = None
best_delta = None
for s in candidates:
s_ts = _parse_ts_safe(s.get("ts"))
if not s_ts:
continue
delta = (r_ts - s_ts).total_seconds()
if delta < 0 or delta > fallback_window_s:
continue
if best_delta is None or delta < best_delta:
best = s
best_delta = delta
if best is not None:
return _normalize_worker_type(best.get("type"))

return _normalize_worker_type(candidates[-1].get("type"))


def match_select_release(lines, fallback_window_s=120):
"""匹配 select/release worker 事件对。

Expand Down Expand Up @@ -536,12 +607,14 @@ def match_select_release(lines, fallback_window_s=120):
# Token-bearing release
trm = RELEASE_TOKENS_RE.search(line)
if trm:
token_type = trm.group(1) or "prefill"
token_type = trm.group(1)
releases.append(
{
"ts": ts,
"worker": trm.group(2),
"type": f'{_normalize_worker_type(token_type)}_tokens',
# 不直接信任日志里的 token type 文本("release prefill tokens" 也可能来自 mixed)
"type": "unknown_tokens",
"raw_token_type": token_type or "",
"tags": tags,
"tokens": int(trm.group(3)),
"line": line_no,
Expand Down Expand Up @@ -716,7 +789,24 @@ def match_select_release(lines, fallback_window_s=120):
"token_releases": counts["token_releases"],
}

# 按 worker type 分类统计(prefill/decode/mixed)
# 为未显式标注 type 的 release 推断 worker type(避免大量 unknown)
inferred_release_types = {}
for i, r in enumerate(releases):
r_type_raw = str(r.get("type", ""))
if r_type_raw.endswith("_tokens"):
base_t = _normalize_worker_type(r_type_raw.replace("_tokens", ""))
if base_t == "unknown":
# token release 的 worker type 由同 worker 邻近 select 推断(prefill/mixed)
base_t = _infer_token_release_worker_type(r, selects, fallback_window_s=fallback_window_s)
inferred_release_types[i] = f"{base_t}_tokens"
continue
base_t = _normalize_worker_type(r_type_raw)
if base_t != "unknown":
inferred_release_types[i] = base_t
continue
inferred_release_types[i] = _infer_release_worker_type(r, selects, fallback_window_s=fallback_window_s)

# 按 worker type 分类统计(prefill/decode/mixed,必要时保留 unknown)
type_summary = defaultdict(
lambda: {
"counter_selects": 0,
Expand All @@ -730,9 +820,10 @@ def match_select_release(lines, fallback_window_s=120):
type_summary[s_type]["counter_selects"] += 1
if s_type in ("prefill", "mixed"):
type_summary[s_type]["token_selects"] += 1
for r in releases:
r_type = _normalize_worker_type(str(r.get("type", "")).replace("_tokens", ""))
if str(r.get("type", "")).endswith("_tokens"):
for i, r in enumerate(releases):
inferred = inferred_release_types.get(i, _normalize_worker_type(str(r.get("type", ""))))
r_type = _normalize_worker_type(str(inferred).replace("_tokens", ""))
if str(inferred).endswith("_tokens"):
type_summary[r_type]["token_releases"] += 1
else:
type_summary[r_type]["counter_releases"] += 1
Expand Down Expand Up @@ -949,6 +1040,16 @@ def check(name, got, expected):
"dial tcp {ip:port}: connection refused",
)

print("\n=== Testing match_select_release (token release type inference) ===")
sample_lines = [
"[INFO] 2026/04/12 10:00:00 logger.go:1: [request_id:r1] select worker (mixed): http://10.0.0.1:9965, count: 1",
"[INFO] 2026/04/12 10:00:01 logger.go:1: [request_id:r1] release prefill tokens: http://10.0.0.1:9965, tokens: 10",
"[INFO] 2026/04/12 10:00:02 logger.go:1: [request_id:r1] release worker: http://10.0.0.1:9965, count: 0",
]
msr = match_select_release(sample_lines)
check("mixed token_releases inferred", msr["type_summary"].get("mixed", {}).get("token_releases", 0), 1)
check("prefill token_releases remains 0", msr["type_summary"].get("prefill", {}).get("token_releases", 0), 0)

print(f'\n{"=" * 40}')
print(f"Results: {passed} passed, {failed} failed")
if failed:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,20 @@ def determine_status(results):
reasons.append(d["message"])

if reasons:
return "DEGRADED", ", ".join(reasons)
# 去重并限制长度,避免状态行过长难读
deduped = []
seen = set()
for r in reasons:
if r not in seen:
deduped.append(r)
seen.add(r)
max_reasons = 4
shown = deduped[:max_reasons]
extra = len(deduped) - len(shown)
summary = ";".join(shown)
if extra > 0:
summary += f";另有 {extra} 项诊断见各维度 detail 报告"
return "DEGRADED", summary

if not results:
return "HEALTHY", "无分析数据"
Expand All @@ -152,6 +165,9 @@ def format_full_report(results, status, status_reason):

# 状态行
parts.append(f"STATUS: {status} — {status_reason}")
parts.append(
"状态定义: HEALTHY=无明显异常;DEGRADED=服务可用但存在性能/稳定性问题(需关注);CRITICAL=服务不可用或高风险故障。"
)
parts.append("=" * 60)
parts.append("")

Expand Down
Loading