diff --git a/test.py b/test.py index 40a9ec9..4e3328f 100644 --- a/test.py +++ b/test.py @@ -5,17 +5,19 @@ Feature: Explicitly logs and plots WSR (Mbps) to verify convergence and trade-offs. """ -import os -import math -import random -import numpy as np -import torch -import torch.nn as nn -import torch.optim as optim -import torch.nn.functional as F -from dataclasses import dataclass -from typing import List, Tuple, Dict -from scipy.special import gammaincinv, erfcinv +import os +import math +import random +import json +import csv +import numpy as np +import torch +import torch.nn as nn +import torch.optim as optim +import torch.nn.functional as F +from dataclasses import dataclass +from typing import List, Tuple, Dict +from scipy.special import gammaincinv, erfcinv # ============================================================================= # 1) Utilities & Physics @@ -207,11 +209,11 @@ def reset(self): self._sample_positions() return np.concatenate([self.Beta.reshape(-1), np.array([self.N_ant], dtype=np.float32)], axis=0).astype(np.float32) - def _decode_action(self, action): - infeasible = 0.0 - ServingAP = [] - for k in range(self.K): - chosen = action[k].astype(np.int64) + def _decode_action(self, action): + infeasible = 0.0 + ServingAP = [] + for k in range(self.K): + chosen = action[k].astype(np.int64) if np.any(chosen < 0) or np.any(chosen >= self.M_ap): infeasible += 1.0 chosen = np.clip(chosen, 0, self.M_ap - 1) @@ -222,90 +224,79 @@ def _decode_action(self, action): if chosen.size == 0: infeasible += 1.0 chosen = np.array([int(np.argmax(self.Beta[:, k]))], dtype=np.int64) - if chosen.size > self.S: - chosen = chosen[: self.S] - ServingAP.append(chosen) - return ServingAP, infeasible - - def step(self, action): - ServingAP, infeasible = self._decode_action(action) - - pilots = wgf_pilot_assignment(self.Beta.astype(np.float64), ServingAP, self.lp) - C_est = compute_C_est(self.Beta.astype(np.float64), pilots, self.lp, self.rho_p) - Eta = compute_Eta(C_est, ServingAP) - mu2, sigma2, C_val = compute_user_statistics_vectorized( - self.Beta.astype(np.float64), C_est, Eta, ServingAP, pilots, self.N_ant, self.lp, self.rho_d - ) - - safe_mask = (sigma2 > 1e-9) & (mu2 > 1e-9) - alpha2 = np.zeros_like(mu2); beta2 = np.zeros_like(mu2) - alpha2[safe_mask] = (mu2[safe_mask] ** 2) / sigma2[safe_mask] - beta2[safe_mask] = sigma2[safe_mask] / mu2[safe_mask] - alpha2[~safe_mask] = 1e-2; beta2[~safe_mask] = 1e-2 - - delta_fbl = math.sqrt(1.0 / self.n) * qinv(self.epsilon_dec) - p = (np.arange(1, self.num_quantiles + 1) - 0.5) / self.num_quantiles - I_mat = gammaincinv(alpha2[:, None], p[None, :]) * beta2[:, None] - R_mat = np.maximum(0.0, self.n * np.log2(1.0 + C_val[:, None] / I_mat) - self.n * delta_fbl) - S_bar = np.mean(R_mat, axis=1) - - mu_arrival_np = self.rho_target * S_bar - R_tensor = torch.from_numpy(R_mat).float().to(self.device) - mu_arr = torch.from_numpy(mu_arrival_np).float().to(self.device) - theta_tensor = solve_theta_jit(mu_arr, R_tensor) - - neg_theta_R = -theta_tensor.unsqueeze(-1) * R_tensor - max_val, _ = torch.max(neg_theta_R, dim=-1, keepdim=True) - log_sum = max_val + torch.log(torch.sum(torch.exp(neg_theta_R - max_val), dim=-1, keepdim=True)) - log_Ms = log_sum.squeeze(-1) - math.log(float(self.num_quantiles)) - - Ks = -log_Ms / (theta_tensor + 1e-10) - dvp = torch.exp(-theta_tensor * Ks * float(self.d_th_frames)).clamp(0, 1).cpu().numpy() - - wsr = float(np.sum(self.w_k * (S_bar * (self.B / self.n)))) - - # === 修改后的平滑奖励逻辑 === - eps = float(self.eps_target_sys) - worst_dvp = float(np.max(dvp)) - safe_dvp = max(worst_dvp, 1e-12) # 防止 log 报错 - - # 归一化 WSR (0.0 ~ 1.2 左右) - r_wsr = wsr / self.R_ref_bps - - # 计算违背程度 (log space) - # 如果 DVP = 1e-2 (目标), log_term = 0 - # 如果 DVP = 1e-1, log_term = 1.0 - # 如果 DVP = 1e-3, log_term = -1.0 - log_term = math.log10(safe_dvp / eps) - - # 惩罚系数 - lambda_p = 1.0 - - # 组合奖励: - # 1. 基础 WSR 奖励 - # 2. 如果 DVP > eps (log_term > 0),施加惩罚 - # 3. 如果 DVP < eps (log_term < 0),给予微小奖励或不惩罚 - # 使用 Softplus 或 ReLU 变体来平滑 - - if log_term > 0: - # 越界了:线性扣分,不要太狠,给它改过的机会 - # 之前的 5.0 太大了,改成 2.0 甚至 1.0 - penalty = 2.0 * log_term - reward = r_wsr - penalty - else: - # 安全区域: - # 稍微给一点点正反馈,鼓励更低的时延,但主要还是看 WSR - reward = r_wsr - 0.1 * log_term # 注意 log_term 是负数,这里变成加分 - - reward -= (self.infeasible_penalty * infeasible) - - next_state = self.reset() - info = { - 'wsr_bps': wsr, - 'worst_dvp': worst_dvp, - 'reward': float(reward) - } - return next_state, float(reward), True, info + if chosen.size > self.S: + chosen = chosen[: self.S] + ServingAP.append(chosen) + return ServingAP, infeasible + + def _evaluate_action(self, ServingAP: List[np.ndarray], infeasible: float) -> Dict[str, float]: + pilots = wgf_pilot_assignment(self.Beta.astype(np.float64), ServingAP, self.lp) + C_est = compute_C_est(self.Beta.astype(np.float64), pilots, self.lp, self.rho_p) + Eta = compute_Eta(C_est, ServingAP) + mu2, sigma2, C_val = compute_user_statistics_vectorized( + self.Beta.astype(np.float64), C_est, Eta, ServingAP, pilots, self.N_ant, self.lp, self.rho_d + ) + + safe_mask = (sigma2 > 1e-9) & (mu2 > 1e-9) + alpha2 = np.zeros_like(mu2); beta2 = np.zeros_like(mu2) + alpha2[safe_mask] = (mu2[safe_mask] ** 2) / sigma2[safe_mask] + beta2[safe_mask] = sigma2[safe_mask] / mu2[safe_mask] + alpha2[~safe_mask] = 1e-2; beta2[~safe_mask] = 1e-2 + + delta_fbl = math.sqrt(1.0 / self.n) * qinv(self.epsilon_dec) + p = (np.arange(1, self.num_quantiles + 1) - 0.5) / self.num_quantiles + I_mat = gammaincinv(alpha2[:, None], p[None, :]) * beta2[:, None] + R_mat = np.maximum(0.0, self.n * np.log2(1.0 + C_val[:, None] / I_mat) - self.n * delta_fbl) + S_bar = np.mean(R_mat, axis=1) + + mu_arrival_np = self.rho_target * S_bar + R_tensor = torch.from_numpy(R_mat).float().to(self.device) + mu_arr = torch.from_numpy(mu_arrival_np).float().to(self.device) + theta_tensor = solve_theta_jit(mu_arr, R_tensor) + + neg_theta_R = -theta_tensor.unsqueeze(-1) * R_tensor + max_val, _ = torch.max(neg_theta_R, dim=-1, keepdim=True) + log_sum = max_val + torch.log(torch.sum(torch.exp(neg_theta_R - max_val), dim=-1, keepdim=True)) + log_Ms = log_sum.squeeze(-1) - math.log(float(self.num_quantiles)) + + Ks = -log_Ms / (theta_tensor + 1e-10) + dvp = torch.exp(-theta_tensor * Ks * float(self.d_th_frames)).clamp(0, 1).cpu().numpy() + + wsr = float(np.sum(self.w_k * (S_bar * (self.B / self.n)))) + + eps = float(self.eps_target_sys) + worst_dvp = float(np.max(dvp)) + safe_dvp = max(worst_dvp, 1e-12) + + r_wsr = wsr / self.R_ref_bps + log_term = math.log10(safe_dvp / eps) + + if log_term > 0: + penalty = 2.0 * log_term + reward = r_wsr - penalty + else: + reward = r_wsr - 0.1 * log_term + + reward -= (self.infeasible_penalty * infeasible) + + return { + 'wsr_bps': wsr, + 'worst_dvp': worst_dvp, + 'reward': float(reward), + 'constraint_satisfied': float(worst_dvp <= eps) + } + + def step(self, action): + ServingAP, infeasible = self._decode_action(action) + metrics = self._evaluate_action(ServingAP, infeasible) + + next_state = self.reset() + info = { + 'wsr_bps': metrics['wsr_bps'], + 'worst_dvp': metrics['worst_dvp'], + 'reward': metrics['reward'] + } + return next_state, float(metrics['reward']), True, info # ============================================================================= # 3) GNN & Agent @@ -340,16 +331,67 @@ def __init__(self, obs_dim: int, hidden: int = 256): self.net = nn.Sequential(nn.Linear(obs_dim, hidden), nn.ReLU(), nn.Linear(hidden, hidden), nn.ReLU(), nn.Linear(hidden, 1)) def forward(self, obs): return self.net(obs).squeeze(-1) -def plackett_luce_sample(logits: torch.Tensor, S: int) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - B, K, M = logits.shape - u = torch.rand_like(logits) - g = -torch.log(-torch.log(u + 1e-12) + 1e-12) - scores = logits + g - topk = torch.topk(scores, k=S, dim=-1) - act_idx = topk.indices - log_probs = F.log_softmax(logits, dim=-1) - selected_logp = torch.gather(log_probs, -1, act_idx).sum(dim=(1, 2)) - return act_idx, selected_logp +def plackett_luce_sample(logits: torch.Tensor, S: int) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + B, K, M = logits.shape + u = torch.rand_like(logits) + g = -torch.log(-torch.log(u + 1e-12) + 1e-12) + scores = logits + g + topk = torch.topk(scores, k=S, dim=-1) + act_idx = topk.indices + log_probs = F.log_softmax(logits, dim=-1) + selected_logp = torch.gather(log_probs, -1, act_idx).sum(dim=(1, 2)) + return act_idx, selected_logp + + +def evaluate(env: Env, policy: nn.Module, candidate_mask: torch.Tensor, mode: str = "ppo", runs: int = 1, + temperature: float = None, greedy_top_s: bool = True) -> Dict[str, float]: + assert mode in {"ppo", "greedy"} + results = [] + if mode == "ppo" and policy is not None: + policy.eval() + for _ in range(runs): + if not env.fixed_positions: + env._sample_positions() + Beta = env.Beta.reshape(env.M_ap, env.K) + + if mode == "greedy": + action = np.argsort(-Beta, axis=0)[:env.S, :].T + else: + Beta_t = torch.from_numpy(Beta).float().to(env.device).unsqueeze(0) + with torch.no_grad(): + logits = policy(Beta_t, candidate_mask) + if temperature is not None and temperature > 0 and not greedy_top_s: + scaled_logits = logits / float(temperature) + act_idx, _ = plackett_luce_sample(scaled_logits, env.S) + else: + act_idx = torch.topk(logits, k=env.S, dim=-1).indices + action = act_idx.squeeze(0).cpu().numpy() + + ServingAP, infeasible = env._decode_action(action) + metrics = env._evaluate_action(ServingAP, infeasible) + results.append(metrics) + + avg_wsr = float(np.mean([r['wsr_bps'] for r in results]) / 1e6) + avg_worst_dvp = float(np.mean([r['worst_dvp'] for r in results])) + constraint_rate = float(np.mean([r['constraint_satisfied'] for r in results])) + return { + "mode": mode, + "avg_wsr_mbps": avg_wsr, + "avg_worst_dvp": avg_worst_dvp, + "constraint_rate": constraint_rate + } + + +def save_eval_results(results: List[Dict[str, float]], json_path: str = "eval_results.json", csv_path: str = "eval_results.csv"): + with open(json_path, "w", encoding="utf-8") as f: + json.dump(results, f, ensure_ascii=False, indent=2) + + fieldnames = ["mode", "avg_wsr_mbps", "avg_worst_dvp", "constraint_rate"] + with open(csv_path, "w", encoding="utf-8", newline="") as f: + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + for row in results: + writer.writerow(row) # ============================================================================= # 4) Training Loop @@ -404,10 +446,10 @@ def train(cfg: Dict): target_kl = 0.015; kl_coef = 1.5 metrics = {"rewards": [], "worst_dvp": [], "wsr": []} - for it in range(cfg["train_iters"]): - obs = env.reset() - Beta = env.Beta.reshape(env.M_ap, env.K) - Beta_t = torch.from_numpy(Beta).float().to(env.device).unsqueeze(0) + for it in range(cfg["train_iters"]): + obs = env.reset() + Beta = env.Beta.reshape(env.M_ap, env.K) + Beta_t = torch.from_numpy(Beta).float().to(env.device).unsqueeze(0) with torch.no_grad(): logits = policy(Beta_t, candidate_mask) @@ -462,29 +504,48 @@ def train(cfg: Dict): if (it + 1) % (cfg["batch_size"] * 2) == 0: avg_r = np.mean(metrics["rewards"][-cfg["batch_size"]:]) avg_wdvp = np.mean(metrics["worst_dvp"][-cfg["batch_size"]:]) - avg_wsr = np.mean(metrics["wsr"][-cfg["batch_size"]:]) - print(f"[Iter {it+1:04d}] Reward: {avg_r:.3f} | WSR: {avg_wsr:.2f} Mbps | WorstDVP: {avg_wdvp:.2e}") - - # Plot - import matplotlib.pyplot as plt - fig, axes = plt.subplots(3, 1, figsize=(10, 10), sharex=True) + avg_wsr = np.mean(metrics["wsr"][-cfg["batch_size"]:]) + print(f"[Iter {it+1:04d}] Reward: {avg_r:.3f} | WSR: {avg_wsr:.2f} Mbps | WorstDVP: {avg_wdvp:.2e}") + + print("\n=== [Phase 3] Evaluation ===") + eval_runs = int(cfg.get("eval_runs", 1)) + eval_temp = cfg.get("eval_temperature", None) + greedy_top_s = bool(cfg.get("eval_greedy_top", True)) + + greedy_res = evaluate(env, policy, candidate_mask, mode="greedy", runs=eval_runs) + ppo_res = evaluate(env, policy, candidate_mask, mode="ppo", runs=eval_runs, temperature=eval_temp, greedy_top_s=greedy_top_s) + eval_results = [greedy_res, ppo_res] + save_eval_results(eval_results) + + print("Evaluation summary (average over runs):") + for res in eval_results: + print( + f" - {res['mode'].upper():6s} | WSR: {res['avg_wsr_mbps']:.3f} Mbps | " + f"Worst DVP: {res['avg_worst_dvp']:.2e} | Constraint rate: {res['constraint_rate']:.2%}" + ) + + # Plot + import matplotlib.pyplot as plt + fig, axes = plt.subplots(3, 1, figsize=(10, 10), sharex=True) # 1. Reward axes[0].plot(metrics["rewards"], color='tab:blue', alpha=0.3) axes[0].set_ylabel('Reward') axes[0].set_title('Training Metrics') - # 2. WSR - axes[1].plot(metrics["wsr"], color='tab:orange', alpha=0.3) - axes[1].set_ylabel('WSR (Mbps)') - w = 50 - if len(metrics["wsr"]) > w: - ma = np.convolve(metrics["wsr"], np.ones(w)/w, mode='valid') - axes[1].plot(range(w-1, len(metrics["wsr"])), ma, color='tab:orange', linewidth=2) - - # 3. DVP - dvp_np = np.maximum(np.array(metrics["worst_dvp"]), 1e-16) - axes[2].plot(dvp_np, color='tab:red', alpha=0.3) + # 2. WSR + axes[1].plot(metrics["wsr"], color='tab:orange', alpha=0.3) + axes[1].set_ylabel('WSR (Mbps)') + w = 50 + if len(metrics["wsr"]) > w: + ma = np.convolve(metrics["wsr"], np.ones(w)/w, mode='valid') + axes[1].plot(range(w-1, len(metrics["wsr"])), ma, color='tab:orange', linewidth=2) + axes[1].axhline(y=greedy_res["avg_wsr_mbps"], color='gray', linestyle='--', label=f"Greedy: {greedy_res['avg_wsr_mbps']:.2f} Mbps") + axes[1].legend() + + # 3. DVP + dvp_np = np.maximum(np.array(metrics["worst_dvp"]), 1e-16) + axes[2].plot(dvp_np, color='tab:red', alpha=0.3) axes[2].set_ylabel('Worst DVP (Log)') axes[2].set_yscale('log') axes[2].axhline(y=cfg['eps_target_sys'], color='green', linestyle='--') @@ -501,10 +562,10 @@ def train(cfg: Dict): n=200, B=2e6, # 保持你的物理参数 - rho_target=0.95, # 0.95 是个很好的挑战值 - eps_target_sys=1e-2, - d_th_time=5e-3, epsilon_dec=1e-2, - R_ref_bps=9e6, + rho_target=0.95, # 0.95 是个很好的挑战值 + eps_target_sys=1e-2, + d_th_time=5e-3, epsilon_dec=1e-2, + R_ref_bps=9e6, # === 关键修改点 === batch_size=512, # 增大 Batch Size 以稳定方差 @@ -520,6 +581,11 @@ def train(cfg: Dict): infeasible_penalty=0.1, P_AP_W=1.0, noise_figure_db=5.0, rho_p=10.0, area=500.0, d0=36.0, alpha_path=3.6, - num_quantiles=50, fixed_positions=True, hidden=128, hidden_v=256 - ) - train(cfg) \ No newline at end of file + num_quantiles=50, fixed_positions=True, hidden=128, hidden_v=256, + # 评估相关 + eval_runs=3, + eval_temperature=None, + eval_greedy_top=True + ) + train(cfg) +