Skip to content

Enhance Env step with adaptive penalties #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
chenmaike wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Enhance Env step with adaptive penalties #1

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
187 changes: 102 additions & 85 deletions test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,26 +161,35 @@ def __init__(self, cfg: dict):
self.fixed_positions = cfg.get("fixed_positions", True)
self.num_quantiles = cfg["num_quantiles"]
self.R_ref_bps = cfg["R_ref_bps"]
self.infeasible_penalty = cfg["infeasible_penalty"]
self.infeasible_penalty = max(float(cfg["infeasible_penalty"]), 1.0)
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

P_noise_W = 10 ** ((-174.0 + 10.0 * math.log10(self.B) + float(cfg["noise_figure_db"]) - 30.0) / 10.0)
self.rho_d = float(cfg["P_AP_W"]) / P_noise_W
self.rho_p = float(cfg["rho_p"])
Tf = self.n / self.B
self.d_th_frames = int(np.round(self.d_th_time / Tf))
self.d_th_frames = int(np.round(self.d_th_time / Tf))

self._rng = np.random.default_rng(int(cfg["seed"]))
self._area = float(cfg["area"])
self._d0 = float(cfg["d0"])
self._alpha_path = float(cfg["alpha_path"])

self._sample_positions()

eps_k = np.full((self.K,), self.eps_target_sys, dtype=np.float32)
dth_k = np.full((self.K,), self.d_th_time, dtype=np.float32)
w = (np.log(1.0 / eps_k) / dth_k).astype(np.float64)
self.w_k = (w / np.sum(w)).astype(np.float32)
self._rng = np.random.default_rng(int(cfg["seed"]))
self._area = float(cfg["area"])
self._d0 = float(cfg["d0"])
self._alpha_path = float(cfg["alpha_path"])

self._sample_positions()
self.candidate_mask = self.get_candidate_mask(top_k_candidates=20)
self.candidate_mask_np = self.candidate_mask.cpu().numpy() >= -1e-6

self.lambda_min = float(cfg.get("lambda_min", 0.5))
self.lambda_max = float(cfg.get("lambda_max", 50.0))
self.lambda_increase = float(cfg.get("lambda_increase", 0.25))
self.lambda_decrease = float(cfg.get("lambda_decrease", 0.05))
self.lambda_gain = float(cfg.get("lambda_gain", 2.0))
self.lambda_p = self.lambda_min

eps_k = np.full((self.K,), self.eps_target_sys, dtype=np.float32)
dth_k = np.full((self.K,), self.d_th_time, dtype=np.float32)
w = (np.log(1.0 / eps_k) / dth_k).astype(np.float64)
self.w_k = (w / np.sum(w)).astype(np.float32)

def _compute_Beta(self, d0, alpha_path):
Beta = np.zeros((self.M_ap, self.K), dtype=np.float64)
Expand All @@ -195,37 +204,55 @@ def _sample_positions(self):
self.ap_loc = self._rng.random((self.M_ap, 2)) * self._area
self.Beta = self._compute_Beta(d0=self._d0, alpha_path=self._alpha_path)

def get_candidate_mask(self, top_k_candidates=20):
mask = torch.full((self.K, self.M_ap), -1e9, device=self.device)
Beta_t = torch.from_numpy(self.Beta.T).to(self.device)
_, indices = torch.topk(Beta_t, k=top_k_candidates, dim=1)
mask.scatter_(1, indices, 0.0)
return mask

def reset(self):
if not self.fixed_positions:
self._sample_positions()
return np.concatenate([self.Beta.reshape(-1), np.array([self.N_ant], dtype=np.float32)], axis=0).astype(np.float32)

def _decode_action(self, action):
infeasible = 0.0
ServingAP = []
for k in range(self.K):
chosen = action[k].astype(np.int64)
if np.any(chosen < 0) or np.any(chosen >= self.M_ap):
infeasible += 1.0
chosen = np.clip(chosen, 0, self.M_ap - 1)
u = np.unique(chosen)
if len(u) < len(chosen):
infeasible += 0.5
chosen = u
if chosen.size == 0:
infeasible += 1.0
chosen = np.array([int(np.argmax(self.Beta[:, k]))], dtype=np.int64)
if chosen.size > self.S:
chosen = chosen[: self.S]
ServingAP.append(chosen)
return ServingAP, infeasible
def get_candidate_mask(self, top_k_candidates=20):
mask = torch.full((self.K, self.M_ap), -1e9, device=self.device)
Beta_t = torch.from_numpy(self.Beta.T).to(self.device)
_, indices = torch.topk(Beta_t, k=top_k_candidates, dim=1)
mask.scatter_(1, indices, 0.0)
self.candidate_mask = mask
self.candidate_mask_np = mask.cpu().numpy() >= -1e-6
return mask

def reset(self):
if not self.fixed_positions:
self._sample_positions()
self.candidate_mask = self.get_candidate_mask(top_k_candidates=20)
self.candidate_mask_np = self.candidate_mask.cpu().numpy() >= -1e-6
return np.concatenate([self.Beta.reshape(-1), np.array([self.N_ant], dtype=np.float32)], axis=0).astype(np.float32)

def _decode_action(self, action):
infeasible = 0.0
ServingAP = []
for k in range(self.K):
chosen = action[k].astype(np.int64)
invalid_mask = (chosen < 0) | (chosen >= self.M_ap)
if np.any(invalid_mask):
infeasible += 1.5 * np.sum(invalid_mask)
chosen = chosen[~invalid_mask]
allowed_mask = None
if getattr(self, "candidate_mask_np", None) is not None:
allowed_mask = self.candidate_mask_np[k]
if allowed_mask is not None and chosen.size > 0:
invalid_candidates = ~allowed_mask[chosen]
if np.any(invalid_candidates):
infeasible += 1.0 * np.sum(invalid_candidates)
chosen = chosen[~invalid_candidates]
u = np.unique(chosen)
if len(u) < len(chosen):
infeasible += 1.0
chosen = u
if chosen.size == 0:
infeasible += 1.5
if allowed_mask is not None:
fallback = np.argmax(self.Beta[:, k] * allowed_mask.astype(np.float64))
else:
fallback = np.argmax(self.Beta[:, k])
chosen = np.array([int(fallback)], dtype=np.int64)
if chosen.size > self.S:
infeasible += 0.5 * (chosen.size - self.S)
chosen = chosen[: self.S]
ServingAP.append(chosen)
return ServingAP, infeasible

def step(self, action):
ServingAP, infeasible = self._decode_action(action)
Expand Down Expand Up @@ -266,46 +293,36 @@ def step(self, action):

# === 修改后的平滑奖励逻辑 ===
eps = float(self.eps_target_sys)
worst_dvp = float(np.max(dvp))
safe_dvp = max(worst_dvp, 1e-12) # 防止 log 报错

# 归一化 WSR (0.0 ~ 1.2 左右)
r_wsr = wsr / self.R_ref_bps

# 计算违背程度 (log space)
# 如果 DVP = 1e-2 (目标), log_term = 0
# 如果 DVP = 1e-1, log_term = 1.0
# 如果 DVP = 1e-3, log_term = -1.0
log_term = math.log10(safe_dvp / eps)

# 惩罚系数
lambda_p = 1.0

# 组合奖励：
# 1. 基础 WSR 奖励
# 2. 如果 DVP > eps (log_term > 0)，施加惩罚
# 3. 如果 DVP < eps (log_term < 0)，给予微小奖励或不惩罚
# 使用 Softplus 或 ReLU 变体来平滑

if log_term > 0:
# 越界了：线性扣分，不要太狠，给它改过的机会
# 之前的 5.0 太大了，改成 2.0 甚至 1.0
penalty = 2.0 * log_term
reward = r_wsr - penalty
else:
# 安全区域：
# 稍微给一点点正反馈，鼓励更低的时延，但主要还是看 WSR
reward = r_wsr - 0.1 * log_term # 注意 log_term 是负数，这里变成加分

reward -= (self.infeasible_penalty * infeasible)

next_state = self.reset()
info = {
'wsr_bps': wsr,
'worst_dvp': worst_dvp,
'reward': float(reward)
}
return next_state, float(reward), True, info
worst_dvp = float(np.max(dvp))
safe_dvp = max(worst_dvp, 1e-12) # 防止 log 报错

# 归一化 WSR (0.0 ~ 1.2 左右)
r_wsr = wsr / self.R_ref_bps

# 自适应拉格朗日系数：超标时递增，低于阈值缓降
dvp_ratio = safe_dvp / eps
log_term = math.log(dvp_ratio + 1e-12)
lambda_candidate = max(self.lambda_min, self.lambda_gain * max(0.0, log_term))
if dvp_ratio > 1.0:
growth = 1.0 + self.lambda_increase * (dvp_ratio - 1.0)
self.lambda_p = min(self.lambda_max, max(self.lambda_p * growth, lambda_candidate))
else:
decay = 1.0 - self.lambda_decrease * (1.0 - dvp_ratio)
self.lambda_p = max(self.lambda_min, min(self.lambda_max, self.lambda_p * decay))
self.lambda_p = max(self.lambda_p, lambda_candidate)

soft_barrier = self.lambda_p * math.log1p(math.exp(log_term))
reward = r_wsr - soft_barrier - (self.infeasible_penalty * infeasible)

next_state = self.reset()
info = {
'wsr_bps': wsr,
'worst_dvp': worst_dvp,
'reward': float(reward),
'lambda_p': float(self.lambda_p),
'infeasible': float(infeasible)
}
return next_state, float(reward), True, info

# =============================================================================
# 3) GNN & Agent
Expand Down Expand Up @@ -522,4 +539,4 @@ def train(cfg: Dict):
area=500.0, d0=36.0, alpha_path=3.6,
num_quantiles=50, fixed_positions=True, hidden=128, hidden_v=256
)
train(cfg)
train(cfg)