2. Double DQN解决过估计问题
任务目标
在CartPole-v1环境中分别实现标准DQN与Double DQN，通过对比两者对动作价值Q值的估计偏差，验证Double DQN能够缓解过估计问题，并展示其在训练稳定性与最终性能上的提升。
核心原理与实现要点
标准DQN使用同一个网络同时完成动作选择和Q值评估，即 max_a Q(s', a; θ)，由于最大化操作会引入正向偏差，导致Q值被高估。Double DQN将动作选择与价值评估解耦：用当前网络选择最优动作 a* = argmax_a Q(s', a; θ)，再用目标网络评估该动作的价值 Q(s', a*; θ⁻)，从而有效降低过估计。
实现中两个智能体共享完全相同的网络结构、超参数与经验回放缓冲区，唯一区别在于目标Q值的计算方式。训练过程中每200步同步一次目标网络参数，每1000步评估一次当前策略的回报与平均Q值。
完整实现代码
import gymnasium as gym
import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
from collections import deque
import random
# ---------- 网络定义 ----------
class QNetwork(nn.Module):
def __init__(self, state_dim, action_dim):
super().__init__()
self.net = nn.Sequential(
nn.Linear(state_dim, 128), nn.ReLU(),
nn.Linear(128, 128), nn.ReLU(),
nn.Linear(128, action_dim)
)
def forward(self, x):
return self.net(x)
# ---------- 经验回放 ----------
class ReplayBuffer:
def __init__(self, capacity):
self.buffer = deque(maxlen=capacity)
def push(self, s, a, r, s2, done):
self.buffer.append((s, a, r, s2, done))
def sample(self, batch_size):
batch = random.sample(self.buffer, batch_size)
s, a, r, s2, d = map(np.stack, zip(*batch))
return (torch.FloatTensor(s), torch.LongTensor(a), torch.FloatTensor(r),
torch.FloatTensor(s2), torch.FloatTensor(d))
def __len__(self):
return len(self.buffer)
# ---------- DQN智能体（use_double控制是否使用Double DQN） ----------
class DQNAgent:
def __init__(self, state_dim, action_dim, use_double=False):
self.action_dim = action_dim
self.use_double = use_double
self.q_net = QNetwork(state_dim, action_dim)
self.target_net = QNetwork(state_dim, action_dim)
self.target_net.load_state_dict(self.q_net.state_dict())
self.optimizer = optim.Adam(self.q_net.parameters(), lr=1e-3)
self.buffer = ReplayBuffer(20000)
self.batch_size = 64
self.gamma = 0.99
self.epsilon = 1.0
self.epsilon_min = 0.01
self.epsilon_decay = 0.995
self.update_counter = 0
def act(self, state, eval_mode=False):
if not eval_mode and random.random() < self.epsilon:
return random.randrange(self.action_dim)
with torch.no_grad():
q = self.q_net(torch.FloatTensor(state).unsqueeze(0))
return q.argmax().item()
def learn(self):
if len(self.buffer) < self.batch_size:
return None
s, a, r, s2, done = self.buffer.sample(self.batch_size)
q_current = self.q_net(s).gather(1, a.unsqueeze(1)).squeeze()
with torch.no_grad():
if self.use_double:
# Double DQN: 当前网络选动作，目标网络给价值
next_actions = self.q_net(s2).argmax(dim=1, keepdim=True)
q_target = self.target_net(s2).gather(1, next_actions).squeeze()
else:
# 标准DQN: 目标网络直接取max
q_target = self.target_net(s2).max(dim=1)[0]
q_target = r + self.gamma * q_target * (1 - done)
loss = nn.MSELoss()(q_current, q_target)
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
self.update_counter += 1
if self.update_counter % 200 == 0:
self.target_net.load_state_dict(self.q_net.state_dict())
self.epsilon = max(self.epsilon_min, self.epsilon * self.epsilon_decay)
return loss.item()
# ---------- 训练与评估 ----------
def train_agent(use_double, episodes=300):
env = gym.make('CartPole-v1')
agent = DQNAgent(env.observation_space.shape[0], env.action_space.n, use_double)
eval_interval = 10
eval_episodes = 5
results = {'returns': [], 'q_values': []}
for ep in range(episodes):
state, _ = env.reset()
done = False
ep_return = 0
while not done:
action = agent.act(state)
next_state, reward, terminated, truncated, _ = env.step(action)
done = terminated or truncated
agent.buffer.push(state, action, reward, next_state, float(done))
state = next_state
ep_return += reward
if len(agent.buffer) > 500:
agent.learn()
if (ep + 1) % eval_interval == 0:
eval_return, avg_q = evaluate(agent, env)
results['returns'].append(eval_return)
results['q_values'].append(avg_q)
print(f"{'DoubleDQN' if use_double else 'DQN'} | Ep {ep+1} | "
f"EvalReturn: {eval_return:.1f} | AvgQ: {avg_q:.2f}")
env.close()
return results
def evaluate(agent, env, episodes=5):
returns = []
q_values = []
for _ in range(episodes):
state, _ = env.reset()
done = False
ep_ret = 0
while not done:
action = agent.act(state, eval_mode=True)
with torch.no_grad():
q = agent.q_net(torch.FloatTensor(state).unsqueeze(0)).max().item()
q_values.append(q)
state, reward, terminated, truncated, _ = env.step(action)
done = terminated or truncated
ep_ret += reward
returns.append(ep_ret)
return np.mean(returns), np.mean(q_values)
# ---------- 运行对比 ----------
if __name__ == '__main__':
seed = 42
torch.manual_seed(seed)
np.random.seed(seed)
random.seed(seed)
print("训练标准DQN...")
dqn_results = train_agent(use_double=False)
print("\n训练Double DQN...")
ddqn_results = train_agent(use_double=True)
# 保存结果用于绘图
np.save('dqn_results.npy', dqn_results)
np.save('ddqn_results.npy', ddqn_results)
print("\n训练完成，结果已保存。")运行与验证
环境依赖：需要安装 gymnasium、torch、numpy。运行命令：
pip install gymnasium torch numpy
python double_dqn_cartpole.py验证方法与判定标准：
- Q值对比：观察训练日志中两个智能体的
AvgQ指标。标准DQN的Q值通常会快速膨胀（例如超过200），而Double DQN的Q值更接近真实回报（CartPole-v1每步奖励为1，因此合理Q值应接近平均回合长度）。 - 性能对比：比较
EvalReturn曲线。Double DQN应达到500分（环境最大步数）所需的训练回合更少，且训练过程更稳定（曲线波动更小）。 - 预期输出：在相同随机种子下，标准DQN的最终平均Q值通常比Double DQN高20%以上，而Double DQN的评估回报达到500的回合数更早。具体数值因随机性略有差异，但过估计缓解的趋势是稳定可复现的。
常见问题：若训练不收敛，可检查经验回放是否达到500条后才开始学习（代码中已设置）；若Q值仍然过高，可适当降低学习率或增大 epsilon_decay 的衰减速度。CartPole-v1的奖励上限为500，当评估回报稳定在500附近时说明策略已收敛到最优。