强化学习代码实践1.DDQN:在CartPole游戏中实现 Double DQN - IT评测·应用市场-qidao123.com技术社区

import random
import torch
import torch.nn as nn
import torch.optim as optim
import numpy as np
import gym
from collections import deque

复制代码

class QNetwork(nn.Module):
def __init__(self, state_dim, action_dim):
super(QNetwork, self).__init__()
self.fc1 = nn.Linear(state_dim, 128)
self.fc2 = nn.Linear(128, 128)
self.fc3 = nn.Linear(128, action_dim)
def forward(self, x):
x = torch.relu(self.fc1(x))
x = torch.relu(self.fc2(x))
return self.fc3(x)

复制代码

class DoubleDQNAgent:
def __init__(self, state_dim, action_dim, gamma=0.99, epsilon=0.1, epsilon_decay=0.995, epsilon_min=0.01, lr=0.0005):
self.state_dim = state_dim
self.action_dim = action_dim
self.gamma = gamma
self.epsilon = epsilon
self.epsilon_decay = epsilon_decay
self.epsilon_min = epsilon_min
self.lr = lr
self.q_network = QNetwork(state_dim, action_dim)
self.target_network = QNetwork(state_dim, action_dim)
self.target_network.load_state_dict(self.q_network.state_dict())
self.optimizer = optim.Adam(self.q_network.parameters(), lr=self.lr)
self.memory = deque(maxlen=10000)
self.batch_size = 64
def select_action(self, state):
if random.random() < self.epsilon:
return random.choice(range(self.action_dim)) # Explore
else:
state = torch.FloatTensor(state).unsqueeze(0)
with torch.no_grad():
q_values = self.q_network(state)
return torch.argmax(q_values).item() # Exploit
def store_experience(self, state, action, reward, next_state, done):
self.memory.append((state, action, reward, next_state, done))
def sample_batch(self):
return random.sample(self.memory, self.batch_size)
def update_target_network(self):
self.target_network.load_state_dict(self.q_network.state_dict())
def train(self):
if len(self.memory) < self.batch_size:
return
batch = self.sample_batch()
states, actions, rewards, next_states, dones = zip(*batch)
states = torch.FloatTensor(states)
actions = torch.LongTensor(actions)
rewards = torch.FloatTensor(rewards)
next_states = torch.FloatTensor(next_states)
dones = torch.FloatTensor(dones)
# Q values for current states
q_values = self.q_network(states)
q_values = q_values.gather(1, actions.unsqueeze(1)).squeeze(1)
# Next Q values using target network
next_q_values = self.target_network(next_states)
next_actions = self.q_network(next_states).argmax(1)
next_q_values = next_q_values.gather(1, next_actions.unsqueeze(1)).squeeze(1)
# Double DQN update
target = rewards + (1 - dones) * self.gamma * next_q_values
# Compute loss
loss = nn.MSELoss()(q_values, target)
# Optimize the Q-network
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
if self.epsilon > self.epsilon_min:
self.epsilon *= self.epsilon_decay

复制代码

def train_cartpole():
env = gym.make('CartPole-v1')
agent = DoubleDQNAgent(state_dim=env.observation_space.shape[0], action_dim=env.action_space.n)
episodes = 1000
for episode in range(episodes):
state, info = env.reset()
done = False
total_reward = 0
while not done:
action = agent.select_action(state)
next_state, reward, done, truncated, info = env.step(action)
agent.store_experience(state, action, reward, next_state, done)
state = next_state
agent.train()
total_reward += reward
agent.update_target_network()
print(f"Episode {episode}, Total Reward: {total_reward}, Epsilon: {agent.epsilon:.4f}")
env.close()
if __name__ == '__main__':
train_cartpole()

复制代码