import numpy as np

# 初始化Q表
q_table = np.zeros((state_space_size, action_space_size))

# Q學習更新公式
alpha = 0.1  # 學習率
gamma = 0.99  # 折扣因子

for episode in range(total_episodes):
    state = env.reset()
    done = False
    
    while not done:
        action = select_action(state)  # ε-greedy策略
        next_state, reward, done, _ = env.step(action)
        
        # Q值更新
        q_table[state, action] = q_table[state, action] + alpha * (
            reward + gamma * np.max(q_table[next_state]) - q_table[state, action]
        )
        state = next_state

import torch
import torch.nn as nn
import torch.optim as optim

class DQN(nn.Module):
    def __init__(self, input_dim, output_dim):
        super(DQN, self).__init__()
        self.fc1 = nn.Linear(input_dim, 128)
        self.fc2 = nn.Linear(128, 128)
        self.fc3 = nn.Linear(128, output_dim)
        
    def forward(self, x):
        x = torch.relu(self.fc1(x))
        x = torch.relu(self.fc2(x))
        return self.fc3(x)

from collections import deque
import random

class ReplayBuffer:
    def __init__(self, capacity):
        self.buffer = deque(maxlen=capacity)
    
    def push(self, state, action, reward, next_state, done):
        self.buffer.append((state, action, reward, next_state, done))
    
    def sample(self, batch_size):
        return random.sample(self.buffer, batch_size)
    
    def __len__(self):
        return len(self.buffer)

target_net = DQN(input_dim, output_dim).to(device)
target_net.load_state_dict(policy_net.state_dict())
target_net.eval（)

# 定期更新目標網(wǎng)絡
if steps_done % TARGET_UPDATE == 0:
    target_net.load_state_dict(policy_net.state_dict())

import gym
import numpy as np
import torch
import random
from collections import deque
import matplotlib.pyplot as plt

# 超參數(shù)
BATCH_SIZE = 128
GAMMA = 0.99
EPS_START = 0.9
EPS_END = 0.05
EPS_DECAY = 200
TARGET_UPDATE = 10
LR = 0.001

# 初始化環(huán)境
env = gym.make('CartPole-v1')
state_dim = env.observation_space.shape[0]
action_dim = env.action_space.n

# 神經(jīng)網(wǎng)絡定義
class DQN(nn.Module):
    def __init__(self, input_dim, output_dim):
        super(DQN, self).__init__()
        self.fc1 = nn.Linear(input_dim, 64)
        self.fc2 = nn.Linear(64, 64)
        self.fc3 = nn.Linear(64, output_dim)
        
    def forward(self, x):
        x = torch.relu(self.fc1(x))
        x = torch.relu(self.fc2(x))
        return self.fc3(x)

# 初始化網(wǎng)絡
policy_net = DQN(state_dim, action_dim).to(device)
target_net = DQN(state_dim, action_dim).to(device)
target_net.load_state_dict(policy_net.state_dict())
optimizer = optim.Adam(policy_net.parameters(), lr=LR)
memory = ReplayBuffer(10000)

# 訓練過程
def train():
    if len(memory) < BATCH_SIZE:
        return
    transitions = memory.sample(BATCH_SIZE)
    batch = list(zip(*transitions))
    
    state_batch = torch.FloatTensor(np.array(batch[0]))
    action_batch = torch.LongTensor(np.array(batch[1]))
    reward_batch = torch.FloatTensor(np.array(batch[2]))
    next_state_batch = torch.FloatTensor(np.array(batch[3]))
    done_batch = torch.FloatTensor(np.array(batch[4]))
    
    current_q = policy_net(state_batch).gather(1, action_batch.unsqueeze(1))
    next_q = target_net(next_state_batch).max(1)[0].detach()
    expected_q = reward_batch + (1 - done_batch) * GAMMA * next_q
    
    loss = nn.MSELoss()(current_q.squeeze(), expected_q)
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()

# 主訓練循環(huán)
episode_rewards = []
for episode in range(500):
    state = env.reset()
    total_reward = 0
    done = False
    
    while not done:
        # ε-greedy動作選擇
        eps_threshold = EPS_END + (EPS_START - EPS_END) * \
            np.exp(-1. * episode / EPS_DECAY)
        if random.random() > eps_threshold:
            with torch.no_grad():
                action = policy_net(torch.FloatTensor(state)).argmax().item()
        else:
            action = random.randint(0, action_dim-1)
        
        next_state, reward, done, _ = env.step(action)
        memory.push(state, action, reward, next_state, done)
        state = next_state
        total_reward += reward
        
        train()
    
    episode_rewards.append(total_reward)
    if episode % 10 == 0:
        print(f"Episode {episode}, Total Reward: {total_reward}")

# 繪制訓練曲線
plt.plot(episode_rewards)
plt.xlabel('Episode')
plt.ylabel('Total Reward')
plt.title('DQN Training Progress')
plt.show()