Prioritized Experience Replay (DQN)——让DQN变得更会学习

 1class SumTree(object):
 2    # 建立 tree 和 data,
 3    # 因为 SumTree 有特殊的数据结构,
 4    # 所以两者都能用一个一维 np.array 来存储
 5    def __init__(self, capacity):
 6
 7    # 当有新 sample 时, 添加进 tree 和 data
 8    def add(self, p, data):
 9
10    # 当 sample 被 train, 有了新的 TD-error, 就在 tree 中更新
11    def update(self, tree_idx, p):
12
13    # 根据选取的 v 点抽取样本
14    def get_leaf(self, v):
15
16    # 获取 sum(priorities)
17    @property
18    def totoal_p(self):

 1class Memory(object):
 2    # 建立 SumTree 和各种参数
 3    def __init__(self, capacity):
 4
 5    # 存储数据, 更新 SumTree
 6    def store(self, transition):
 7
 8    # 抽取 sample
 9    def sample(self, n):
10
11    # train 完被抽取的 samples 后更新在 tree 中的 sample 的 priority
12    def batch_update(self, tree_idx, abs_errors):

 1class DQNPrioritizedReplay:
 2    def _build_net(self)
 3        ...
 4        # self.prioritized 时 eval net 的 input 多加了一个 ISWeights
 5        self.s = tf.placeholder(tf.float32, [None, self.n_features], name='s')  # input
 6        self.q_target = tf.placeholder(tf.float32, [None, self.n_actions], name='Q_target')  # for calculating loss
 7        if self.prioritized:
 8            self.ISWeights = tf.placeholder(tf.float32, [None, 1], name='IS_weights')
 9
10        ...
11        # 为了得到 abs 的 TD error 并用于修改这些 sample 的 priority, 我们修改如下
12        with tf.variable_scope('loss'):
13            if self.prioritized:
14                self.abs_errors = tf.reduce_sum(tf.abs(self.q_target - self.q_eval), axis=1)    # for updating Sumtree
15                self.loss = tf.reduce_mean(self.ISWeights * tf.squared_difference(self.q_target, self.q_eval))
16            else:
17                self.loss = tf.reduce_mean(tf.squared_difference(self.q_target, self.q_eval))

 1class DQNPrioritizedReplay:
 2    def store_transition(self, s, a, r, s_):
 3        if self.prioritized:    # prioritized replay
 4            transition = np.hstack((s, [a, r], s_))
 5            self.memory.store(transition)
 6        else:       # random replay
 7            if not hasattr(self, 'memory_counter'):
 8                self.memory_counter = 0
 9            transition = np.hstack((s, [a, r], s_))
10            index = self.memory_counter % self.memory_size
11            self.memory[index, :] = transition
12            self.memory_counter += 1

 1class DQNPrioritizedReplay:
 2    def learn(self):
 3        ...
 4        # 相对于 DQN 代码, 改变的部分
 5        if self.prioritized:
 6            tree_idx, batch_memory, ISWeights = self.memory.sample(self.batch_size)
 7        else:
 8            sample_index = np.random.choice(self.memory_size, size=self.batch_size)
 9            batch_memory = self.memory[sample_index, :]
10
11        ...
12
13        if self.prioritized:
14            _, abs_errors, self.cost = self.sess.run([self._train_op, self.abs_errors, self.loss],
15                                         feed_dict={self.s: batch_memory[:, :self.n_features],
16                                                    self.q_target: q_target,
17                                                    self.ISWeights: ISWeights})
18            self.memory.batch_update(tree_idx, abs_errors)   # update priority
19        else:
20            _, self.cost = self.sess.run([self._train_op, self.loss],
21                                         feed_dict={self.s: batch_memory[:, :self.n_features],
22                                                    self.q_target: q_target})
23
24        ...