浅谈Attention-based Model【源码篇】

def embedding_attention_seq2seq(encoder_inputs,# 编码器的输入
                                decoder_inputs,# 解码器的输入
                                cell,# 所用的cell
                                num_encoder_symbols,# 编码的符号总数
                                num_decoder_symbols,# 解码的符号总数
                                embedding_size,# 向量的维度
                                num_heads=1,# 这个也不知道啊，默认为1，基本上可以当它不存在了
                                output_projection=None, # 输出投影
                                feed_previous=False,# 当前输入是否要考虑前一个时刻的输出，这个一般在训练的时候为False，预测的时候为True
                                dtype=None,
                                scope=None,
                                initial_state_attention=False):

返回值：
由（outputs，state）组成的元组
outputs的shape为[batch_size x num_decoder_symbols]
state为最后一个时刻decoder的状态，shape为[batch_size x cell.state_size]

def embedding_attention_decoder(decoder_inputs,
                                initial_state,
                                attention_states,
                                cell,
                                num_symbols,
                                embedding_size,
                                num_heads=1,
                                output_size=None,# 输出的大小，tensorflow中为num_decoder_symbols
                                output_projection=None,
                                feed_previous=False,
                                update_embedding_for_previous=True,
                                dtype=None,
                                scope=None,
                                initial_state_attention=False):# 初始attention向量的值，默认为0

返回值：
同上                                

def attention_decoder(decoder_inputs,# 编码器的输入，向量形式
                      initial_state,# 初始状态，tensorflow照顾你为rnn编码器的最后一个时刻的state
                      attention_states,# 需要attention的向量，即为编码器的每一个时刻的输出
                      cell,
                      output_size=None,
                      num_heads=1,
                      loop_function=None,
                      dtype=None,
                      scope=None,
                      initial_state_attention=False):

 for a in xrange(num_heads):
      # filter的大小，输入的通道数为输入embedding_size的大小，输出的通道为attention_vec_size，但正如前面所说，tensorflow里面这两个值是相等的
      k = variable_scope.get_variable("AttnW_%d" % a,
                                      [1, 1, attn_size, attention_vec_size])
      hidden_features.append(nn_ops.conv2d(hidden, k, [1, 1, 1, 1], "SAME"))
      v.append(
          variable_scope.get_variable("AttnV_%d" % a, [attention_vec_size]))

<font size=4>
#循环，依次将解码状态的每一个时刻的是state都做一次attention，然后和该时刻的decoder_inputs值共同决定该时刻的输入
    for i, inp in enumerate(decoder_inputs):
      if i > 0:
        variable_scope.get_variable_scope().reuse_variables()
      # If loop_function is set, we use it instead of decoder_inputs.
      if loop_function is not None and prev is not None:
        with variable_scope.variable_scope("loop_function", reuse=True):
          inp = loop_function(prev, i)
      # Merge input and previous attentions into one vector of the right size.
      input_size = inp.get_shape().with_rank(2)[1]
      if input_size.value is None:
        raise ValueError("Could not infer input size from input: %s" % inp.name)
      # 和decoder_inputs该时刻的值共同决定该时刻的输入
      # 这里的因为原文用的词为concat，但是这里应该是每一个维度的值相加
      x = linear([inp] + attns, input_size, True)
      # Run the RNN.
      # 使用输入和上一个时刻的隐状态共同决定当前时刻的隐状态和解码的输出
      cell_output, state = cell(x, state)
      # Run the attention mechanism.
      if i == 0 and initial_state_attention:
        with variable_scope.variable_scope(
            variable_scope.get_variable_scope(), reuse=True):
          attns = attention(state)
      else:
        attns = attention(state)
      with variable_scope.variable_scope("AttnOutputProjection"):
        output = linear([cell_output] + attns, output_size, True)
      if loop_function is not None:
        prev = output
      outputs.append(output)

    def attention(query):
      """Put attention masks on hidden using hidden_features and query."""
      ds = []  # Results of attention reads will be stored here.
      if nest.is_sequence(query):  # If the query is a tuple, flatten it.
        query_list = nest.flatten(query)
        for q in query_list:  # Check that ndims == 2 if specified.
          ndims = q.get_shape().ndims
          if ndims:
            assert ndims == 2
        query = array_ops.concat(query_list, 1)
      for a in xrange(num_heads):
        with variable_scope.variable_scope("Attention_%d" % a):
          # 公式（1）中的$W_2^{d_t}$
          y = linear(query, attention_vec_size, True)
          y = array_ops.reshape(y, [-1, 1, 1, attention_vec_size])
          # Attention mask is a softmax of v^T * tanh(...).
          # 公式（1）结果
          s = math_ops.reduce_sum(v[a] * math_ops.tanh(hidden_features[a] + y),
                                  [2, 3])
          # 公式（2）
          a = nn_ops.softmax(s)
          # Now calculate the attention-weighted vector d.
          # 公式（3）
          d = math_ops.reduce_sum(
              array_ops.reshape(a, [-1, attn_length, 1, 1]) * hidden, [1, 2])
          ds.append(array_ops.reshape(d, [-1, attn_size]))
      return ds