CNN预测股票走势基于Tensorflow（思路+程序）

class DQN():#初始化
  def __init__(self, env): #创建Q网络
  def create_Q_network(self): #创建训练方法
  def create_training_method(self): #感知存储信息
  def perceive(self,state,action,reward,next_state,done):#训练网络
  def train_Q_network(self):#输出带随机的动作
  def egreedy_action(self,state):#输出动作
  def action(self,state):

self.replay_buffer = deque()

def __init__(self, env):
    # init experience replay
    self.replay_buffer = deque()        # init some parameters
    self.time_step = 0
    self.epsilon = INITIAL_EPSILON
    self.state_dim = env.observation_space.shape[0]
    self.action_dim = env.action_space.n

    self.create_Q_network()
    self.create_training_method()    # Init session
    self.session = tf.InteractiveSession()
    self.session.run(tf.initialize_all_variables())

def create_Q_network(self):
    # network weights
    W1 = self.weight_variable([self.state_dim,20])
    b1 = self.bias_variable([20])
    W2 = self.weight_variable([20,self.action_dim])
    b2 = self.bias_variable([self.action_dim])        # input layer
    self.state_input = tf.placeholder("float",[None,self.state_dim])        # hidden layers
    h_layer = tf.nn.relu(tf.matmul(self.state_input,W1) + b1)        # Q Value layer
    self.Q_value = tf.matmul(h_layer,W2) + b2  def weight_variable(self,shape):
    initial = tf.truncated_normal(shape)        return tf.Variable(initial)  def bias_variable(self,shape):
    initial = tf.constant(0.01, shape = shape)        return tf.Variable(initial)

def perceive(self,state,action,reward,next_state,done):
    one_hot_action = np.zeros(self.action_dim)
    one_hot_action[action] = 1
     self.replay_buffer.append((state,one_hot_action,reward,next_state,done))        if len(self.replay_buffer) > REPLAY_SIZE:
      self.replay_buffer.popleft()        if len(self.replay_buffer) > BATCH_SIZE:
      self.train_Q_network()

def egreedy_action(self,state):
    Q_value = self.Q_value.eval(feed_dict = {
      self.state_input:[state]
      })[0]        if random.random() <= self.epsilon:             return random.randint(0,self.action_dim - 1)        else:             return np.argmax(Q_value)
    self.epsilon -= (INITIAL_EPSILON - FINAL_EPSILON)/10000def action(self,state):
    return np.argmax(self.Q_value.eval(feed_dict = {
      self.state_input:[state]
      })[0])

  def create_training_method(self):
    self.action_input = tf.placeholder("float",[None,self.action_dim]) # one hot presentation
    self.y_input = tf.placeholder("float",[None])
    Q_action = tf.reduce_sum(tf.mul(self.Q_value,self.action_input),reduction_indices = 1)
    self.cost = tf.reduce_mean(tf.square(self.y_input - Q_action))
    self.optimizer = tf.train.AdamOptimizer(0.0001).minimize(self.cost)

def train_Q_network(self):
    self.time_step += 1
    # Step 1: obtain random minibatch from replay memory
    minibatch = random.sample(self.replay_buffer,BATCH_SIZE)
    state_batch = [data[0] for data in minibatch]
    action_batch = [data[1] for data in minibatch]
    reward_batch = [data[2] for data in minibatch]
    next_state_batch = [data[3] for data in minibatch]    # Step 2: calculate y
    y_batch = []
    Q_value_batch = self.Q_value.eval(feed_dict={self.state_input:next_state_batch})    for i in range(0,BATCH_SIZE):
      done = minibatch[i][4]      if done:
        y_batch.append(reward_batch[i])      else :
        y_batch.append(reward_batch[i] + GAMMA * np.max(Q_value_batch[i]))

    self.optimizer.run(feed_dict={
      self.y_input:y_batch,
      self.action_input:action_batch,
      self.state_input:state_batch
      })

class TWStock():
    def __init__(self, stock_data,label):
        self.stock_data = stock_data
        self.stock_index = 0
        self.label=label    def render(self):
        return 

    def reset(self):
        self.stock_index = 0
        return self.stock_data[self.stock_index]        # 0: 观望, 1: 持有多单, 2: 持有空单
    def step(self, action): 
        self.stock_index += 1
        #action_reward = self.stock_data[self.stock_index][day_len-1] - self.stock_data[self.stock_index+10][day_len-1] 
        action_reward=self.label[self.stock_index]                if (action == 0):
            action_reward = 0

        if (action == 2):
            action_reward = -1 * action_reward

        stock_done = False
        if self.stock_index+10 >= len(self.stock_data)-1:
            stock_done = True
        else:
            stock_done = False
        return self.stock_data[self.stock_index], action_reward, stock_done, 0def conv2d(x, W,s):
  return tf.nn.conv2d(x, W, strides=[1, s,s,1], padding='SAME')def max_pool_2x2(x):
  return tf.nn.max_pool(x, ksize=[1, 2, 2, 1],
                        strides=[1, 2, 2, 1], padding='SAME')

def main():
 env = TWStock(my_img,label_train) 
 agent = DQN(env)

 train_output=""
 rate_string=""
 for episode in xrange(EPISODE):   # initialize task
   state = env.reset()   # Train
   out="train\n"

   train_reward=0
   for step in xrange(STEP):
     action = agent.egreedy_action(state) # e-greedy action for trai

     next_state,reward,done,_ = env.step(action)
     out+=str(reward)+" "
     train_reward+=reward     # Define reward for agent
     reward_agent = -1 if done else 0.1
     agent.perceive(state,action,reward,next_state,done)
     state = next_state     if done:       break
   anal=out.split()
   p=0.0
   n=0.0
   for x in range(1,len(anal)-1):     if(float(anal[x])>0):
      p+=float(anal[x])     elif(float(anal[x])<0):
      n+=float(anal[x])

   rate=round(p/(n*(-1)+p),2)
   rate_string+=str(rate)+" "  
   fo.write(out+"\n")
   train_output+=str(train_reward)+" "
   # Test every 100 episodes
   if episode % 10 == 0:
     out="test\n"     
     env1=TWStock(my_test,label_test)
     total_reward = 0

for x in xrange(0,len(label_test)):
     sum_+=label_test[x]     if(x % 20==0 and x!=0):
         label_line=np.append(label_line,sum_)
         sum_=0
 label_line=np.delete(label_line,0)
 month_arr=np.delete(month_arr,0)
 buy_rate=np.delete(buy_rate,0) for x in xrange(0,len(month_arr)):        #month_arr[x]=month_arr[x]*buy_rate[x]
        label_line[x]=label_line[x]*buy_rate[x] print (buy_rate)
 plt.figure()
 plt.plot(label_line, 'ro',label='baseline',linewidth=1,color=[1,0,0])
 plt.title("monthly profit with baseline") #plt.plot(month_arr,label='predict',linewidth=5,color=[0,1,0])
 width = 0.6
 plt.bar([0,0.7,1.7,2.7,3.7,4.7,5.7,6.7,7.7,8.7,9.3],month_arr,width=width,label='predict') #plt.axis([0, 14, min_y, max_y])
 plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)

 plt.show()if __name__ == '__main__':
 main()

dimension: 1024
Successfully loaded: saved_year_r/network-dqn-652000
Evaluation Total Reward: 182.0
[0.55 0.35 0.6 0.55 0.4 0.6 0.7 0.45 0.3 0.15 0.0]

 def create_training_method(self):
    self.action_input = tf.placeholder("float",[None,self.action_dim])    # one hot presentation
    self.y_input = tf.placeholder("float",[None])
    Q_action = tf.reduce_sum(tf.mul(self.Q_value,self.action_input),reduction_indices = 1)
    self.cost = tf.reduce_mean(tf.square(self.y_input - Q_action))
    self.optimizer =  tf.train.RMSPropOptimizer(0.00025,0.99,0.0,1e-6).minimize(self.cost)    #tf.scalar_summary("cost", values=self.cost)
    #tf.histogram_summary("cost", values=self.cost)

  def perceive(self,state,action,reward,next_state,done):
    one_hot_action = np.zeros(self.action_dim)
    one_hot_action[action] = 1
    self.replay_buffer.append((state,one_hot_action,reward,next_state,done))    if len(self.replay_buffer) > REPLAY_SIZE:
      self.replay_buffer.popleft()    if len(self.replay_buffer) > BATCH_SIZE:
      self.train_Q_network()

from __future__ import absolute_importfrom __future__ import divisionfrom __future__ import print_functionimport mathimport sysimport reimport tempfileimport timeimport tensorflow as tffrom tensorflow.examples.tutorials.mnist import input_datafrom tensorflow.contrib import layersimport tensorflow.contrib.learn as skflow

flags = tf.app.flags
flags.DEFINE_string("data_dir", "./MNIST_data",                    "Directory for storing mnist data")
flags.DEFINE_boolean("download_only", False,                     "Only perform downloading of data; Do not proceed to "
                     "session preparation, model definition or training")#airflags.DEFINE_string("job_name", "","One of 'ps' or 'worker'")'''flags.DEFINE_string("ps_hosts", "10.81.103.124:7777","List of hostname:port for ps jobs."
            "This string should be the same on every host!!")
flags.DEFINE_string("worker_hosts", "10.81.103.124:2224,10.81.103.122:2222,10.81.103.118:2218,10.81.103.119:2219,10.81.103.120:2220,10.81.103.121:2221","List of hostname:port for worker jobs."
            "This string should be the same on every host!!")'''flags.DEFINE_string("ps_hosts", "10.81.103.122:7777","List of hostname:port for ps jobs."
            "This string should be the same on every host!!")
flags.DEFINE_string("worker_hosts", "10.81.103.122:2222,  10.81.103.122:2226","List of hostname:port for worker jobs."
            "This string should be the same on every host!!")

python DQN_kd_pic.py //this call KD_draw.py and build model.