''' Created on 2017年5月13日 @author: weizhen ''' import numpy as np import tensorflow as tf import ptb_iterator as reader from tensorflow.contrib import rnn DATA_PATH = "/path/to/ptb/data" # 数据存放的路径 HIDDEN_SIZE = 200 # 隐藏层的规模 NUM_LAYERS = 2 # 深层循环神经网络中LSTM结构的层数 VOCAB_SIZE = 10000 # 词典规模，加上语句结束标识符和稀有单词标识符总共一万个单词 LEARNING_RATE = 1.0 # 学习速率 TRAIN_BATCH_SIZE = 20 # 训练数据batch的大小 TRAIN_NUM_STEP = 35 # 训练数据截断长度 # 在测试时不需要使用截断，所以可以将测试数据看成一个超长的序列 EVAL_BATCH_SIZE = 1 # 测试数据batch的大小 EVAL_NUM_STEP = 1 # 测试数据截断长度 NUM_EPOCH = 2 # 使用训练数据的轮数 KEEP_PROB = 0.5 # 节点不被dropout的概率 MAX_GRAD_NORM = 5 # 用于控制梯度膨胀的参数 def LstmCell(is_training): lstm_cell = rnn.BasicLSTMCell(HIDDEN_SIZE,reuse=tf.get_variable_scope().reuse) if is_training: lstm_cell = rnn.DropoutWrapper(lstm_cell, output_keep_prob=KEEP_PROB) return lstm_cell # 通过一个PTBModel类来描述模型，这样方便维护循环神经网络中的状态 class PTBModel(object): def __init__(self, is_training, batch_size, num_steps): # 记录使用的batch大小和截断长度 self.batch_size = batch_size self.num_steps = num_steps # 定义输入层。可以看到输入层的维度为batch_size*num_steps,这和 # ptb_iterator函数输出的训练数据batch是一致的 self.input_data = tf.placeholder(tf.int32, [batch_size, num_steps]) # 定义预期输出，它的维度和ptb_iterator函数输出的正确答案维度也是一样的 self.targets = tf.placeholder(tf.int32, [batch_size, num_steps]) # 定义使用LSTM结构为循环体结构且使用dropout的深层循环神经网络 # lstm_cell = rnn.BasicLSTMCell(HIDDEN_SIZE) # if is_training: # lstm_cell = rnn.DropoutWrapper(lstm_cell, output_keep_prob=KEEP_PROB) cell = rnn.MultiRNNCell([LstmCell(is_training) for _ in range(NUM_LAYERS)]) # 初始化最初的状态，也就是全零的向量 self.initial_state = cell.zero_state(batch_size, tf.float32) # 将单词ID转换成为单词向量。因为总共有VOCAB_SIZE个单词，每个单词向量的维度为HIDDEN_SIZE # 所以embedding参数的维度为VOCAB_SIZE*HIDDEN_SIZE embedding = tf.get_variable("embedding", [VOCAB_SIZE, HIDDEN_SIZE]) # 将原本batch_size*num_steps个单词ID转化为单词向量，转换后的输入层维度为batch_size*num_steps*HIDDEN_SIZE inputs = tf.nn.embedding_lookup(embedding, self.input_data) # 只在训练时使用dropout if is_training: inputs = tf.nn.dropout(inputs, KEEP_PROB) # 定义输出列表，在这里先将不同时刻LSTM结构的输出收集起来，再通过一个全连接层得到最终的输出 outputs = [] # state存储不同batch种LSTM的状态，将其初始化为0 state = self.initial_state with tf.variable_scope("RNN"): for time_step in range(num_steps): if time_step > 0: tf.get_variable_scope().reuse_variables() # 从输入数据中获取当前时刻的输入并传入LSTM结构 cell_output, state = cell(inputs[:, time_step, :], state) #cell_output, state = tf.nn.dynamic_rnn(cell,inputs[:, time_step, :], state,time_major=False) # 当前输出加入输出队列 outputs.append(cell_output) # 把输出队列展开成[batch*hidden_size*num_steps]的形状，然后再 # reshape成[batch*numsteps,hidden_size]的形状 output = tf.reshape(tf.concat(outputs, 1), [-1, HIDDEN_SIZE]) # 将从LSTM中得到的输出再经过一个全连接层得到最后的预测结果，最终的预测结果在每一个时刻上都是一个长度为VOCAB_Size的数组， # 经过softmax层之后表示下一个位置是不同单词的概率 weight = tf.get_variable("weight", [HIDDEN_SIZE, VOCAB_SIZE]) bias = tf.get_variable("bias", [VOCAB_SIZE]) logits = tf.matmul(output, weight) + bias # 定义交叉熵损失函数。TensorFlow提供了sequence_loss_by_example函数来计算一个序列的交叉熵的和 loss = tf.contrib.legacy_seq2seq.sequence_loss_by_example( [logits], # 预测的结果 [tf.reshape(self.targets, [-1])], # 期待的正确答案，这里讲[batch_size,num_steps]二维数组压缩成一维数组 [tf.ones([batch_size * num_steps], dtype=tf.float32)] # 损失的权重，在这里所有的权重都为1，也就是说不同batch和不同时刻的重要程度是一样的 ) # 计算得到每个batch的平均损失 self.cost = tf.reduce_sum(loss) / batch_size self.final_state = state # 只在训练模型时定义方向传播操作 if not is_training: return trainable_variables = tf.trainable_variables() # 通过clip_by_global_norm函数控制梯度的大小，避免梯度膨胀的问题 grads, _ = tf.clip_by_global_norm(tf.gradients(self.cost, trainable_variables), MAX_GRAD_NORM) # 定义优化方法 optimizer = tf.train.GradientDescentOptimizer(LEARNING_RATE) # 定义训练步骤 self.train_op = optimizer.apply_gradients(zip(grads, trainable_variables)) # 使用给定的模型model在数据data上运行train_op并返回在全部数据上的perplexity值 def run_epoch(session, model, data, train_op, output_log): # 计算perplexity的辅助变量。 total_costs = 0.0 iters = 0 state = session.run(model.initial_state) # 训练一个epoch。 for step,(x,y) in enumerate(reader.ptb_iterator(data,model.batch_size,model.num_steps)): #在当前batch上运行train_op并计算损失值。交叉熵损失函数计算的就是下一个单词为给定单词的概率 cost,state,_ = session.run([model.cost,model.final_state,train_op], {model.input_data:x,model.targets:y, model.initial_state:state}) #将不同时刻，不同batch的概率加起来就可以得到第二个perplexity公司等号右边的部分， #再将这个和做指数运算就可以得到perplexity值 total_costs+=cost iters+=model.num_steps #只有在训练时输出日志 if output_log and step % 100 == 0: print("after % step ,perplexity is %.3f" %(step,np.exp(total_costs/iters))) #返回给定模型在给定数据上的perplexity值 return np.exp(total_costs/iters) def main(_): # 获取原始数据 train_data, valid_data, test_data, _ = reader.ptb_raw_data(DATA_PATH) # 计算一个epoch需要训练的次数 #train_data_len = len(train_data) #train_batch_len = train_data_len # # TRAIN_BATCH_SIZE #train_epoch_size = (train_batch_len - 1) # # TRAIN_NUM_STEP #valid_data_len = len(valid_data) #valid_batch_len = valid_data_len # # EVAL_BATCH_SIZE #valid_epoch_size = (valid_batch_len - 1) # # EVAL_NUM_STEP #test_data_len = len(test_data) #test_batch_len = test_data_len # # EVAL_BATCH_SIZE #test_epoch_size = (test_batch_len - 1) # # EVAL_NUM_STEP # 定义初始化函数 initializer = tf.random_uniform_initializer(-0.05, 0.05) with tf.variable_scope("language_model", reuse=None, initializer=initializer): train_model = PTBModel(True, TRAIN_BATCH_SIZE, TRAIN_NUM_STEP) # 定义评测用的循环神经网络模型 with tf.variable_scope("language_model", reuse=True, initializer=initializer): eval_model = PTBModel(False, EVAL_BATCH_SIZE, EVAL_NUM_STEP) with tf.Session() as session: tf.global_variables_initializer().run() # 使用训练数据训练模型 for i in range(NUM_EPOCH): print("In iteration:%d" % (i + 1)) # 在所有训练数据上训练循环神经网络模型 run_epoch(session, train_model, train_data, train_model.train_op, True) # 使用验证数据评测模型效果 valid_perplexity = run_epoch(session, eval_model, valid_data, tf.no_op(), False) print("Epoch: %d Validation Perplexity : %.3f" % (i + 1, valid_perplexity)) # 最后使用测试数据测试模型效果 test_perplexity = run_epoch(session, eval_model, test_data, tf.no_op(), False) print("Test Perplexity:%.3f" % test_perplexity) if __name__ == "__main__": tf.app.run()
不过感觉很奇怪，就训练了两轮就结束了
2017-05-21 10:47:16.695751: W c: f_jenkinshomeworkspace elease-windevicecpuoswindows ensorflowcoreplatformcpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE instructions, but these are available on your machine and could speed up CPU computations. 2017-05-21 10:47:16.696456: W c: f_jenkinshomeworkspace elease-windevicecpuoswindows ensorflowcoreplatformcpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE2 instructions, but these are available on your machine and could speed up CPU computations. 2017-05-21 10:47:16.696955: W c: f_jenkinshomeworkspace elease-windevicecpuoswindows ensorflowcoreplatformcpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE3 instructions, but these are available on your machine and could speed up CPU computations. 2017-05-21 10:47:16.697919: W c: f_jenkinshomeworkspace elease-windevicecpuoswindows ensorflowcoreplatformcpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE4.1 instructions, but these are available on your machine and could speed up CPU computations. 2017-05-21 10:47:16.698685: W c: f_jenkinshomeworkspace elease-windevicecpuoswindows ensorflowcoreplatformcpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE4.2 instructions, but these are available on your machine and could speed up CPU computations. 2017-05-21 10:47:16.699159: W c: f_jenkinshomeworkspace elease-windevicecpuoswindows ensorflowcoreplatformcpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use AVX instructions, but these are available on your machine and could speed up CPU computations. 2017-05-21 10:47:16.699770: W c: f_jenkinshomeworkspace elease-windevicecpuoswindows ensorflowcoreplatformcpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use AVX2 instructions, but these are available on your machine and could speed up CPU computations. 2017-05-21 10:47:16.700265: W c: f_jenkinshomeworkspace elease-windevicecpuoswindows ensorflowcoreplatformcpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use FMA instructions, but these are available on your machine and could speed up CPU computations. In iteration:1 after 0tep ,perplexity is 9962.385 Epoch: 1 Validation Perplexity : 2167.129 In iteration:2 after 0tep ,perplexity is 5994.104 Epoch: 2 Validation Perplexity : 417.495 Test Perplexity:418.547
下面是用到的解析ptb数据的工具类ptb_reader
# Copyright 2015 The TensorFlow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== """Utilities for parsing PTB text files.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function import collections import os import numpy as np import tensorflow as tf def _read_words(filename): with tf.gfile.GFile(filename, "r") as f: return f.read().replace(" ", "<eos>").split() #读取文件， 将换行符替换为 <eos>， 然后将文件按空格分割。 返回一个 1-D list def _build_vocab(filename): #用于建立字典 data = _read_words(filename) counter = collections.Counter(data) #输出一个字典： key是word， value是这个word出现的次数 count_pairs = sorted(counter.items(), key=lambda x: (-x[1], x[0])) #counter.items() 会返回一个tuple列表， tuple是(key, value), 按 value的降序，key的升序排列 words, _ = list(zip(*count_pairs)) #感觉这个像unzip 就是把key放在一个tuple里，value放在一个tuple里 word_to_id = dict(zip(words, range(len(words))))#对每个word进行编号， 按照之前words输出的顺序（value降序，key升序） return word_to_id #返回dict， key：word， value：id def _file_to_word_ids(filename, word_to_id): #将file表示为word_id的形式 data = _read_words(filename) return [word_to_id[word] for word in data] def ptb_raw_data(data_path=None): """Load PTB raw data from data directory "data_path". Reads PTB text files, converts strings to integer ids, and performs mini-batching of the inputs. The PTB dataset comes from Tomas Mikolov's webpage: http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz Args: data_path: string path to the directory where simple-examples.tgz has been extracted. Returns: tuple (train_data, valid_data, test_data, vocabulary) where each of the data objects can be passed to PTBIterator. """ train_path = os.path.join(data_path, "ptb.train.txt") valid_path = os.path.join(data_path, "ptb.valid.txt") test_path = os.path.join(data_path, "ptb.test.txt") word_to_id = _build_vocab(train_path) #使用训练集确定word id train_data = _file_to_word_ids(train_path, word_to_id) valid_data = _file_to_word_ids(valid_path, word_to_id) test_data = _file_to_word_ids(test_path, word_to_id) vocabulary = len(word_to_id)#字典的大小 return train_data, valid_data, test_data, vocabulary def ptb_iterator(raw_data, batch_size, num_steps): """Iterate on the raw PTB data. This generates batch_size pointers into the raw PTB data, and allows minibatch iteration along these pointers. Args: raw_data: one of the raw data outputs from ptb_raw_data. batch_size: int, the batch size. num_steps: int, the number of unrolls. Yields: Pairs of the batched data, each a matrix of shape [batch_size, num_steps]. The second element of the tuple is the same data time-shifted to the right by one. Raises: ValueError: if batch_size or num_steps are too high. """ raw_data = np.array(raw_data, dtype=np.int32)#raw data : train_data | vali_data | test data data_len = len(raw_data) #how many words in the data_set batch_len = data_len // batch_size data = np.zeros([batch_size, batch_len], dtype=np.int32)#batch_len 就是几个word的意思 for i in range(batch_size): data[i] = raw_data[batch_len * i:batch_len * (i + 1)] epoch_size = (batch_len - 1) // num_steps if epoch_size == 0: raise ValueError("epoch_size == 0, decrease batch_size or num_steps") for i in range(epoch_size): x = data[:, i*num_steps:(i+1)*num_steps] y = data[:, i*num_steps+1:(i+1)*num_steps+1] yield (x, y)
ptb数据集放置在C盘的根目录下
