tensorflow 相关

1.从checkpoint中获取全部的变量名和变量值

tf.contrib.framework.list_variables(model_dir)

tf.contrib.framework.load_variable(model_dir, var_name)

2.清除 tf.Session

tf.reset_default_graph() 重置计算图

3。 使用tf_record

numpy数据可以直接制作dataset ds = tf.data.Dataset.from_tensor_slices(trg)

正常情况的话

a = np.random.randint(0,10,(10))
 2 b = np.random.rand(10,20)
 3 a1 = a.tobytes()
 4 b1 = b.tobytes()
 5 writer= tf.python_io.TFRecordWriter("./tfr/train.tfrecords")
 6 example = tf.train.Example(features=tf.train.Features(feature={
 7             "soft_targets": tf.train.Feature(bytes_list=tf.train.BytesList(value=[b1])),
 8             'src_wids': tf.train.Feature(bytes_list=tf.train.BytesList(value=[a1]))
 9         }))
10 for _ in range(10):
11     writer.write(example.SerializeToString())
12 writer.close()

 def _parse_function(example_proto):
 2     features = tf.parse_single_example(
 3         example_proto,
 4         features={
 5             'src_wids': tf.FixedLenFeature([], tf.string),
 6             'soft_targets': tf.FixedLenFeature([], tf.string)
 7         }
 8     )
 9     # 取出我们需要的数据（标签，图片）
10     label = features['soft_targets']
11     feature = features['src_wids']
12     label = tf.decode_raw(label, tf.float32)
13     feature = tf.decode_raw(feature, tf.int64)
14     return feature, label
15 
16 dataset = tf.contrib.data.TFRecordDataset("./tfr/train.tfrecords")
17 dataset = dataset.map(_parse_function)
18 dataset = dataset.batch(2)
19 iterator = dataset.make_initializable_iterator()
20 with tf.Session() as sess:
21     sess.run(iterator.initializer)
22     ids,trg = sess.run(iterator.get_next())

    #Author:Anthony  
    #导入相应的模块  
    import tensorflow as tf  
    import os  
    import random  
    import math  
    import sys  
    #划分验证集训练集  
    _NUM_TEST = 40  
    #random seed  
    _RANDOM_SEED = 0  
    #数据块  
    _NUM_SHARDS = 2  
    #数据集路径  
    DATASET_DIR = '/home/anthony/文档/数据集_带标签/SHIYAN_SAMEZIZE'  
    #标签文件  
    LABELS_FILENAME = '/home/anthony/文档/数据集_带标签/SHIYAN_SAMEZIZE_labels.txt'  
    #定义tfrecord 的路径和名称  
    def _get_dataset_filename(dataset_dir,split_name,shard_id):  
        output_filename = 'image_%s_%05d-of-%05d.tfrecord' % (split_name,shard_id,_NUM_SHARDS)  
        return os.path.join(dataset_dir,output_filename)  
    #判断tfrecord文件是否存在  
    def _dataset_exists(dataset_dir):  
        for split_name in ['train','test']:  
            for shard_id in range(_NUM_SHARDS):  
                #定义tfrecord的路径名字  
                output_filename = _get_dataset_filename(dataset_dir,split_name,shard_id)  
            if not tf.gfile.Exists(output_filename):  
                return False  
        return True  
    #获取图片以及分类  
    def _get_filenames_and_classes(dataset_dir):  
        #数据目录  
        directories = []  
        #分类名称  
        class_names = []  
        for filename in os.listdir(dataset_dir):  
            #合并文件路径  
            path = os.path.join(dataset_dir,filename)  
            #判断路径是否是目录  
            if os.path.isdir(path):  
                #加入数据目录  
                directories.append(path)  
                #加入类别名称  
                class_names.append(filename)  
        photo_filenames = []  
        #循环分类的文件夹  
        for directory in directories:  
            for filename in os.listdir(directory):  
                path = os.path.join(directory,filename)  
                #将图片加入图片列表中  
                photo_filenames.append(path)  
        #返回结果  
        return photo_filenames ,class_names  
    def int64_feature(values):  
        if not isinstance(values,(tuple,list)):  
            values = [values]  
        return tf.train.Feature(int64_list=tf.train.Int64List(value=values))  
    def bytes_feature(values):  
        return tf.train.Feature(bytes_list=tf.train.BytesList(value=[values]))  
    #图片转换城tfexample函数  
    def image_to_tfexample(image_data,image_format,class_id):  
        return tf.train.Example(features=tf.train.Features(feature={  
            'image/encoded': bytes_feature(image_data),  
            'image/format': bytes_feature(image_format),  
            'image/class/label': int64_feature(class_id)  
        }))  
    def write_label_file(labels_to_class_names,dataset_dir,filename=LABELS_FILENAME):  
        label_filename = os.path.join(dataset_dir,filename)  
        with tf.gfile.Open(label_filename,'w') as f:  
            for label in labels_to_class_names:  
                class_name = labels_to_class_names[label]  
                f.write('%d:%s
' % (label, class_name))  
    #数据转换城tfrecorad格式  
    def _convert_dataset(split_name,filenames,class_names_to_ids,dataset_dir):  
        assert split_name in ['train','test']  
        #计算每个数据块的大小  
        num_per_shard = int(len(filenames) / _NUM_SHARDS)  
        with tf.Graph().as_default():  
            with tf.Session() as sess:  
                for shard_id in range(_NUM_SHARDS):  
                #定义tfrecord的路径名字  
                    output_filename = _get_dataset_filename(dataset_dir,split_name,shard_id)  
                    with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:  
                        #每个数据块开始的位置  
                        start_ndx = shard_id * num_per_shard  
                        #每个数据块结束的位置  
                        end_ndx = min((shard_id+1) * num_per_shard,len(filenames))  
                        for i in range(start_ndx,end_ndx):  
                            try:  
                                sys.stdout.write('
>> Converting image %d/%d shard %d '% (i+1,len(filenames),shard_id))  
                                sys.stdout.flush()  
                                #读取图片  
                                image_data = tf.gfile.FastGFile(filenames[i],'rb').read()  
                                #获取图片的类别名称  
                                #basename获取图片路径最后一个字符串  
                                #dirname是除了basename之外的前面的字符串路径r  
                                class_name = os.path.basename(os.path.dirname(filenames[i]))  
                                #获取图片的id  
                                class_id = class_names_to_ids[class_name]  
                                #生成tfrecord文件  
                                example = image_to_tfexample(image_data,b'jpg',class_id)  
                                #写入数据  
                                tfrecord_writer.write(example.SerializeToString())  
                            except IOError  as e:  
                                print ('could not read:',filenames[1])  
                                print ('error:' , e)  
                                print ('skip it 
')  
        sys.stdout.write('
')  
        sys.stdout.flush()  
      
    if __name__ == '__main__':  
        #判断tfrecord文件是否存在  
        if _dataset_exists(DATASET_DIR):  
            print ('tfrecord exists')  
        else:  
            #获取图片以及分类  
            photo_filenames,class_names = _get_filenames_and_classes(DATASET_DIR)  
            #将分类的list转换成dictionary{‘house':3,'flowers:2'}  
            class_names_to_ids = dict(zip(class_names,range(len(class_names))))  
            #切分数据为测试训练集  
            random.seed(_RANDOM_SEED)  
            random.shuffle(photo_filenames)  
            training_filenames = photo_filenames[_NUM_TEST:]  
            testing_filenames = photo_filenames[:_NUM_TEST]  
            #数据转换  
            _convert_dataset('train',training_filenames,class_names_to_ids,DATASET_DIR)  
            _convert_dataset('test',testing_filenames,class_names_to_ids,DATASET_DIR)  
            #输出lables文件  
            #与前面的 class_names_to_ids中的元素位置相反{1:'people,2:'flowers'}  
            labels_to_class_names = dict(zip(range(len(class_names)),class_names))  
            write_label_file(labels_to_class_names,DATASET_DIR)

 4.tf,argmax(data, axis)

获取最大元素的索引

-------------------------------------------keras--------------------------------------

1 . Keras---text.Tokenizer: https://blog.csdn.net/lovebyz/article/details/77712003