避免对大文件全部读取到内存中,浪费时间,也能避免内存溢出;
先对文件先进行抽样,抽出很小一部分,测试程序的语法正确性,再用全部文件测试程序的功能正确性;
import tensorflow as tf import numpy as np import pandas as pd import matplotlib.pyplot as plt import csv
# 原始csv文件包含222个类,每个类500个样本,连续排列 num_sample = 10; with open("data/clean_data/normalized_training.csv", "rb") as fi: with open("data/clean_data/normalized_training_part.csv",'wb') as fo: fo.write(fi.readline()) # 读取csv文件column name行,也就是第一行 for i in range(222): for j in range(num_sample): fo.write(fi.readline()) for j in range(500-num_sample): fi.readline();
num_sample = 10;with open("data/clean_data/normalized_training.csv", "rb") as fi: with open("data/clean_data/normalized_training_part.csv",'wb') as fo: fo.write(fi.readline()) for i in range(222): for j in range(num_sample): fo.write(fi.readline()) for j in range(500-num_sample): fi.readline();