Java快速读取大文件
最近公司服务器监控系统需要做一个东西来分析Java应用程序的日志。
第一步探索：
首先我想到的是使用RandomAccessFile，因为他可以很方便的去获取和设置文件指针，下面是我的代码。
package cn.mucang.exception.analyzer;
import cn.mucang.exception.analyzer.analyze.LogAnalyzer;
import cn.mucang.exception.analyzer.config.AnalyseConfig;
import cn.mucang.exception.analyzer.support.DefaultLogLineBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.RandomAccessFile;
/**
* @author Gao Youbo
* @since 2015/3/16.
*/
public class LogUtils {
private static final Logger LOG = LoggerFactory.getLogger(LogUtils.class);
/**
* 分析日志
*
* @param analyzer 分析器
* @throws IOException
*/
public static void analyse(LogAnalyzer analyzer) throws IOException {
AnalyseConfig analyseConfig = analyzer.getAnalyseConfig();
File file = new File(analyseConfig.getPath());
LOG.info("开始分析日志文件...{}", file.getAbsolutePath());
if (!file.exists()) {
throw new IOException("日志文件不存在:" + analyseConfig);
}
if (analyseConfig.getFilePointer() < 0) {
analyseConfig.setFilePointer(0);
}
FileInputStream stream = new FileInputStream(file);
InputStreamReader reader = new InputStreamReader(stream);
BufferedReader bufferedReader = new BufferedReader(reader);
try (RandomAccessFile logFile = new RandomAccessFile(file, "r")) {
long length = logFile.length();
analyzer.getAnalyseConfig().setFileLenght(length); //设置文件字节长度
if (analyseConfig.getFilePointer() > length) {
throw new IllegalArgumentException("开始指针位置越界");
} else {
logFile.seek(analyseConfig.getFilePointer());
}
String line; //行数据
int lineNumber = analyseConfig.getLineNumber(); //行号
DefaultLogLineBuilder lb = null;
long start = System.currentTimeMillis();
while ((line = logFile.readLine()) != null) {
bufferedReader.readLine();
lineNumber++;
long filePointer = logFile.getFilePointer();
if (ParseUtils.isNewLine(lineNumber, line)) {
if (lb != null) {
analyzer.analyse(lb.getLogLine());
}
lb = new DefaultLogLineBuilder();
}
if (lb != null) {
lb.append(lineNumber, filePointer, line);
if (length == logFile.getFilePointer()) { //文档读取完了，调用一下分析
analyzer.analyse(lb.getLogLine());
}
}
if (lineNumber % 10000 == 0) {
long end = System.currentTimeMillis();
System.out.println(String.format("line=%s, used=%sms", lineNumber, end - start));
start = System.currentTimeMillis();
}
}
}
}
}
下面看一下性能，分析一万行日志平均需要1500毫秒，因为我的日志分析使用到了正则，开始速度慢我以为是大量的正则运算造成的。
第二部探索：
我自己写了一个LogReader，自己控制指针位置。下面看一下代码：
package cn.mucang.exception.analyzer;
import java.io.BufferedReader;
import java.io.Closeable;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
/**
* @author Gao Youbo
* @since 2015-03-25 09:02
*/
public class LogReader implements Closeable {
/**
* 文件大小
*/
private long length;
/**
* 文件指针位置
*/
private long filePointer;
private FileInputStream inputStream;
private InputStreamReader inputStreamReader;
private BufferedReader bufferedReader;
public LogReader(File logFile) throws FileNotFoundException {
this.inputStream = new FileInputStream(logFile);
this.inputStreamReader = new InputStreamReader(inputStream);
this.bufferedReader = new BufferedReader(inputStreamReader);
this.length = logFile.length();
}
public int read() throws IOException {
filePointer++;
return bufferedReader.read();
}
public String readLine() throws IOException {
StringBuffer input = new StringBuffer();
int c = -1;
boolean eol = false; //end of line
while (!eol) {
switch (c = read()) {
case -1:
case '
':
eol = true;
break;
case '
':
eol = true;
long cur = getFilePointer();
if ((read()) != '
') {
skip(cur);
}
default:
input.append((char) c);
break;
}
}
if ((c == -1) && (input.length() == 0)) {
return null;
}
return input.toString();
}
/**
* 获取当前读取到的指针
*
* @return
* @throws IOException
*/
public long getFilePointer() throws IOException {
return filePointer;
}
/**
* 从当前位置跳过n个char
*
* @param n
* @return 实际跳过多少个char
* @throws IOException
*/
public long skip(long n) throws IOException {
return inputStreamReader.skip(n);
}
/**
* 返回日志文件的大小
*
* @return
*/
public long length() {
return length;
}
@Override
public void close() throws IOException {
if (bufferedReader != null) {
bufferedReader.close();
}
if (inputStreamReader != null) {
inputStreamReader.close();
}
if (inputStream != null) {
inputStream.close();
}
}
}
package cn.mucang.exception.analyzer;
import cn.mucang.exception.analyzer.analyze.LogAnalyzer;
import cn.mucang.exception.analyzer.config.AnalyseConfig;
import cn.mucang.exception.analyzer.support.DefaultLogLineBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
/**
* @author Gao Youbo
* @since 2015/3/16.
*/
public class LogUtils {
private static final Logger LOG = LoggerFactory.getLogger(LogUtils.class);
/**
* 分析日志
*
* @param analyzer 分析器
* @throws java.io.IOException
*/
public static void analyse(LogAnalyzer analyzer) throws IOException {
AnalyseConfig analyseConfig = analyzer.getAnalyseConfig();
File file = new File(analyseConfig.getPath());
System.out.println(file.getAbsolutePath());
LOG.info("开始分析日志文件...{}", file.getAbsolutePath());
if (!file.exists()) {
throw new IOException("日志文件不存在:" + analyseConfig);
}
if (analyseConfig.getFilePointer() < 0) {
analyseConfig.setFilePointer(0);
}
try (LogReader logReader = new LogReader(file)) {
long length = logReader.length();
analyzer.getAnalyseConfig().setFileLenght(length); //设置文件字节长度
if (analyseConfig.getFilePointer() > length) {
throw new IllegalArgumentException("开始指针位置越界");
} else {
logReader.skip(analyseConfig.getFilePointer());
}
String line; //行数据
int lineNumber = analyseConfig.getLineNumber(); //行号
DefaultLogLineBuilder lb = null;
long start = System.currentTimeMillis();
while ((line = logReader.readLine()) != null) {
lineNumber++;
long filePointer = logReader.getFilePointer();
if (ParseUtils.isNewLine(lineNumber, line)) {
if (lb != null) {
analyzer.analyse(lb.getLogLine());
}
lb = new DefaultLogLineBuilder();
}
if (lb != null) {
lb.append(lineNumber, filePointer, line);
if (length == filePointer) { //文档读取完了，调用一下分析
analyzer.analyse(lb.getLogLine());
}
}
if (lineNumber % 10000 == 0) {
long end = System.currentTimeMillis();
System.out.println(String.format("line=%s, used=%s", lineNumber, end - start));
start = System.currentTimeMillis();
}
}
}
}
}
接下来是测试的性能：
日志解析速度提高了10倍。