先在微信公众号上看到网文《将20M文件从30秒压缩到1秒,我是如何做到的?》,然后在网上搜索了一下,看到了原文:https://www.jianshu.com/p/2e46ccb125ef
很惊奇他把时间压缩到了三十分之一,于是就有了做实验验证的想法。
我的实验对象是apache-tomcat-9.0.30.zip,Redis-x64-3.2.100.msi,Redis-x64-3.2.100.zip这三个文件,加起来21M,比作者的十张2M图片略多。
任务是把它们三个压缩成一个result.zip文件。
首先我实验的是rugularZip方法,也是作者提到的第一种方法:
// time elapsed:1s652ms private boolean rugularZip(String[] fromFiles,String toFile) { File zipFile=new File(toFile); byte[] buffer=new byte[BUFFER_SIZE]; int readLen=0; try { ZipOutputStream zipOut=new ZipOutputStream(new FileOutputStream(zipFile)) ; for(String file:fromFiles) { File fileWillZip=new File(file); if(fileWillZip.exists()) { InputStream inputStream=new BufferedInputStream(new FileInputStream(fileWillZip)); String entryName=fileWillZip.getName();// entryName should be a valid filename,no path seperater allowed zipOut.putNextEntry(new ZipEntry(entryName)); while((readLen=inputStream.read(buffer,0,BUFFER_SIZE))!=-1) { zipOut.write(buffer,0,readLen); } inputStream.close(); } } zipOut.close(); }catch(Exception e) { e.printStackTrace(); return false; } return true; }
用时1秒652毫秒。
接下来实验的是bufferOuputZip方法,也就是作者提到的第二种方法:
// time elapsed:1s207ms private boolean bufferOuputZip(String[] fromFiles,String toFile) { File zipFile=new File(toFile); byte[] buffer=new byte[BUFFER_SIZE]; int readLen=0; try { ZipOutputStream zipOut=new ZipOutputStream(new FileOutputStream(zipFile)) ; BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(zipOut) ; for(String file:fromFiles) { File fileWillZip=new File(file); if(fileWillZip.exists()) { InputStream inputStream=new BufferedInputStream(new FileInputStream(fileWillZip)); String entryName=fileWillZip.getName();// entryName should be a valid filename,no path seperater allowed zipOut.putNextEntry(new ZipEntry(entryName)); while((readLen=inputStream.read(buffer,0,BUFFER_SIZE))!=-1) { bufferedOutputStream.write(buffer,0,readLen); } inputStream.close(); } } bufferedOutputStream.flush(); zipOut.close(); return true; }catch(Exception e) { e.printStackTrace(); return false; } }
用时1秒207毫秒,少了四分之一。
接下来实验作者提出的第三种方法:
// elapsed:1s188ms private boolean nioChannalZip(String[] fromFiles,String toFile) { File zipFile=new File(toFile); try { ZipOutputStream zipOut=new ZipOutputStream(new FileOutputStream(zipFile)) ; WritableByteChannel wChannel=Channels.newChannel(zipOut); for(String file:fromFiles) { File fileWillZip=new File(file); if(fileWillZip.exists()) { FileChannel readChannel=new FileInputStream(fileWillZip).getChannel(); String entryName=fileWillZip.getName();// entryName should be a valid filename,no path seperater allowed zipOut.putNextEntry(new ZipEntry(entryName)); readChannel.transferTo(0, readChannel.size(),wChannel) ; readChannel.close(); } } wChannel.close(); zipOut.close(); return true; }catch(Exception e) { e.printStackTrace(); return false; } }
一秒188毫秒,没少多少。
接下来再看作者又提到了内存映射文件,他又说和第三种方法速度差不多哦,算了我就不试了。
作者最后又提到pip,我一看原来是用线程,这当然快了,因为主线程只要调用压缩线程就可以返回了,压缩线程消耗的时间不计算在主线程运行时间内,运行当然是秒回。
当然pip里面有个阻塞回调的过程,这消耗了一些时间。
更新DB,写文件都可以另起线程运行,这也是以空间换时间的方法之一。
完整程序:
package zip; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.InputStream; import java.nio.channels.Channels; import java.nio.channels.FileChannel; import java.nio.channels.WritableByteChannel; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; // Used to zip a file public class FileZipper { private static final int BUFFER_SIZE = 1024; public boolean compressFilesToZip(String[] files,String zipfile) { return threadZip(files,zipfile); } private boolean threadZip(String[] fromFiles,String toFile) { new ZipThread(fromFiles,toFile).start();; return true; } // elapsed:1s188ms private boolean nioChannalZip(String[] fromFiles,String toFile) { File zipFile=new File(toFile); try { ZipOutputStream zipOut=new ZipOutputStream(new FileOutputStream(zipFile)) ; WritableByteChannel wChannel=Channels.newChannel(zipOut); for(String file:fromFiles) { File fileWillZip=new File(file); if(fileWillZip.exists()) { FileChannel readChannel=new FileInputStream(fileWillZip).getChannel(); String entryName=fileWillZip.getName();// entryName should be a valid filename,no path seperater allowed zipOut.putNextEntry(new ZipEntry(entryName)); readChannel.transferTo(0, readChannel.size(),wChannel) ; readChannel.close(); } } wChannel.close(); zipOut.close(); return true; }catch(Exception e) { e.printStackTrace(); return false; } } // time elapsed:1s207ms private boolean bufferOuputZip(String[] fromFiles,String toFile) { File zipFile=new File(toFile); byte[] buffer=new byte[BUFFER_SIZE]; int readLen=0; try { ZipOutputStream zipOut=new ZipOutputStream(new FileOutputStream(zipFile)) ; BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(zipOut) ; for(String file:fromFiles) { File fileWillZip=new File(file); if(fileWillZip.exists()) { InputStream inputStream=new BufferedInputStream(new FileInputStream(fileWillZip)); String entryName=fileWillZip.getName();// entryName should be a valid filename,no path seperater allowed zipOut.putNextEntry(new ZipEntry(entryName)); while((readLen=inputStream.read(buffer,0,BUFFER_SIZE))!=-1) { bufferedOutputStream.write(buffer,0,readLen); } inputStream.close(); } } bufferedOutputStream.flush(); zipOut.close(); return true; }catch(Exception e) { e.printStackTrace(); return false; } } // time elapsed:1s652ms private boolean rugularZip(String[] fromFiles,String toFile) { File zipFile=new File(toFile); byte[] buffer=new byte[BUFFER_SIZE]; int readLen=0; try { ZipOutputStream zipOut=new ZipOutputStream(new FileOutputStream(zipFile)) ; for(String file:fromFiles) { File fileWillZip=new File(file); if(fileWillZip.exists()) { InputStream inputStream=new BufferedInputStream(new FileInputStream(fileWillZip)); String entryName=fileWillZip.getName();// entryName should be a valid filename,no path seperater allowed zipOut.putNextEntry(new ZipEntry(entryName)); while((readLen=inputStream.read(buffer,0,BUFFER_SIZE))!=-1) { zipOut.write(buffer,0,readLen); } inputStream.close(); } } zipOut.close(); }catch(Exception e) { e.printStackTrace(); return false; } return true; } /** * change seconds to DayHourMinuteSecond format * * @param startMs * @param endMs * @return */ private static String ms2DHMS(long startMs, long endMs) { String retval = null; long secondCount = (endMs - startMs) / 1000; String ms = (endMs - startMs) % 1000 + "ms"; long days = secondCount / (60 * 60 * 24); long hours = (secondCount % (60 * 60 * 24)) / (60 * 60); long minutes = (secondCount % (60 * 60)) / 60; long seconds = secondCount % 60; if (days > 0) { retval = days + "d" + hours + "h" + minutes + "m" + seconds + "s"; } else if (hours > 0) { retval = hours + "h" + minutes + "m" + seconds + "s"; } else if (minutes > 0) { retval = minutes + "m" + seconds + "s"; } else if(seconds > 0) { retval = seconds + "s"; }else { return ms; } return retval + ms; } public static String calculateElaspedTime(long startMs) { long endMs = System.currentTimeMillis(); return ms2DHMS(startMs,endMs); } public static void main(String[] args) { String[] files= {"D:\usr\apache-tomcat-9.0.30.zip", "D:\usr\Redis-x64-3.2.100.msi", "D:\usr\Redis-x64-3.2.100.zip"}; String zipfile="D:\usr\result.zip"; long startMs = System.currentTimeMillis(); FileZipper fz=new FileZipper(); boolean isCreated=fz.compressFilesToZip(files, zipfile); if(isCreated) { System.out.println("File:'"+zipfile+"' created,time elapsed:"+calculateElaspedTime(startMs)); } } }
package zip; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.nio.channels.Channels; import java.nio.channels.FileChannel; import java.nio.channels.WritableByteChannel; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; public class ZipThread extends Thread{ private String[] files; private String zipfile; public ZipThread(String[] files,String zipfile) { this.files=files; this.zipfile=zipfile; } public void run() { nioChannalZip(this.files,this.zipfile); } private boolean nioChannalZip(String[] fromFiles,String toFile) { File zipFile=new File(toFile); try { ZipOutputStream zipOut=new ZipOutputStream(new FileOutputStream(zipFile)) ; WritableByteChannel wChannel=Channels.newChannel(zipOut); for(String file:fromFiles) { File fileWillZip=new File(file); if(fileWillZip.exists()) { FileChannel readChannel=new FileInputStream(fileWillZip).getChannel(); String entryName=fileWillZip.getName();// entryName should be a valid filename,no path seperater allowed zipOut.putNextEntry(new ZipEntry(entryName)); readChannel.transferTo(0, readChannel.size(),wChannel) ; readChannel.close(); } } wChannel.close(); zipOut.close(); return true; }catch(Exception e) { e.printStackTrace(); return false; } } }
--END-- 2020-01-06 14:54