java 爬虫：开源java爬虫 swing工具 Imgraber

1实现点：

• 1.返回给定URL网页内，所有图像url list
• 2.返回给定URL网页内，自动生成图像文件路径.txt 文件
• 3.返回给定URL网页内，下载txt文件指定的图片url,并将所有图像保存在 ./img文件夹下
• 4.实现简易swing 界面，有空再改造

2基于开源jsoup实现，鸣谢！

效果

github

imgraber

package himi.crawler;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.jsoup.Jsoup;
import org.jsoup.helper.StringUtil;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;

/**
 * 
 * @ClassName: ImgTool
 * @Description:IMG tool ，请先赋值URL 获取指定ULR的某个网页里面 
 * 					1.返回所有图像url list 
 * 					2.图像文件夹路径.txt
 *                  3.所有图像保存在img文件夹里面
 * @author penny
 * @date 2018年3月7日 下午10:07:49
 *
 */
public class ImgTool {
	// 参数域
	/** 指定URL */
	public static String URL = "https://www.oschina.net";
	private int imgNumbs = 0;
	public static List<String> downloadMsg=new ArrayList<String>();
	public String imgUrlTxt = "imgURLs.txt";
	public static String regex= "^((https|http|ftp|rtsp|mms)?://)" 
								+ "?(([0-9a-z_!~*'().&=+$%-]+: )?[0-9a-z_!~*'().&=+$%-]+@)?" //ftp的user@ 
								+ "(([0-9].)[0-9]" // IP形式的URL- 199.194.52.184 
								+ "|" // 允许IP和DOMAIN（域名）
								+ "([0-9a-z_!~*'()-]+.)*" // 域名-  
								+ "([0-9a-z][0-9a-z-])?[0-9a-z]." // 二级域名 
								+ "[a-z])" // first level domain- .com or .museum 
								+ "(:[0-9])?" // 端口- :80 
								+ "((/?)|" // a slash isn't required if there is no file name 
								+ "(/[0-9a-z_!~*'().;?:@&=+$,%#-]+)+/?)$";
	private ImgTool() {
	};
	
	private static ImgTool instance = new ImgTool();

	/** 获取ImgTool 单例 */
	public static ImgTool getInstance() {
		return instance;
	}

	public List<String> getURLs() {
		return getURLs(null);
	}
	public boolean isURL(String str) {
		if(StringUtil.isBlank(str)){
			return false;
		}else{
//		String regex = "^(?:https?://)?[\w]{1,}(?:\.?[\w]{1,})+[\w-_/?&=#%:]*$";
//		String regex = "^([hH][tT]{2}[pP]:/*|[hH][tT]{2}[pP][sS]:/*|[fF][tT][pP]:/*)(([A-Za-z0-9-~]+).)+([A-Za-z0-9-~\/])+(\?{0,1}(([A-Za-z0-9-~]+\={0,1})([A-Za-z0-9-~]*)\&{0,1})*)$";
		Pattern pattern = Pattern.compile(regex);
		Matcher matcher = pattern.matcher(str);
		if(matcher.matches()){
			return true;
		}else{
			return false;
		}
	}};
	/***
	 * @Title: getURLs
	 * @Description: 给定cssQuery对象
	 * @param @param cssQuery HTML中的CSS(或者 JQuery)选择器语法，更多详细用法见Jsoup介绍 < a
	 *        href="https://jsoup.org/apidocs/org/jsoup/select/Selector.html"
	 *        ></a>
	 * @param @return List
	 * @throws
	 * 
	 */
	public List<String> getURLs(String cssQuery) {
		List<String> urls = null;
		Document doc;
		Elements imgElements ;
		if (!isURL(URL)) {
			return null;
		}
		if(StringUtil.isBlank(cssQuery)){
			cssQuery="img";
		}
		try {
			doc = Jsoup.connect(URL).get();
		} catch (IOException e) {
			e.printStackTrace();
			return null;
		}
		if(doc==null)return null;
		imgElements = doc.select(cssQuery);
		urls = new ArrayList<String>();
		for (Object eleObj : imgElements) {
			//"(https?|ftp|http)://[-A-Za-z0-9+&@#/%?=~_|!:,.;]+[-A-Za-z0-9+&@#/%=~_|]"
			Pattern pattern = Pattern.compile("(https?|http)://[-A-Za-z0-9+&@#/%?=~_|!:,.;]+[-A-Za-z0-9+&@#/%=~_|]");
			Matcher matcher = pattern.matcher(eleObj.toString());
			if (matcher.find()) {
				String url = matcher.group();
				urls.add(url);
			}
		}
		imgNumbs = imgElements.size();
		return urls;
	}


	/**
	 * 
	 * @Title: createImgURLTxt
	 * @Description:
	 * @param @param cssQuery：默认使用img HTML中的CSS(或者 JQuery)选择器语法，更多详细用法见Jsoup介绍 <
	 *        a href="https://jsoup.org/apidocs/org/jsoup/select/Selector.html">
	 *        </a>
	 * @throws 生成imgURLs.txt
	 */
	public String createImgURLTxt(String cssQuery) {
		long start = System.currentTimeMillis();
		List<String> urls;
		urls = getURLs(cssQuery);
		BufferedWriter os = null;
		File urlsFiles = new File("imgURLs.txt");
		if(urlsFiles.exists()){
			try {
				urlsFiles.createNewFile();
			} catch (IOException e) {
				e.printStackTrace();
			}
		}
		try {
			os = new BufferedWriter(new FileWriter(urlsFiles));
			if(urls==null)return null;
			for (int i = 0; i < urls.size(); i++) {
				os.write(urls.get(i) + "
");
			}
			String result = "执行完毕,生成imgURLs.txt,耗时"
					+ (System.currentTimeMillis() - start) / 1000 + "s";
			System.out.println(result);
			return result;
		} catch (IOException e) {
			e.printStackTrace();
		} finally {
			try {
				if (os != null)
					os.close();

			} catch (IOException e) {
				e.printStackTrace();
			}
		}
		return null;
	}

	/**
	 * 
	 * @Title: createImgs 
	 * @Description: 
	 * @param @param cssQuery
	 * @param @throws IOException    
	 * @throws
	 */
	public void createImgs(String cssQuery) throws IOException {
		long startTime = System.currentTimeMillis();
		downloadMsg.add("Your images is downloading ");
		BufferedReader br = null;
		OutputStream out = null;
		InputStream in = null;
		ArrayList<String> imgList = null;
		HttpURLConnection con = null;
		
		String url;			 	//待下载文件url
		int fileSize = 0;		//单个文件大小
		int totalFileNum=0;		//总文件数
		int downLoadFileNum=0;	//已下载文件
		long totalTime=0;		//总耗时/s
		long singleTime=0;		//单个文件耗时/ms
		
		br = new BufferedReader(new FileReader(imgUrlTxt));
		imgList = new ArrayList<String>();
		while ((url = br.readLine()) != null) {
			imgList.add(url);
		}
		downLoadFileNum=totalFileNum= imgList.size();
		downloadMsg.add("总文件数"+(totalFileNum));
		for (String listUrl : imgList) {
			startTime = System.currentTimeMillis();
			String fileName = listUrl.substring(listUrl.lastIndexOf('/') + 1);// 截取文件名
			URL imgUrl = new URL(listUrl.trim());
			if (con != null)
				con.disconnect();
			con = (HttpURLConnection) imgUrl.openConnection();
			con.setRequestMethod("GET");
			con.setDoInput(true);
			con.setConnectTimeout(1000 * 30);
			con.setReadTimeout(1000 * 30);
			fileSize = con.getContentLength();
			con.connect();
			try {
				in = con.getInputStream();
				File file = new File("img" + File.separator, fileName);
				if (!file.exists()) {
					file.getParentFile().mkdirs();
					file.createNewFile();
				}
				out = new BufferedOutputStream(new FileOutputStream(file));
				int len = 0;
				byte[] buff = new byte[1024 * 1024];
				while ((len = new BufferedInputStream(in).read(buff)) != -1) {
					out.write(buff, 0, len);
				}
				out.flush();
				downLoadFileNum--;
			} catch (Exception e) {
				e.printStackTrace();
			} finally {
				if (br != null)
					try {
						br.close();
					} catch (IOException e) {
						e.printStackTrace();
					}
				if (in != null)
					try {
						in.close();
					} catch (IOException e) {
						e.printStackTrace();
					}
				if (out != null)
					try {
						out.close();
					} catch (IOException e) {
						e.printStackTrace();
					}
				singleTime=System.currentTimeMillis() - startTime;
				totalTime+=singleTime;
				downloadMsg.add("文件名" + fileName + "  文件大小" + fileSize
								  +"  未下载文件数"+(downLoadFileNum)
								  +"  下载耗时"+ singleTime + "ms" );
				System.out.println(downloadMsg.get(downloadMsg.size()-1));
			}
		}
		downloadMsg.add("总耗时"+totalTime/1000+"s");
	}

	/**
	 * @throws IOException
	 * 
	 * @Title: main
	 * @Description: test
	 * @param @param args
	 * @throws
	 */
	public static void main(String[] args) throws IOException {
		ImgTool img = ImgTool.getInstance();
		 img.createImgURLTxt("img");
		 List<String> urls = img.getURLs("img");
		 for (Object str : urls) {
			System.out.println(str.toString());
		}
		img.createImgs(null);
	}
}