宽度优先爬虫-爬虫学习（四）

　　我们用爬虫去遍历互联网的时候，可以将互联网看作是一个有向图，链接就是图中的有向边，所以我们可以利用图的遍历方法去遍历这个巨大的互联网，图的遍历有宽度优先遍历和深度优先遍历。但是深度优先遍历可能会在遍历的时候遍历过深，导致浪费资源，所以我们这里采用宽度优先遍历。

　　下面使用Java实现一个简单的宽度优先爬虫的例子，其中用到来HttpClient和HtmlParser两个开源的工具包。

/**
 * @introduction 队列 保留将要访问的URL 
 * @author Guo
 */
public class Queue {
    /**
     * @introduction 底层使用LinkedList去封装一个队列，用来存储未访问的URL
     */
    private LinkedList<String> queue = new LinkedList<String>();
    
    /**
     * @introduction 想队列中加入URL
     * @param url
     */
    public void in(String url) {
        queue.addLast(url);
    }
    
    /**
     * @introduction 移除队列中的URL
     * @return
     */
    public String out() {
        return queue.removeFirst();
    }
    
    /**
     * @introduction 判断是否包含URL
     * @param url
     * @return
     */
    public boolean contains(String url) {
        return queue.contains(url);
    }
    
    /**
     * @introduction 判断队列是否为空
     * @return
     */
    public boolean isEmpty() {
        return queue.isEmpty();
    }
}

/**
 * @introduction 一个处理用过的URL和没用过的URL的类
 * @author Guo
 */
public class LinkQueue {
    /**
     * @introduction 用HashSet去封装一个访问过的URL队列
     */
    private static Set<String> visitedURL = new HashSet<String>();
    /**
     * @introduction 未访问的URL队列
     */
    private static Queue unVisitedURL = new Queue();
    
    /**
     * @introduction 添加已经访问过的URL
     * @param url
     */
    public static void addVisitedURL(String url) {
        visitedURL.add(url);
    }
    
    /**
     * @introduction 移除访问过的URL
     * @param url
     */
    public static void removeVisitedURL(String url) {
        visitedURL.remove(url);
    }
    
    /**
     * @introduction 拿出未访问的URL
     * @return
     */
    public static String unVisitedURLOutQueue() {
        return unVisitedURL.out();
    }
    
    /**
     * @introduction 添加未访问的URL
     * @param url
     */
    public static void addUnVisitedURL(String url) {
        if(url != null && !url.trim().equals("") && 
                !visitedURL.contains(url) && !unVisitedURL.contains(url)) {
            unVisitedURL.in(url);
        }
    }
    
    /**
     * @introduction 拿到访问过的URL的数目
     * @return
     */
    public static int getVisitedURLNumber() {
        return visitedURL.size();
    }

    public static Set<String> getVisitedURL() {
        return visitedURL;
    }

    public static void setVisitedURL(Set<String> visitedURL) {
        LinkQueue.visitedURL = visitedURL;
    }

    public static Queue getUnVisitedURL() {
        return unVisitedURL;
    }

    public static void setUnVisitedURL(Queue unVisitedURL) {
        LinkQueue.unVisitedURL = unVisitedURL;
    }
}

public class DownloadFile {
    /**
     * @introduction 将文件存储到本地
     * @param input
     * @param filePath
     */
    public void saveToLocal(InputStream input, String filePath) {
        try {
            DataOutputStream out = new DataOutputStream(new FileOutputStream(new File(filePath)));
            byte[] data = new byte[1024];
            int l;
            while ((l = input.read(data)) != -1) {
                out.write(data);
            }
            out.flush();
            out.close();
            input.close();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
    
    /**
     * @introduction 下载该链接的URL
     * @param url
     * @return
     */
    public String downloadFile(String url) {
        String filePath = null;
        try {
            HttpClient httpClient = new DefaultHttpClient();
            HttpGet httpGet = new HttpGet(url);
            HttpResponse response = httpClient.execute(httpGet);
            HttpEntity entity = response.getEntity();
            if(entity != null) {
                InputStream input = entity.getContent();
                filePath = "G:/temp/" + response.getAllHeaders().toString() + ".html";
                saveToLocal(input, filePath);
            }
            return filePath;
        }catch(Exception e) {
            e.printStackTrace();
            return filePath;
        }
    }
}

/**
 * @introduction 用来过滤链接的Filter 
 * @author Guo
 */
public interface LinkFilter {
    public boolean accept(String url);
}

/**
 * @introduction 专门对HTML页面进行处理 
 * @author Guo
 */
public class HtmlParserTool {
    /**
     * @introduction 根据url这个参数，拿到这个url中的链接
     * @param url
     * @param filter
     * @return
     */
    public static Set<String> getLinks(String url, LinkFilter filter) {
        Set<String> links = new HashSet<String>();
        try {
            Parser parser = new Parser(url);
            parser.setEncoding("utf-8");
            NodeFilter frameFilter = new NodeFilter() {
                public boolean accept(Node node) {
                    if(node.getText().startsWith("frame src=")) {
                        return true;
                    }else {
                        return false;
                    }
                }
            };
            OrFilter linkFilter = new OrFilter(new NodeClassFilter(LinkTag.class), frameFilter);
            NodeList list = parser.extractAllNodesThatMatch(linkFilter);
            for(int i = 0; i < list.size(); i++) {
                Node tag = list.elementAt(i);
                if(tag instanceof LinkTag) {
                    LinkTag link = (LinkTag)tag;
                    String linkUrl = link.getLink();
                    if(filter.accept(linkUrl))
                        links.add(linkUrl);
                }else {
                    String frame = tag.getText();
                    int start = frame.indexOf("src=");
                    frame = frame.substring(start);
                    int end = frame.indexOf(" ");
                    if(end == -1)
                        end = frame.indexOf(">");
                    String frameUrl = frame.substring(5, end - 1);
                    if(filter.accept(frameUrl))
                        links.add(frameUrl);
                }
            }
        }catch(Exception e) {
            e.printStackTrace();
        }
        return links;
    }
}

public class MyCrawler {
    /**
     * @introduction 初始化LinkQueue
     * @param seeds
     */
    private void initCrawlerWithSeeds(String[] seeds) {
        for(String seed : seeds) 
            LinkQueue.addUnVisitedURL(seed);
    }
    
    /**
     * @introduction 爬虫具体的爬行
     * @param seeds
     */
    public void crawling(String[] seeds) {
        LinkFilter filter = new LinkFilter() {
            public boolean accept(String url) {
                if(url.startsWith("http://www.baidu.com"))
                    return true;
                else
                    return false;
            }
        };
        
        initCrawlerWithSeeds(seeds);
        
        while(!LinkQueue.getUnVisitedURL().isEmpty() && LinkQueue.getVisitedURLNumber() <= 1000) {
            String visitUrl = LinkQueue.unVisitedURLOutQueue();
            if(visitUrl == null)
                continue;
            DownloadFile downloader = new DownloadFile();
            downloader.downloadFile(visitUrl);
            LinkQueue.addVisitedURL(visitUrl);
            Set<String> links = HtmlParserTool.getLinks(visitUrl, filter);
            for(String link : links) 
                LinkQueue.addUnVisitedURL(link);
        }
    }
    
    public static void main(String[] args) {
        MyCrawler crawler = new MyCrawler();
        crawler.crawling(new String[]{"http://www.baidu.com"});
    }
}