Java爬虫学习（1）之爬取新浪微博博文

本次学习采用了webmagic框架，完成的是一个简单的小demo

package com.mieba.spiader;


import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.FilePipeline;
import us.codecraft.webmagic.processor.PageProcessor;

public class SinaPageProcessor implements PageProcessor
{
    public static final String URL_LIST = "http://blog\.sina\.com\.cn/s/articlelist_1487828712_0_\d+\.html";

    public static final String URL_POST = "http://blog\.sina\.com\.cn/s/blog_\w+\.html";

    private Site site = Site.me().setDomain("blog.sina.com.cn").setRetryTimes(3).setSleepTime(3000).setUserAgent(

                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31");

    @Override

    public void process(Page page)
    {

        // 列表页

        if (page.getUrl().regex(URL_LIST).match())
        {
            // 从页面发现后续的url地址来抓取
            page.addTargetRequests(page.getHtml().xpath("//div[@class="articleList"]").links().regex(URL_POST).all());

            
            page.addTargetRequests(page.getHtml().links().regex(URL_LIST).all());

            // 文章页

        } else
        {
            // 定义如何抽取页面信息，并保存下来
            page.putField("title", page.getHtml().xpath("//div[@class='articalTitle']/h2"));

            page.putField("content", page.getHtml().xpath("//div[@id='articlebody']//div[@class='articalContent']"));

            page.putField("date", page.getHtml().xpath("//div[@id='articlebody']//span[@class='time SG_txtc']").regex("\((.*)\)"));

        } 

    }

    @Override

    public Site getSite()
    {

        return site;

    }

    public static void main(String[] args)
    {

        Spider.create(new SinaPageProcessor())
        //从"http://blog.sina.com.cn/s/articlelist_1487828712_0_1.html"开始抓
        .addUrl("http://blog.sina.com.cn/s/articlelist_1487828712_0_1.html")
        //结果用文件的格式保存下来
        .addPipeline(new FilePipeline("E:\webmagic\"))
        //开启5个线程抓取
        .thread(5)
        //启动爬虫
        .run();

    }
}

运行截图

 爬取的网页