定时爬虫系统(以爬取[百度7日关注]为例)

1、web.xml加载servlet

<?xml version="1.0" encoding="UTF-8"?>
<web-app xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://java.sun.com/xml/ns/javaee" xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd" id="WebApp_ID" version="2.5">
  <display-name>TaskSchedule</display-name>
  <welcome-file-list>
    <welcome-file>index.html</welcome-file>
    <welcome-file>index.htm</welcome-file>
    <welcome-file>index.jsp</welcome-file>
    <welcome-file>default.html</welcome-file>
    <welcome-file>default.htm</welcome-file>
    <welcome-file>default.jsp</welcome-file>
  </welcome-file-list>
  <servlet>
      <servlet-name>hotword</servlet-name>
      <servlet-class>com.richinfo.asynctask.servlet.TaskScheduleServlet</servlet-class>
      <load-on-startup>1</load-on-startup>
  </servlet>
</web-app>

2、TaskScheduleServlet初始化init

public class TaskScheduleServlet extends HttpServlet{
    private static final Log logger = LogFactory.getLog(TaskScheduleServlet.class);
    private static final long serialVersionUID = 9089148097823231232L;
    
    @Override
    public void init() throws ServletException {
        super.init();
        logger.info("TaskScheduleServlet init!");
        System.setProperty("sun.net.client.defaultConnectTimeout", "30000");
        System.setProperty("sun.net.client.defaultReadTimeout", "30000");
        TaskRegister.getInstance().start();
        printLocalHost();
        logger.info("TaskScheduleServlet start!");
    }
    private void printLocalHost() {
        try {
            InetAddress local = InetAddress.getLocalHost();
            String ip = local.getHostAddress();
            String host = local.getHostName();
            logger.info("服务器信息：ip=" + ip + ", host=" + host);
        } catch (UnknownHostException e) {
            logger.error(e.getMessage());
        }
    }

    @Override
    public void destroy() {
        super.destroy();
        TaskRegister.getInstance().shutdown();
        logger.info("TaskScheduleServlet destroy!");
    }
}

3、TaskRegister.getInstance().start()

4、定时任务的设置，这里不做赘述(有兴趣可以看我的另一篇有关定时任务配置的文章http://www.cnblogs.com/zhuziyu/p/7704661.html)

5、爬虫具体实现

（1）调用任务

public class BaiduDayHotFocusGrabJob implements Job{
    private final static Log logger = LogFactory.getLog(BaiduDayHotFocusGrabJob.class);
    private UnifiedPosMgrPlatService service = new UnifiedPosMgrPlatSeviceImpl();
    @Override
    public void execute(JobExecutionContext context)
            throws JobExecutionException {
        JobKey key = context.getJobDetail().getKey();
        logger.info("开始任务["+key.getGroup()+"."+key.getName()+"]");
        
        try {
            resolve();
        } catch (Exception e) {
            logger.error(Tools.getStackInfo(e));
        }
    }
    
    public void resolve() throws IOException {
        Document doc = Jsoup.connect("http://top.baidu.com/").timeout(10000).get();
        Element attentionDiv = doc.select("#main div.tab-bd div.tab-box:eq(1)").first();
        Elements attentionEles = attentionDiv.select("ul li a.list-title");
        // 七日关注
        
        List<HotSpot> attentionList = new ArrayList<HotSpot>();
        int len = attentionEles.size();
        for (int i = 0; i < len; i++) {
            Element ele = attentionEles.get(i);
            String name = ele.text().trim();
            String uri = ele.attr("href").trim();
            attentionList.add(new HotSpot(Source.BAIDU_TOP, uri, name, i));
        }
        HotFocus focus = new HotFocus(Type.DAILY, Source.BAIDU_TOP, attentionList);
        int ret = service.addHotFocus(focus);
        if (ret != 1) {
            logger.warn("更新七日关注失败！");
        }
        else {
            logger.info("更新七日关注成功！");
        }
    }
}

（2）具体实现

public class UnifiedPosMgrPlatSeviceImpl implements UnifiedPosMgrPlatService {
    private static final Logger logger = Logger.getLogger(UnifiedPosMgrPlatSeviceImpl.class);
    
    @Override
    public int addHotFocus(HotFocus focus) throws IOException {
        if (focus == null) return -1;
        List<HotSpot> spots = focus.getHots();
        if (spots == null || spots.isEmpty()) return -1;
        String uri = getAddHotWordInvokeUrl();
        String requestXml = requestAddHotWord(focus);
        String responseXml = invokeAddHotWords(uri, requestXml);
        return responseAddHotWords(responseXml);
    }
}

对应要调用的接口系统：本人的是http://127.0.0.1:8080/接口系统/方法

6、对应系统的接口实现这里不赘述，最终执行数据库操作，将爬取内容写入数据库表