准备知识
1.HTML, CSS, HTML DOM树
参考http://www.w3school.com.cn/htmldom/
2.Jsoup的使用，使用DOM方法遍历一个document对象，使用选择器语法来选择一个元素，从元素中抽取数据。
参考www.open-open.com/jsoup/example-list-links.htm
3.java正则表达式及其语法
参考http://www.cnblogs.com/chuiyuan/p/5187359.html
我们先来做一个单线程的爬虫。
整体步骤：
1.定义爬取内容的对象Poem结构。
2.完成从网上爬取Document对象的模块HttpService功能。
3.从Document对象中解析出所有唐诗的href，并保存到List<Poem>中。
4.从3中得到的href再爬取出每首古诗的内容。
下面帖一下代码实现。
1.Poem对象只列出其属性
2.抓取Document对象模块HttpService
首先定义一个Rule，封装所有的请求，不管是get还是post。
在HttpService中，使用Rule来抓取document对象。
/**
* Created by chuiyuan on 2/11/16.
*/
public class HttpService {
/**
*
* @param rule
* @return doc
*/
public Document extrace(Rule rule){
validateRule(rule);
String url = rule.getUrl();
String [] params = rule.getParams() ;
String [] values = rule.getValues() ;
String resultTagName = rule.getResultTagName() ;
int type = rule.getType();
int requestMethod = rule.getRequestMethod() ;
Document doc =null ;
try {
Connection conn = Jsoup.connect(url);
conn.userAgent("Mozilla/5.0 (X11; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/38.0 Iceweasel/38.5.0");
if (params!= null){
for (int i =0; i<params.length ;i++){
conn.data(params[i], values[i]);
}
}
//Document doc = null ;
switch (requestMethod){
case Rule.GET:
doc =conn.timeout(10000).get();
break;
case Rule.POST:
doc =conn.timeout(10000).post();
break;
}
}catch (IOException e ){
System.out.println("No network");
//e.printStackTrace();
}
return doc ;
}
/**
* validate input params
*/
private static void validateRule(Rule rule){
String url = rule.getUrl() ;
if (url == null || url.length()==0){
throw new RuleException("url can't be null");
}
if (!url.startsWith("http://")){
throw new RuleException("url not in correct format");
}
/**
* not consider total right
*/
if (rule.getParams()!= null && rule.getValues()!= null){
if (rule.getParams().length!= rule.getValues().length){
throw new RuleException("params length!= values length");
}
}
}
}
可以看到，我们先对rule进行了验证，rule中的resultTagName并没有用到，可以去掉。
3.从Document中解析出所有唐诗的href
4.从每首诗文的href中提到详细的内容
public class ProcessDoc {
/**
*
* @param url
* @param httpService
* @return List<Poem></Poem>
*/
public List<Poem> processGuShiWen(String url, HttpService httpService ){
Rule rule = new Rule(url,
null,null,
null,
-1,
Rule.GET);
Document doc = httpService.extrace(rule);
if (doc == null){
System.out.println("doc null");
return null;
}
//left panel
Elements mainEles = doc.select("div.leftlei");
//String dynasty = mainEles.select("div.son1").first().text();
Elements poemsEles = mainEles.select("div.son2");//size=7 category
//System.out.println(poemsEles.size());
List<Poem> poemList = new ArrayList<Poem>();
//Pattern p =Pattern.compile("\(|\)");
//some times there is no author
for (Element poemsEle : poemsEles){
Elements poemEles = poemsEle.select("span");
String category1 = poemEles.get(0).text(); //category with ":"
String category = category1.substring(0,category1.length()-1);
for (int i =1;i<poemEles.size();i++){
Poem poem = new Poem() ;
//poem.setDynasty(dynasty);
poem.setCategory(category);
poem.setHref(poemEles.get(i).select("a").attr("abs:href")+"/");//ref
poem.setTitle(poemEles.get(i).select("a").text());
poemList.add(poem);
}
}
/*for (Poem poem : poemList){
System.out.println(poem.getCategory()+
" "+poem.getTitle()+
" "+poem.getHref());
}*/
return poemList ;
}
/**
* get details of poem
* @param poem
* @param httpService
*/
public void processDetails(Poem poem, HttpService httpService ){
String url = poem.getHref() ;
//System.out.println(url);
Rule rule = new Rule(url,
null,null,
null,
-1,
Rule.GET);
Document doc = httpService.extrace(rule);
if (doc == null) {
System.out.println("doc=null");
return;
}
Elements mainEles = doc.select("div.son2");
//title already ok
Elements poemDetailEles = mainEles.select("p");
String dynasty = poemDetailEles.get(0).text().split("：")[1];//note，chinese :
//Stng dynasty = poemDetailEles.get(0).getElementsByTag("span").text();
//System.out.println(dynasty);
String author = poemDetailEles.get(1).getElementsByTag("a").text();
//System.out.println(author);
String content = mainEles.text().split("原文：")[1];//note
//System.out.println(content);
//do not consider translation
//Element translationEle = doc.select("#")
poem.setDynasty(dynasty);
poem.setAuthor(author);
poem.setContent(content);
//System.out.println(poem.toString());
}
}
整体调用如下
/**
* single thread model,
* save to mysql
*/
public void getGuShiWenSingleThread(){
String url ="http://so.gushiwen.org/gushi/tangshi.aspx/";
HttpService httpService = new HttpService();
ProcessDoc processDoc= new ProcessDoc();
List<Poem> poemList = processDoc.processGuShiWen(url, httpService);
//get poem content details
for (Poem poem : poemList){
processDoc.processDetails(poem,httpService);
System.out.println(poem.toString());
}
//store to mysql
PoemDao poemDao = new PoemDaoImpl() ;//not PoemDaoImpl
for (Poem poem: poemList){
try {
poemDao.add(poem);
}catch (SQLException e){
e.printStackTrace();
}
}
}
最后结果保存到了MySQL中，数据库部分将在下一篇文章中讲解。