网页抓取

//根据书名，获取相关图书的ISBN号。抓取的是豆瓣读书的搜索页面。

package cn.edu.xmu.zgy;

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;

public class WebpageCapture {
    public ArrayList<String> captureHtml(String urls,String begins,String ends) throws Exception {
        String strURL =urls;
        URL url = new URL(strURL);
        HttpURLConnection httpConn = (HttpURLConnection) url.openConnection();
        InputStreamReader input = new InputStreamReader(
                httpConn.getInputStream(), "utf-8");
        BufferedReader bufReader = new BufferedReader(input);
        String line = "";
        StringBuilder contentBuf = new StringBuilder();
        while ((line = bufReader.readLine()) != null) {
            contentBuf.append(line);
        }
        String buf = contentBuf.toString();
        ArrayList<String> al=new ArrayList<String>();
        while(true){
        int beginIx = buf.indexOf(begins);
        
        if(beginIx==-1)
            break;
        String result = buf.substring(beginIx);
        int endIx = result.indexOf(ends);
        if(endIx==-1)
            break;
        buf=result.substring(endIx+10);
        result = result.substring(0, endIx);
        al.add(result);
        //System.out.println("captureHtml()的结果：
" + result);
        }
        return al;
    }
        

    public void captureJavascript(String postid) throws Exception {
        String strURL = "http://www.kiees.cn/sf.php?wen=" + postid
                + "&channel=";
        URL url = new URL(strURL);
        HttpURLConnection httpConn = (HttpURLConnection) url.openConnection();
        InputStreamReader input = new InputStreamReader(
                httpConn.getInputStream(), "utf-8");
        BufferedReader bufReader = new BufferedReader(input);
        String line = "";
        StringBuilder contentBuf = new StringBuilder();
        while ((line = bufReader.readLine()) != null) {
            contentBuf.append(line);
        }
        System.out.println("captureJavascript()的结果：
" + contentBuf.toString());
    }

public static void main(String[] args) {
        WebpageCapture demo = new WebpageCapture();
        try {
            String ip="算法导论";
            String strURL = "http://book.douban.com/subject_search?search_text="+ip+"&cat=1001";
            String begin="a class="nbg" href="http://book.";
            String end="/" "+
"  onclick=&#34";
            ArrayList<String> AL1=new ArrayList<String>();
            ArrayList<String> AL2=new ArrayList<String>();
            String result;
            AL1=demo.captureHtml(strURL,begin,end);
            for(int i=0;i<AL1.size();i++)
            {
                AL1.set(i, AL1.get(i).substring(AL1.get(i).indexOf("http:")));
                System.out.println(AL1.get(i));
            }
            
            begin="ISBN:</span> ";
            end="<br/>";
            for(int i=0;i<AL1.size();i++){
            strURL=AL1.get(i);
            AL2=demo.captureHtml(strURL,begin,end);
            AL2.set(0,  AL2.get(0).substring(13));
            System.out.println(AL2.get(0));
            }
            //demo.captureJavascript("107818590577");
        } catch (Exception e) {
            e.printStackTrace();
        }
    }


}

参考链接：http://blog.csdn.net/zgyulongfei/article/details/7909006

网页图片下载：http://blog.csdn.net/oyzl68/article/details/9706373

代码很好

附带一个看不懂的抓取：http://www.oschina.net/code/snippet_1021353_35133?p=1