【java】抓取页面内容，提取链接（此方法可以http get无需账号密码的请求）

package 网络编程;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.net.URL;

public class TestBaidu {
    public static void main(String[] args) throws IOException {
        URL url=new URL("http://www.baidu.com");
        /*此方法会有乱码输出
        InputStream is=url.openStream();
        byte[] b=new byte[1024];
        int len=0;
        while((len=is.read(b))!=-1){
            System.out.println(new String(b,0,len));
        }
        */
        BufferedReader br=new BufferedReader(new InputStreamReader(url.openStream(),"utf-8"));
        BufferedWriter bw=new BufferedWriter(new OutputStreamWriter(new FileOutputStream("baidu.html"),"utf-8"));
        String str=null;
        while((str=br.readLine())!=null){
            bw.append(str);
            bw.newLine();            
        }
            //System.out.print(str);
        bw.flush();
        bw.close();
        br.close();
    }
}

package 网络编程;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URL;
import java.nio.charset.Charset;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class Get163URL {
    public static void main(String[] args) throws IOException {
        URL url=new URL("http://www.163.com");
        BufferedReader br=new BufferedReader(new InputStreamReader(url.openStream(),Charset.forName("gbk")));
        StringBuffer sb=new StringBuffer();
        String tmp=null;
        while((tmp=br.readLine())!=null){
            sb.append(tmp);            
        }
//        System.out.println(sb.toString());
        Pattern p=Pattern.compile(""(http:\/\/.+?)"");
        Matcher m=p.matcher(sb);
        while(m.find())
            System.out.println(m.group(1));
    }
}

---

public class WikiDownload {
    static final String name = "username";
    static final String pwd = "password";

    public static void main(String[] args){
        CookieManager manager = new CookieManager();
        CookieHandler.setDefault(manager);
        String wikiUrl = "http://wiki.xxxxx.org/pages/viewpage.action?pageId=71709153";
        String loginUrl = "http://wiki.xxxxx.org/login.action?os_destination=%2Fpages%2Fviewpage.action%3FpageId%3D71709153";
        try{
            URL url = new URL(loginUrl);
            HttpURLConnection connection = (HttpURLConnection)url.openConnection();
            connection.setRequestProperty("accept", "*/*");
            connection.setRequestProperty("connection", "Keep-Alive");
            connection.setRequestProperty("user-agent",
                    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36");
            String line;
            connection.setDoInput(true);
            connection.setDoOutput(true);
            connection.setUseCaches(false);
            connection.setRequestMethod("POST");
            try(OutputStreamWriter writer = new OutputStreamWriter(connection.getOutputStream())){
                writer.write("os_username=" + name
                        +"&os_password="+ pwd
                        + "&login=%E7%99%BB%E5%BD%95&os_destination="
                        + URLEncoder.encode(wikiUrl.split("http://wiki.xxxxx.org")[0],"utf-8"));
            }
            try(InputStreamReader reader = new InputStreamReader(connection.getInputStream())){
                BufferedReader in = new BufferedReader(reader);
                StringBuilder result= new StringBuilder("");
                while ((line = in.readLine()) != null) {
                    result.append("
");
                    result.append(line);
                }
                System.out.println(result);
            }
        }catch (Exception e){
            e.printStackTrace();
        }

    }

}