闲的无聊，写了个抓图片的网络爬虫

闲的无聊，写了个抓图片的网络爬虫

有喜欢的，拿去，不要把人家辛苦做的站抓宕机就好了。欢迎评论指点；

import com.google.common.collect.Lists;


import org.apache.commons.io.IOUtils;

import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicHeader;

import org.apache.http.protocol.HTTP;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;


import java.io.*;
import java.net.URI;
import java.net.URISyntaxException;


import java.util.List;
import java.util.Stack;


public class SexlCrawler {

    //抓取路径以及保存到本地的路径

    public static String sexpage_url="https://www.mm131.net/xinggan/5354.html";
    public static String local_path="d://temp//sexImg//";

    public java.util.Stack imgStack =new Stack();
    public String pageNum;




    public SexlCrawler(){
        imgStack.push(sexpage_url);
    }

    public static void main(String[] args) {

        SexlCrawler sc=new SexlCrawler();
        while(!sc.imgStack.empty()){

            String url= (String) sc.imgStack.pop();
            sc.startCrawler(url);

        }


    }


    public void startCrawler(String startPage_url){


        //构造请求路径，并添加参数
        try {


            URI uri = new URIBuilder(startPage_url).build();
            List<Header> headerList = Lists.newArrayList();

            headerList.add(new BasicHeader(HttpHeaders.ACCEPT_ENCODING, "gzip, deflate"));
            headerList.add(new BasicHeader(HTTP.CONN_KEEP_ALIVE, "keep-alive"));
            headerList.add(new BasicHeader(HttpHeaders.ACCEPT_LANGUAGE, "zh-cn,zh;q=0.5"));



            //构造HttpClient
            HttpClient httpClient = HttpClients.custom().setDefaultHeaders(headerList).build();



            //构造HttpGet请求
            HttpUriRequest httpUriRequest = RequestBuilder.get().setUri(uri).build();


            //---加入参数，让反爬虫失效
            httpUriRequest.setHeader("Referer","sogou.com");
            //httpUriRequest.setHeader("User-Agent","Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36");
            httpUriRequest.setHeader("User-Agent","sogou.com");

            //获取结果
            HttpResponse httpResponse = httpClient.execute(httpUriRequest);
            //获取返回结果中的实体
            HttpEntity entity = httpResponse.getEntity();
            //查看页面内容结果
            //---内容编码
            String rawHTMLContent = EntityUtils.toString(entity, "GBK");


            //关闭HttpEntity流
            EntityUtils.consume(entity);



            this.setPageNum(startPage_url);

            //---获取路径---//
            String imgUrl= this.dealWith(rawHTMLContent,startPage_url);
            //---存图片---//
            this.savepic(imgUrl,httpClient);





        } catch (URISyntaxException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } catch (ClientProtocolException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }


    }



    public String dealWith(String inputHtml,String startPage_url){

        Document doc = Jsoup.parse(inputHtml);
        String imgurl="";


        Elements elements = doc.select("div[class=content-pic]").select("img");

        Elements next_elements = doc.select("div[class=content-pic]").select("a");

        for( Element element : elements ){

            imgurl= element.attr("src");
        }

        for( Element element : next_elements ){
            String next_url="";

            next_url= element.attr("href");

            //--从开始第一页，每页都是类似 "666_6.html"，但是在最后一页，跳转到新的例如"667.html"时，页面又加上了类似于 https开头的链接
            //---所以要特殊处理
            if(next_url.indexOf("http")<0) {
                int p = startPage_url.lastIndexOf("/");
                next_url = startPage_url.substring(0, p + 1) + next_url;
            }
            else{
                try {
                    Thread.sleep(2000);
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
            }


            this.imgStack.push(next_url);
        }

        return imgurl;

    }

    public String setPageNum(String url){
        String out="";

        int m = url.indexOf(".html");
        int p=0;
        if(url.lastIndexOf("_")>0) {
           int j=url.lastIndexOf("_");
           int k=url.lastIndexOf("/");
            out=url.substring(k,j);
        }
        else{
           p= url.lastIndexOf("/");
           out=url.substring(p+1,m);
        }

        this.pageNum=out;

        return out;


    }


    public void savepic(String ImgURL,HttpClient httpClient){
        if(ImgURL == null){
            return ;
        }

        String[] strs = ImgURL.split("/");
        String fileName = strs[strs.length-1];
        String savePath = local_path+ File.separator+this.pageNum+"_"+fileName;
        HttpEntity entity = null;
        try {

            HttpGet get = new HttpGet(ImgURL);

//            RequestConfig requestConfig = RequestConfig.custom()
//                    .setConnectTimeout(50000).setConnectionRequestTimeout(1000)
//                    .setSocketTimeout(50000).build();
//            get.setConfig(requestConfig);


            //get.setHeader("Referer",ImgURL);
            //get.setHeader("User-Agent","Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36");

            //---这步很关键，是网站反爬虫的策略，只要加上 Referer参数即可，注意参数一定是这个网站名，否则抓的图片都是一个；
            get.setHeader("Referer","https://www.mm131.net/");
            get.setHeader("User-Agent","Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36");
            get.setHeader("User-Agent","sogou.com");

            HttpResponse response = httpClient.execute(get);

            entity = response.getEntity();
            System.out.println("保存图片>>>>.>>>>>>"+fileName);
            InputStream is = entity.getContent();
            OutputStream os = new FileOutputStream(savePath);
            IOUtils.copy(is, os);
            IOUtils.closeQuietly(os);
            IOUtils.closeQuietly(is);

        } catch (Exception e) {
            e.printStackTrace();
            System.out.println("图片保存失败");
            return ;
        }
    }


}

花了2个小时，随便写写的；

欢迎点评；