netty的http客户端与apache的http客户端比较

　　之前学习了netty和http异步连接池，跟仓颉大神问的结果是netty的http客户端性能比apache的好。

　　咱今儿就用三种http连接池进行测试。

　　首先是pom.xml：

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>

  <groupId>com.company</groupId>
  <artifactId>websocket_demo</artifactId>
  <version>0.0.1-SNAPSHOT</version>
  <packaging>jar</packaging>

  <name>websocket_demo</name>
  <url>http://maven.apache.org</url>

  <properties>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    <junit.version>4.12</junit.version>
    <log4j.version>1.2.17</log4j.version>
    <netty.version>4.1.36.Final</netty.version>
    <google.gson.version>2.3</google.gson.version>
    <guava.version>19.0</guava.version>
  </properties>

  <dependencies>
     <!-- junit -->
     <dependency>
         <groupId>junit</groupId>
         <artifactId>junit</artifactId>
         <version>${junit.version}</version>
         <scope>test</scope>
     </dependency>

     <!-- log4j -->
     <dependency>
         <groupId>log4j</groupId>
         <artifactId>log4j</artifactId>
         <version>${log4j.version}</version>
     </dependency>
     
     <!-- netty -->
     <dependency>
        <groupId>io.netty</groupId>
        <artifactId>netty-all</artifactId>
        <version>${netty.version}</version>
     </dependency>
     
     <!-- gson -->
     <dependency>
         <groupId>com.google.code.gson</groupId>
         <artifactId>gson</artifactId>
         <version>${google.gson.version}</version>
     </dependency>
     
     <!-- guava -->
     <dependency>
         <groupId>com.google.guava</groupId>
         <artifactId>guava</artifactId>
         <version>${guava.version}</version>
     </dependency>
     
     

<dependency>
    <groupId>org.apache.httpcomponents</groupId>
    <artifactId>httpclient</artifactId>
    <version>4.5.8</version>
</dependency>

     <dependency>
    <groupId>org.apache.commons</groupId>
    <artifactId>commons-io</artifactId>
    <version>1.3.2</version>
</dependency>
     
  </dependencies>
  
  <build>
      <finalName>Netty_WebSocket</finalName>
      <plugins>
          <!--编译版本-->
          <plugin>
              <artifactId>maven-compiler-plugin</artifactId>
              <version>2.3.1</version>
              <configuration>
                  <source>1.8</source>
                  <target>1.8</target>
                  <encoding>UTF-8</encoding>
                  <compilerArguments>
                      <extdirs>src/main/webapp/WEB-INF/lib</extdirs>
                  </compilerArguments>
              </configuration>
          </plugin>
       </plugins>
  </build>
  
</project>

　　导入好包之后，创建一个用于测试http连接池连接时间的基类BaseHttpClientTest（在仓颉大神的基础之上做的测试，测试时发现某些数据有问题，就再观察下载的网页代码长度）：

package com.company.client;

import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;

/**
 * 连接池基类
 * 
 * @author 五月的仓颉https://www.cnblogs.com/xrq730/p/10963689.html
 */
public class BaseHttpClientTest {

    protected static final int REQUEST_COUNT = 15;

    protected static final String SEPERATOR = "   ";
    
    protected static final AtomicInteger NOW_COUNT = new AtomicInteger(0);
    
    protected static final StringBuilder EVERY_REQ_COST = new StringBuilder(200);
    
    protected static final String TEST_URL = "http://sohu.com";
    
    protected static long averageLength = 0;
    
    /**
     * 获取待运行的线程
     */
    protected List<Thread> getRunThreads(Runnable runnable) {
        List<Thread> tList = new ArrayList<Thread>(REQUEST_COUNT);
        
        for (int i = 0; i < REQUEST_COUNT; i++) {
            tList.add(new Thread(runnable));
        }
        
        return tList;
    }
    
    /**
     * 启动所有线程
     */
    protected void startUpAllThreads(List<Thread> tList) {
        for (Thread t : tList) {
            t.start();
            // 这里需要加一点延迟，保证请求按顺序发出去
            try {
                Thread.sleep(300);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }
    }
    
    protected synchronized void addContentLength(long len) {
        averageLength += len;
    }
    
    protected synchronized void addCost(long cost) {
        EVERY_REQ_COST.append(cost);
        EVERY_REQ_COST.append("ms");
        EVERY_REQ_COST.append(SEPERATOR);
    }
    
}

　　不用连接池的子类HttpClientWithoutPoolTest：

package com.company.client;

import java.io.InputStream;

import org.apache.commons.io.IOUtils;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.junit.Test;

/**
 * 不使用连接池测试
 * 
 * @author 五月的仓颉https://www.cnblogs.com/xrq730/p/10963689.html
 */
public class HttpClientWithoutPoolTest extends BaseHttpClientTest {

    @Test
    public void test() throws Exception {
        startUpAllThreads(getRunThreads(new HttpThread()));
        // 等待线程运行
        for (;;);
    }
    
    private class HttpThread implements Runnable {

        @Override
        public void run() {
            /**
             * HttpClient是线程安全的，因此HttpClient正常使用应当做成全局变量，但是一旦全局共用一个，HttpClient内部构建的时候会new一个连接池
             * 出来，这样就体现不出使用连接池的效果，因此这里每次new一个HttpClient，保证每次都不通过连接池请求对端
             */
            CloseableHttpClient httpClient = HttpClients.custom().build();
            HttpGet httpGet = new HttpGet(TEST_URL);
            
            long startTime = System.currentTimeMillis();
            try {
                CloseableHttpResponse response = httpClient.execute(httpGet);
                if (response != null) {
                    addContentLength(IOUtils.toString(response.getEntity().getContent()).length());
                    response.close();
                }
            } catch (Exception e) {
                e.printStackTrace();
            } finally {
                addCost(System.currentTimeMillis() - startTime);
                
                if (NOW_COUNT.incrementAndGet() == REQUEST_COUNT) {
                    System.out.println(EVERY_REQ_COST.toString());
                    System.out.println(averageLength / REQUEST_COUNT);
                }
            }
        }
        
    }
    
}

　　异步连接池的子类HttpclientWithPoolTest：

package com.company.client;

import java.io.ByteArrayOutputStream;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.TimeUnit;

import org.apache.commons.io.IOUtils;
import org.apache.http.HttpHost;
import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.conn.routing.HttpRoute;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.client.StandardHttpRequestRetryHandler;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.junit.Before;
import org.junit.Test;

/**
 * 使用连接池测试
 * 
 * @author 五月的仓颉https://www.cnblogs.com/xrq730/p/10963689.html
 */
public class HttpclientWithPoolTest extends BaseHttpClientTest {

    private CloseableHttpClient httpClient = null;
    
    PoolingHttpClientConnectionManager connectionManager = null;
    
    @Before
    public void before() {
        initHttpClient();
    }
    
    @Test
    public void test() throws Exception {
        startUpAllThreads(getRunThreads(new HttpThread()));
        connectionManager.shutdown();
    }
    
    private class HttpThread implements Runnable {

        @Override
        public void run() {
            HttpGet httpGet = new HttpGet(TEST_URL);
            // 长连接标识，不加也没事，HTTP1.1默认都是Connection: keep-alive的
            httpGet.addHeader("Connection", "keep-alive");
            
            long startTime = System.currentTimeMillis();
            try {
                CloseableHttpResponse response = httpClient.execute(httpGet);
                if (response != null) {
                	addContentLength(IOUtils.toString(response.getEntity().getContent()).length());
                    response.close();
                }
            } catch (Exception e) {
                e.printStackTrace();
            } finally {
                addCost(System.currentTimeMillis() - startTime);
                
                if (NOW_COUNT.incrementAndGet() == REQUEST_COUNT) {
                    System.out.println(EVERY_REQ_COST.toString());
                    System.out.println(averageLength / REQUEST_COUNT);
                }
            }
        }
        
    }
    
    private void initHttpClient() {
        connectionManager = new PoolingHttpClientConnectionManager();
        // 总连接池数量
        connectionManager.setMaxTotal(1);
        // 可为每个域名设置单独的连接池数量
        try {
			connectionManager.setMaxPerRoute(new HttpRoute(new HttpHost(new URL(TEST_URL).getHost())), 1);
		} catch (MalformedURLException e) {
			e.printStackTrace();
		}
        // setConnectTimeout表示设置建立连接的超时时间
        // setConnectionRequestTimeout表示从连接池中拿连接的等待超时时间
        // setSocketTimeout表示发出请求后等待对端应答的超时时间
        RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(10000).setConnectionRequestTimeout(20000)
                .setSocketTimeout(30000).build();
        // 重试处理器，StandardHttpRequestRetryHandler这个是官方提供的，看了下感觉比较挫，很多错误不能重试，可自己实现HttpRequestRetryHandler接口去做
        HttpRequestRetryHandler retryHandler = new StandardHttpRequestRetryHandler();
        
        httpClient = HttpClients.custom().setConnectionManager(connectionManager).setDefaultRequestConfig(requestConfig)
                .setRetryHandler(retryHandler).build();
        
        // 服务端假设关闭了连接，对客户端是不透明的，HttpClient为了缓解这一问题，在某个连接使用前会检测这个连接是否过时，如果过时则连接失效，但是这种做法会为每个请求
        // 增加一定额外开销，因此有一个定时任务专门回收长时间不活动而被判定为失效的连接，可以某种程度上解决这个问题
        Timer timer = new Timer();
        timer.schedule(new TimerTask() {
            @Override
            public void run() {
                try {
                    // 关闭失效连接并从连接池中移除
                    connectionManager.closeExpiredConnections();
                    // 关闭30秒钟内不活动的连接并从连接池中移除，空闲时间从交还给连接管理器时开始
                    connectionManager.closeIdleConnections(20, TimeUnit.SECONDS);
                } catch (Throwable t) {
                    t.printStackTrace();
                }
            }
        }, 0 , 1000 * 5);
    }
    
}

　　使用netty管理http客户端连接的HttpclientNettyTest：

package com.company.client;

import java.net.MalformedURLException;
import java.net.URL;

import org.apache.commons.io.IOUtils;
import org.junit.Before;
import org.junit.Test;

import io.netty.bootstrap.Bootstrap;
import io.netty.channel.Channel;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelOption;
import io.netty.channel.ChannelPipeline;
import io.netty.channel.EventLoopGroup;
import io.netty.channel.SimpleChannelInboundHandler;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioSocketChannel;
import io.netty.handler.codec.http.DefaultFullHttpRequest;
import io.netty.handler.codec.http.DefaultHttpContent;
import io.netty.handler.codec.http.DefaultHttpResponse;
import io.netty.handler.codec.http.FullHttpRequest;
import io.netty.handler.codec.http.HttpClientCodec;
import io.netty.handler.codec.http.HttpContentDecompressor;
import io.netty.handler.codec.http.HttpHeaderNames;
import io.netty.handler.codec.http.HttpHeaderValues;
import io.netty.handler.codec.http.HttpMethod;
import io.netty.handler.codec.http.HttpObjectAggregator;
import io.netty.handler.codec.http.HttpVersion;
import io.netty.handler.codec.http.LastHttpContent;

/**
 * 使用连接池测试
 * 
 * @author 五月的仓颉https://www.cnblogs.com/xrq730/p/10963689.html
 */
public class HttpclientNettyTest extends BaseHttpClientTest {

    //线程组
    private static EventLoopGroup bossGroup = null;

    //启动类
    private static Bootstrap bootstrap = null;
    
    @Before
    public void before() {
        initHttpClient();
    }
    
    public static void closePool() {
        if(null != bossGroup) {
            //优雅退出，释放线程池资源
            bossGroup.shutdownGracefully();
        }
        else {
            System.out.println("is null");
        }
    }
    
    public static Channel getChannel(String url) throws MalformedURLException, InterruptedException {
        URL urlObj  = new URL(url);
        String host = urlObj.getHost();
        int port = urlObj.getPort();
        if(port == -1 && urlObj.getProtocol().equals("http")) {
            port = 80;
        }
        if(port == -1 && urlObj.getProtocol().equals("https")) {
            port = 443;
        }
        return bootstrap.connect(host, port).sync().channel();
    }
    
    public static void get(Channel channel, String url) throws MalformedURLException, InterruptedException {
        URL urlObj  = new URL(url);
        String path = urlObj.getFile();
        if(path == "") {
            path = "/";
        }
        FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, url);
        request.headers().set(HttpHeaderNames.HOST, urlObj.getHost());
        request.headers().set(HttpHeaderNames.CONNECTION, HttpHeaderValues.CLOSE);
        request.headers().set(HttpHeaderNames.ACCEPT_ENCODING, HttpHeaderValues.GZIP + ", deflate");
        channel.writeAndFlush(request);
    }
    
    @Test
    public void test() throws Exception {
        startUpAllThreads(getRunThreads(new HttpThread()));
        closePool();
    }
    
    private class HttpThread implements Runnable {

        @Override
        public void run() {
            Channel channel;
            long startTime = System.currentTimeMillis();
            try {
                channel = getChannel(TEST_URL);
                get(channel, TEST_URL);
                channel.closeFuture().sync();
            } catch (Exception e) {
                e.printStackTrace();
            } finally {
                addCost(System.currentTimeMillis() - startTime);
                if (NOW_COUNT.incrementAndGet() == REQUEST_COUNT) {
                    System.out.println(EVERY_REQ_COST.toString());
                    System.out.println(averageLength / REQUEST_COUNT);
                }
            }
        }
        
    }
    
    private void initHttpClient() {
        try {
            bossGroup = new NioEventLoopGroup();
            bootstrap = new Bootstrap();
            bootstrap.group(bossGroup)
                .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, 30000)
                .option(ChannelOption.SO_KEEPALIVE, true)
                .channel(NioSocketChannel.class)
                .handler(new ChannelInitializer<SocketChannel>() {
                    @Override
                    protected void initChannel(SocketChannel ch) throws Exception {
                        ChannelPipeline p = ch.pipeline();
                        p.addLast(new HttpClientCodec());
                        p.addLast(new HttpContentDecompressor());//这里要添加解压，不然打印时会乱码
                        p.addLast(new HttpObjectAggregator(1234330));//添加HttpObjectAggregator， HttpClientMsgHandler才会收到FullHttpResponse
                        p.addLast("handler", new SimpleChannelInboundHandler<Object>() {
                            @Override
                            protected void channelRead0(ChannelHandlerContext ctx, Object msg) throws Exception {
                                LastHttpContent d = (LastHttpContent)msg;
                                addContentLength(d.content().toString(io.netty.util.CharsetUtil.UTF_8).length());
                            }
                        });
                    }
                });
        }catch(Exception e) {
            e.printStackTrace();
        }
    }
    
}

　　在测试不同网址时，观察到一些想不到的情况。

442ms   443ms   449ms   460ms   178ms   153ms   131ms   146ms   161ms   145ms   135ms   128ms   142ms   145ms   140ms   
23149

384ms   249ms   165ms   161ms   158ms   103ms   128ms   129ms   89ms   147ms   143ms   144ms   126ms   146ms   155ms   
23149

372ms   73ms   67ms   61ms   69ms   64ms   107ms   61ms   68ms   61ms   137ms   68ms   72ms   59ms   57ms   
256

　　可以看到，在连接cnblogs这种https站时，异步连接的优势明显，下载性能从优到劣是：netty>异步连接池>不用连接池。

　　但是如果是http站呢？

　　咱们把基类第22行的网页url改成"http://mini.eastday.com"

101ms   100ms   102ms   27ms   22ms   23ms   21ms   20ms   21ms   21ms   20ms   21ms   29ms   20ms   22ms   
31126

106ms   44ms   34ms   506ms   359ms   157ms   60ms   44ms   37ms   38ms   34ms   36ms   37ms   36ms   40ms   
31126

539ms   239ms   43ms   317ms   82ms   46ms   38ms   100ms   42ms   35ms   41ms   40ms   43ms   35ms   48ms   
31126

　　对于东方头条门户这种加载速度本身就快的网页，使用异步机制对整体加载速度影响不大。

　　但是如果需要加载大一点的网页呢？

　　咱们把基类第22行的网页url改成"https://mini.eastday.com/a/190919091345041.html"

　　无连接池的东方头条新闻网页：

510ms   374ms   526ms   539ms   124ms   74ms   72ms   62ms   70ms   75ms   70ms   75ms   207ms   86ms   74ms   
65325

　　有连接池的东方头条新闻网页：

424ms   185ms   92ms   63ms   74ms   77ms   77ms   66ms   92ms   64ms   107ms   65ms   70ms   51ms   90ms   
65325

　　netty的东方头条新闻网页：

318ms   44ms   33ms   41ms   42ms   41ms   47ms   34ms   49ms   38ms   30ms   34ms   47ms   40ms   44ms   
264

　　如果下载比较大的网页，异步机制确实能让系统总体的加载速度加快。