简单的分析下Netty的启动源码和接收请求的源码，以下面代码为例子：
1. 主启动类
/* * Copyright 2012 The Netty Project * * The Netty Project licenses this file to you under the Apache License, * version 2.0 (the "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at: * * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations * under the License. */ package cn.xm.netty.example.echo; import io.netty.bootstrap.ServerBootstrap; import io.netty.channel.ChannelFuture; import io.netty.channel.ChannelInitializer; import io.netty.channel.ChannelOption; import io.netty.channel.ChannelPipeline; import io.netty.channel.EventLoopGroup; import io.netty.channel.nio.NioEventLoopGroup; import io.netty.channel.socket.SocketChannel; import io.netty.channel.socket.nio.NioServerSocketChannel; import io.netty.handler.logging.LogLevel; import io.netty.handler.logging.LoggingHandler; import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslContextBuilder; import io.netty.handler.ssl.util.SelfSignedCertificate; /** * Echoes back any received data from a client. */ public final class EchoServer { static final boolean SSL = System.getProperty("ssl") != null; static final int PORT = Integer.parseInt(System.getProperty("port", "8007")); public static void main(String[] args) throws Exception { // Configure SSL. final SslContext sslCtx; if (SSL) { SelfSignedCertificate ssc = new SelfSignedCertificate(); sslCtx = SslContextBuilder.forServer(ssc.certificate(), ssc.privateKey()).build(); } else { sslCtx = null; } // Configure the server. EventLoopGroup bossGroup = new NioEventLoopGroup(1); EventLoopGroup workerGroup = new NioEventLoopGroup(); final EchoServerHandler serverHandler = new EchoServerHandler(); try { ServerBootstrap b = new ServerBootstrap(); b.group(bossGroup, workerGroup) .channel(NioServerSocketChannel.class) .option(ChannelOption.SO_BACKLOG, 100) .handler(new LoggingHandler(LogLevel.INFO)) .childHandler(new ChannelInitializer<SocketChannel>() { @Override public void initChannel(SocketChannel ch) throws Exception { ChannelPipeline p = ch.pipeline(); if (sslCtx != null) { p.addLast(sslCtx.newHandler(ch.alloc())); } //p.addLast(new LoggingHandler(LogLevel.INFO)); p.addLast(serverHandler); } }); // Start the server. ChannelFuture f = b.bind(PORT).sync(); // Wait until the server socket is closed. f.channel().closeFuture().sync(); } finally { // Shut down all event loops to terminate all threads. bossGroup.shutdownGracefully(); workerGroup.shutdownGracefully(); } } }
2. EchoServerHandle 处理器类
/* * Copyright 2012 The Netty Project * * The Netty Project licenses this file to you under the Apache License, * version 2.0 (the "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at: * * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the * License for the specific language governing permissions and limitations * under the License. */ package cn.xm.netty.example.echo; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import io.netty.channel.ChannelHandler.Sharable; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelInboundHandlerAdapter; import io.netty.util.CharsetUtil; /** * Handler implementation for the echo server. */ @Sharable public class EchoServerHandler extends ChannelInboundHandlerAdapter { @Override public void channelRead(ChannelHandlerContext ctx, Object msg) { System.out.println("ctx = " + ctx); // 强转为netty的ByteBuffer(实际就是包装的ByteBuffer) ByteBuf byteBuf = (ByteBuf) msg; System.out.println("客户端发送的消息是：" + byteBuf.toString(CharsetUtil.UTF_8)); System.out.println("客户端地址：" + ctx.channel().remoteAddress()); } @Override public void channelReadComplete(ChannelHandlerContext ctx) { // 将数据写到客户端(write + flush) ctx.writeAndFlush(Unpooled.copiedBuffer("hello, 客户端！", CharsetUtil.UTF_8)); } @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { // Close the connection when an exception is raised. cause.printStackTrace(); ctx.close(); } }
1. 服务器端启动源码
1. 首先启动类创建了关于SSL的配置类
2. 创建了两个 EventLoopGroup 对象
// Configure the server. EventLoopGroup bossGroup = new NioEventLoopGroup(1); EventLoopGroup workerGroup = new NioEventLoopGroup();
(1) 这两个对象是Netty的核心对象，可以说，整个Netty 的运作都依赖他们。 bossGroup 用于接收TCP请求，然后将请求交给workerGroup， workerGroup 会获取到真正的连接，然后和连接进行通信，比如读写编码解码操作。
(2) EventLoopGroup 是事件循环组(线程组) 含有多个EventLoop， 可以注册channel，用于在事件循环中去进行选择(和选择器相关)。
(3) new NioEventLoopGroup(1); 表示这个group 事件组有1个线程可以指定， 不传参数默认是CPU核数 * 2。
protected MultithreadEventLoopGroup(int nThreads, Executor executor, Object... args) { super(nThreads == 0 ? DEFAULT_EVENT_LOOP_THREADS : nThreads, executor, args); } ... private static final int DEFAULT_EVENT_LOOP_THREADS; static { DEFAULT_EVENT_LOOP_THREADS = Math.max(1, SystemPropertyUtil.getInt( "io.netty.eventLoopThreads", NettyRuntime.availableProcessors() * 2)); if (logger.isDebugEnabled()) { logger.debug("-Dio.netty.eventLoopThreads: {}", DEFAULT_EVENT_LOOP_THREADS); } }
然后io.netty.util.concurrent.MultithreadEventExecutorGroup#MultithreadEventExecutorGroup(int, java.util.concurrent.Executor, io.netty.util.concurrent.EventExecutorChooserFactory, java.lang.Object...)会创建EventExecutor 数组， 每个对象都是NioEventLoop 类型(io.netty.channel.nio.NioEventLoopGroup#newChild)，NioEventLoop 实现了EventExecutor 接口和EventLoop 接口。 内部维护selector 等相关属性。
protected MultithreadEventExecutorGroup(int nThreads, Executor executor, EventExecutorChooserFactory chooserFactory, Object... args) { this.terminatedChildren = new AtomicInteger(); this.terminationFuture = new DefaultPromise(GlobalEventExecutor.INSTANCE); if (nThreads <= 0) { throw new IllegalArgumentException(String.format("nThreads: %d (expected: > 0)", nThreads)); } else { if (executor == null) { executor = new ThreadPerTaskExecutor(this.newDefaultThreadFactory()); } this.children = new EventExecutor[nThreads]; int j; for(int i = 0; i < nThreads; ++i) { boolean success = false; boolean var18 = false; try { var18 = true; this.children[i] = this.newChild((Executor)executor, args); success = true; var18 = false; } catch (Exception var19) { throw new IllegalStateException("failed to create a child event loop", var19); } finally { if (var18) { if (!success) { int j; for(j = 0; j < i; ++j) { this.children[j].shutdownGracefully(); } for(j = 0; j < i; ++j) { EventExecutor e = this.children[j]; try { while(!e.isTerminated()) { e.awaitTermination(2147483647L, TimeUnit.SECONDS); } } catch (InterruptedException var20) { Thread.currentThread().interrupt(); break; } } } } } if (!success) { for(j = 0; j < i; ++j) { this.children[j].shutdownGracefully(); } for(j = 0; j < i; ++j) { EventExecutor e = this.children[j]; try { while(!e.isTerminated()) { e.awaitTermination(2147483647L, TimeUnit.SECONDS); } } catch (InterruptedException var22) { Thread.currentThread().interrupt(); break; } } } } this.chooser = chooserFactory.newChooser(this.children); FutureListener<Object> terminationListener = new FutureListener<Object>() { public void operationComplete(Future<Object> future) throws Exception { if (MultithreadEventExecutorGroup.this.terminatedChildren.incrementAndGet() == MultithreadEventExecutorGroup.this.children.length) { MultithreadEventExecutorGroup.this.terminationFuture.setSuccess((Object)null); } } }; EventExecutor[] var24 = this.children; j = var24.length; for(int var26 = 0; var26 < j; ++var26) { EventExecutor e = var24[var26]; e.terminationFuture().addListener(terminationListener); } Set<EventExecutor> childrenSet = new LinkedHashSet(this.children.length); Collections.addAll(childrenSet, this.children); this.readonlyChildren = Collections.unmodifiableSet(childrenSet); } }
io.netty.channel.nio.NioEventLoopGroup#newChild 创建EventLoop 如下：
@Override protected EventLoop newChild(Executor executor, Object... args) throws Exception { EventLoopTaskQueueFactory queueFactory = args.length == 4 ? (EventLoopTaskQueueFactory) args[3] : null; return new NioEventLoop(this, executor, (SelectorProvider) args[0], ((SelectStrategyFactory) args[1]).newSelectStrategy(), (RejectedExecutionHandler) args[2], queueFactory); }
io.netty.channel.nio.NioEventLoop#NioEventLoop 构造如下：
NioEventLoop(NioEventLoopGroup parent, Executor executor, SelectorProvider selectorProvider, SelectStrategy strategy, RejectedExecutionHandler rejectedExecutionHandler, EventLoopTaskQueueFactory queueFactory) { super(parent, executor, false, newTaskQueue(queueFactory), newTaskQueue(queueFactory), rejectedExecutionHandler); if (selectorProvider == null) { throw new NullPointerException("selectorProvider"); } if (strategy == null) { throw new NullPointerException("selectStrategy"); } provider = selectorProvider; final SelectorTuple selectorTuple = openSelector(); selector = selectorTuple.selector; unwrappedSelector = selectorTuple.unwrappedSelector; selectStrategy = strategy; }
NioEventLoop 的继承图如下：
io.netty.channel.nio.NioEventLoop#run 方法如下：
@Override protected void run() { for (;;) { try { try { switch (selectStrategy.calculateStrategy(selectNowSupplier, hasTasks())) { case SelectStrategy.CONTINUE: continue; case SelectStrategy.BUSY_WAIT: // fall-through to SELECT since the busy-wait is not supported with NIO case SelectStrategy.SELECT: select(wakenUp.getAndSet(false)); // 'wakenUp.compareAndSet(false, true)' is always evaluated // before calling 'selector.wakeup()' to reduce the wake-up // overhead. (Selector.wakeup() is an expensive operation.) // // However, there is a race condition in this approach. // The race condition is triggered when 'wakenUp' is set to // true too early. // // 'wakenUp' is set to true too early if: // 1) Selector is waken up between 'wakenUp.set(false)' and // 'selector.select(...)'. (BAD) // 2) Selector is waken up between 'selector.select(...)' and // 'if (wakenUp.get()) { ... }'. (OK) // // In the first case, 'wakenUp' is set to true and the // following 'selector.select(...)' will wake up immediately. // Until 'wakenUp' is set to false again in the next round, // 'wakenUp.compareAndSet(false, true)' will fail, and therefore // any attempt to wake up the Selector will fail, too, causing // the following 'selector.select(...)' call to block // unnecessarily. // // To fix this problem, we wake up the selector again if wakenUp // is true immediately after selector.select(...). // It is inefficient in that it wakes up the selector for both // the first case (BAD - wake-up required) and the second case // (OK - no wake-up required). if (wakenUp.get()) { selector.wakeup(); } // fall through default: } } catch (IOException e) { // If we receive an IOException here its because the Selector is messed up. Let's rebuild // the selector and retry. https://github.com/netty/netty/issues/8566 rebuildSelector0(); handleLoopException(e); continue; } cancelledKeys = 0; needsToSelectAgain = false; final int ioRatio = this.ioRatio; if (ioRatio == 100) { try { processSelectedKeys(); } finally { // Ensure we always run tasks. runAllTasks(); } } else { final long ioStartTime = System.nanoTime(); try { processSelectedKeys(); } finally { // Ensure we always run tasks. final long ioTime = System.nanoTime() - ioStartTime; runAllTasks(ioTime * (100 - ioRatio) / ioRatio); } } } catch (Throwable t) { handleLoopException(t); } // Always handle shutdown even if the loop processing threw an exception. try { if (isShuttingDown()) { closeAll(); if (confirmShutdown()) { return; } } } catch (Throwable t) { handleLoopException(t); } } }
3. 然后创建了一个ServerBootstrap 启动引导类，用于启动服务器和引导整个程序的初始化。 它和ServerChannel 关联，而ServerChannel 继承了Channel， 有一些remoteAddress等属性。
public class ServerBootstrap extends AbstractBootstrap<ServerBootstrap, ServerChannel> { private static final InternalLogger logger = InternalLoggerFactory.getInstance(ServerBootstrap.class); private final Map<ChannelOption<?>, Object> childOptions = new LinkedHashMap(); private final Map<AttributeKey<?>, Object> childAttrs = new LinkedHashMap(); private final ServerBootstrapConfig config = new ServerBootstrapConfig(this); private volatile EventLoopGroup childGroup; private volatile ChannelHandler childHandler; public ServerBootstrap() { } ...
(1) 然后b.group(bossGroup, workerGroup) 将两个group 设为自己的属性，用于后期的引导使用(也可以理解为parentGroup 和 childGroup)
io.netty.bootstrap.ServerBootstrap#group(io.netty.channel.EventLoopGroup, io.netty.channel.EventLoopGroup)：
public ServerBootstrap group(EventLoopGroup parentGroup, EventLoopGroup childGroup) { super.group(parentGroup); if (childGroup == null) { throw new NullPointerException("childGroup"); } else if (this.childGroup != null) { throw new IllegalStateException("childGroup set already"); } else { this.childGroup = childGroup; return this; } }
(2) .channel(NioServerSocketChannel.class) 创建一个channel， 引导类通过这个class 对象反射创建ChannelFactory。 然后添加一些TCP的参数。io.netty.bootstrap.AbstractBootstrap#channel
public B channel(Class<? extends C> channelClass) { if (channelClass == null) { throw new NullPointerException("channelClass"); } else { return this.channelFactory((io.netty.channel.ChannelFactory)(new ReflectiveChannelFactory(channelClass))); } } public B channelFactory(io.netty.channel.ChannelFactory<? extends C> channelFactory) { return this.channelFactory((ChannelFactory)channelFactory); }
(3) .option(ChannelOption.SO_BACKLOG, 100) 方法设置属性，也就是将属性维护在自己的map 中。
public <T> B option(ChannelOption<T> option, T value) { if (option == null) { throw new NullPointerException("option"); } else { if (value == null) { synchronized(this.options) { this.options.remove(option); } } else { synchronized(this.options) { this.options.put(option, value); } } return this; } }
(4) .handler(new LoggingHandler(LogLevel.INFO)) 是给服务器添加一个专属的日志处理器handler，用于bossGroup。 是继承自父类的方法
io.netty.bootstrap.AbstractBootstrap#handler(io.netty.channel.ChannelHandler)
public B handler(ChannelHandler handler) { if (handler == null) { throw new NullPointerException("handler"); } else { this.handler = handler; return this; } }
(5) .childHandler 是给workerGroup 添加handler， 是采用链条方式添加多个handler
public ServerBootstrap childHandler(ChannelHandler childHandler) { if (childHandler == null) { throw new NullPointerException("childHandler"); } else { this.childHandler = childHandler; return this; } }
(6) 然后绑定端口并阻塞至连接成功
ChannelFuture f = b.bind(PORT).sync();
io.netty.bootstrap.AbstractBootstrap#bind(int) 方法如下：
public ChannelFuture bind(int inetPort) { return this.bind(new InetSocketAddress(inetPort)); } public ChannelFuture bind(SocketAddress localAddress) { this.validate(); if (localAddress == null) { throw new NullPointerException("localAddress"); } else { return this.doBind(localAddress); } } private ChannelFuture doBind(final SocketAddress localAddress) { final ChannelFuture regFuture = this.initAndRegister(); final Channel channel = regFuture.channel(); if (regFuture.cause() != null) { return regFuture; } else if (regFuture.isDone()) { ChannelPromise promise = channel.newPromise(); doBind0(regFuture, channel, localAddress, promise); return promise; } else { final AbstractBootstrap.PendingRegistrationPromise promise = new AbstractBootstrap.PendingRegistrationPromise(channel); regFuture.addListener(new ChannelFutureListener() { public void operationComplete(ChannelFuture future) throws Exception { Throwable cause = future.cause(); if (cause != null) { promise.setFailure(cause); } else { promise.registered(); AbstractBootstrap.doBind0(regFuture, channel, localAddress, promise); } } }); return promise; } }
核心方法是initAndRegister 和 AbstractBootstrap.doBind0(regFuture, channel, localAddress, promise); 方法
1》 io.netty.bootstrap.AbstractBootstrap#initAndRegister 方法
final ChannelFuture initAndRegister() { Channel channel = null; try { channel = this.channelFactory.newChannel(); this.init(channel); } catch (Throwable var3) { if (channel != null) { channel.unsafe().closeForcibly(); } return (new DefaultChannelPromise(channel, GlobalEventExecutor.INSTANCE)).setFailure(var3); } ChannelFuture regFuture = this.config().group().register(channel); if (regFuture.cause() != null) { if (channel.isRegistered()) { channel.close(); } else { channel.unsafe().closeForcibly(); } } return regFuture; }
》1.1 首先调用io.netty.channel.ReflectiveChannelFactory#newChannel 反射创建Channel： 创建对象，初始化相关的pipeline、unsafe 等属性
public T newChannel() { try { return (Channel)this.constructor.newInstance(); } catch (Throwable var2) { throw new ChannelException("Unable to create Channel from class " + this.constructor.getDeclaringClass(), var2); } }
最终会调用到io.netty.channel.socket.nio.NioServerSocketChannel#NioServerSocketChannel()
public NioServerSocketChannel() { this(newSocket(DEFAULT_SELECTOR_PROVIDER)); } public NioServerSocketChannel(java.nio.channels.ServerSocketChannel channel) { super((Channel)null, channel, 16); this.config = new NioServerSocketChannel.NioServerSocketChannelConfig(this, this.javaChannel().socket()); } private static java.nio.channels.ServerSocketChannel newSocket(SelectorProvider provider) { try { return provider.openServerSocketChannel(); } catch (IOException var2) { throw new ChannelException("Failed to open a server socket.", var2); } }
sun.nio.ch.SelectorProviderImpl#openServerSocketChannel 如下：
public ServerSocketChannel openServerSocketChannel() throws IOException { return new ServerSocketChannelImpl(this); }
io.netty.channel.nio.AbstractNioChannel#AbstractNioChannel 如下：
protected AbstractNioChannel(Channel parent, SelectableChannel ch, int readInterestOp) { super(parent); this.ch = ch; this.readInterestOp = readInterestOp; try { ch.configureBlocking(false); } catch (IOException e) { try { ch.close(); } catch (IOException e2) { logger.warn( "Failed to close a partially initialized socket.", e2); } throw new ChannelException("Failed to enter non-blocking mode.", e); } }
io.netty.channel.AbstractChannel#AbstractChannel(io.netty.channel.Channel) 如下：
protected AbstractChannel(Channel parent) { this.parent = parent; id = newId(); unsafe = newUnsafe(); pipeline = newChannelPipeline(); }
》1.2 io.netty.bootstrap.ServerBootstrap#init 进行channel 的init 操作
void init(Channel channel) throws Exception { Map<ChannelOption<?>, Object> options = this.options0(); synchronized(options) { setChannelOptions(channel, options, logger); } Map<AttributeKey<?>, Object> attrs = this.attrs0(); synchronized(attrs) { Iterator var5 = attrs.entrySet().iterator(); while(true) { if (!var5.hasNext()) { break; } Entry<AttributeKey<?>, Object> e = (Entry)var5.next(); AttributeKey<Object> key = (AttributeKey)e.getKey(); channel.attr(key).set(e.getValue()); } } ChannelPipeline p = channel.pipeline(); final EventLoopGroup currentChildGroup = this.childGroup; final ChannelHandler currentChildHandler = this.childHandler; final Entry[] currentChildOptions; synchronized(this.childOptions) { currentChildOptions = (Entry[])this.childOptions.entrySet().toArray(newOptionArray(this.childOptions.size())); } final Entry[] currentChildAttrs; synchronized(this.childAttrs) { currentChildAttrs = (Entry[])this.childAttrs.entrySet().toArray(newAttrArray(this.childAttrs.size())); } p.addLast(new ChannelHandler[]{new ChannelInitializer<Channel>() { public void initChannel(final Channel ch) throws Exception { final ChannelPipeline pipeline = ch.pipeline(); ChannelHandler handler = ServerBootstrap.this.config.handler(); if (handler != null) { pipeline.addLast(new ChannelHandler[]{handler}); } ch.eventLoop().execute(new Runnable() { public void run() { pipeline.addLast(new ChannelHandler[]{new ServerBootstrap.ServerBootstrapAcceptor(ch, currentChildGroup, currentChildHandler, currentChildOptions, currentChildAttrs)}); } }); } }}); }
这里在最后加了个io.netty.bootstrap.ServerBootstrap.ServerBootstrapAcceptor 处理器，这个处理器是一个重要的处理器，用于接收到请求注册到workerGroup
public void channelRead(ChannelHandlerContext ctx, Object msg) { final Channel child = (Channel)msg; child.pipeline().addLast(new ChannelHandler[]{this.childHandler}); AbstractBootstrap.setChannelOptions(child, this.childOptions, ServerBootstrap.logger); Entry[] var4 = this.childAttrs; int var5 = var4.length; for(int var6 = 0; var6 < var5; ++var6) { Entry<AttributeKey<?>, Object> e = var4[var6]; child.attr((AttributeKey)e.getKey()).set(e.getValue()); } try { this.childGroup.register(child).addListener(new ChannelFutureListener() { public void operationComplete(ChannelFuture future) throws Exception { if (!future.isSuccess()) { ServerBootstrap.ServerBootstrapAcceptor.forceClose(child, future.cause()); } } }); } catch (Throwable var8) { forceClose(child, var8); } }
io.netty.channel.DefaultChannelPipeline#addLast(io.netty.util.concurrent.EventExecutorGroup, java.lang.String, io.netty.channel.ChannelHandler) 添加方法如下：
public final ChannelPipeline addLast(EventExecutorGroup group, String name, ChannelHandler handler) { AbstractChannelHandlerContext newCtx; synchronized(this) { checkMultiplicity(handler); newCtx = this.newContext(group, this.filterName(name, handler), handler); this.addLast0(newCtx); if (!this.registered) { newCtx.setAddPending(); this.callHandlerCallbackLater(newCtx, true); return this; } EventExecutor executor = newCtx.executor(); if (!executor.inEventLoop()) { this.callHandlerAddedInEventLoop(newCtx, executor); return this; } } this.callHandlerAdded0(newCtx); return this; } private void addLast0(AbstractChannelHandlerContext newCtx) { AbstractChannelHandlerContext prev = this.tail.prev; newCtx.prev = prev; newCtx.next = this.tail; prev.next = newCtx; this.tail.prev = newCtx; }
1》检查handler 是否符合标准；2》创建一个AbstractChannelHandlerContext 对象，这个对象ChannelHandler 和 ChannelPipeline 之间的关联，每当有ChannelHandler 添加到Pipeline 中时，都会创建Context。 Context 的主要功能是管理他所关联的Handler 和 同一个Pipeline中的其他handler 之间的交互。 3》 将context 添加到联众中，也就是添加到tail节点的前面。
》1.3 ChannelFuture regFuture = this.config().group().register(channel); 注册 事件 0
io.netty.channel.AbstractChannel.AbstractUnsafe#register
public final void register(EventLoop eventLoop, final ChannelPromise promise) { if (eventLoop == null) { throw new NullPointerException("eventLoop"); } else if (AbstractChannel.this.isRegistered()) { promise.setFailure(new IllegalStateException("registered to an event loop already")); } else if (!AbstractChannel.this.isCompatible(eventLoop)) { promise.setFailure(new IllegalStateException("incompatible event loop type: " + eventLoop.getClass().getName())); } else { AbstractChannel.this.eventLoop = eventLoop; if (eventLoop.inEventLoop()) { this.register0(promise); } else { try { eventLoop.execute(new Runnable() { public void run() { AbstractUnsafe.this.register0(promise); } }); } catch (Throwable var4) { AbstractChannel.logger.warn("Force-closing a channel whose registration task was not accepted by an event loop: {}", AbstractChannel.this, var4); this.closeForcibly(); AbstractChannel.this.closeFuture.setClosed(); this.safeSetFailure(promise, var4); } } } }
io.netty.channel.nio.AbstractNioChannel#doRegister：
protected void doRegister() throws Exception { boolean selected = false; while(true) { try { this.selectionKey = this.javaChannel().register(this.eventLoop().unwrappedSelector(), 0, this); return; } catch (CancelledKeyException var3) { if (selected) { throw var3; } this.eventLoop().selectNow(); selected = true; } } }
2》 第二个重要的方法是： io.netty.bootstrap.AbstractBootstrap#doBind0
private static void doBind0(final ChannelFuture regFuture, final Channel channel, final SocketAddress localAddress, final ChannelPromise promise) { channel.eventLoop().execute(new Runnable() { public void run() { if (regFuture.isSuccess()) { channel.bind(localAddress, promise).addListener(ChannelFutureListener.CLOSE_ON_FAILURE); } else { promise.setFailure(regFuture.cause()); } } }); }
最后调用到：io.netty.channel.AbstractChannelHandlerContext#bind(java.net.SocketAddress, io.netty.channel.ChannelPromise)
public ChannelFuture bind(final SocketAddress localAddress, final ChannelPromise promise) { if (localAddress == null) { throw new NullPointerException("localAddress"); } else if (this.isNotValidPromise(promise, false)) { return promise; } else { final AbstractChannelHandlerContext next = this.findContextOutbound(); EventExecutor executor = next.executor(); if (executor.inEventLoop()) { next.invokeBind(localAddress, promise); } else { safeExecute(executor, new Runnable() { public void run() { next.invokeBind(localAddress, promise); } }, promise, (Object)null); } return promise; } }
继续调用到io.netty.channel.AbstractChannel.AbstractUnsafe#bind：
public final void bind(SocketAddress localAddress, ChannelPromise promise) { this.assertEventLoop(); if (promise.setUncancellable() && this.ensureOpen(promise)) { if (Boolean.TRUE.equals(AbstractChannel.this.config().getOption(ChannelOption.SO_BROADCAST)) && localAddress instanceof InetSocketAddress && !((InetSocketAddress)localAddress).getAddress().isAnyLocalAddress() && !PlatformDependent.isWindows() && !PlatformDependent.maybeSuperUser()) { AbstractChannel.logger.warn("A non-root user can't receive a broadcast packet if the socket is not bound to a wildcard address; binding to a non-wildcard address (" + localAddress + ") anyway as requested."); } boolean wasActive = AbstractChannel.this.isActive(); try { AbstractChannel.this.doBind(localAddress); } catch (Throwable var5) { this.safeSetFailure(promise, var5); this.closeIfClosed(); return; } if (!wasActive && AbstractChannel.this.isActive()) { this.invokeLater(new Runnable() { public void run() { AbstractChannel.this.pipeline.fireChannelActive(); } }); } this.safeSetSuccess(promise); } }
》3.1 调用到io.netty.channel.socket.nio.NioServerSocketChannel#doBind
protected void doBind(SocketAddress localAddress) throws Exception { if (PlatformDependent.javaVersion() >= 7) { this.javaChannel().bind(localAddress, this.config.getBacklog()); } else { this.javaChannel().socket().bind(localAddress, this.config.getBacklog()); } }
继续调用NIO的bind和listen 方法(调用内核的bind 、 listen 方法进行绑定和监听)sun.nio.ch.ServerSocketChannelImpl#bind
@Override public ServerSocketChannel bind(SocketAddress local, int backlog) throws IOException { synchronized (stateLock) { ensureOpen(); if (localAddress != null) throw new AlreadyBoundException(); InetSocketAddress isa = (local == null) ? new InetSocketAddress(0) : Net.checkAddress(local); SecurityManager sm = System.getSecurityManager(); if (sm != null) sm.checkListen(isa.getPort()); NetHooks.beforeTcpBind(fd, isa.getAddress(), isa.getPort()); Net.bind(fd, isa.getAddress(), isa.getPort()); Net.listen(fd, backlog < 1 ? 50 : backlog); localAddress = Net.localAddress(fd); } return this; }
》 3.2 绑定完成后触发通道激活事件 AbstractChannel.this.pipeline.fireChannelActive(); 会触发 io.netty.channel.nio.AbstractNioChannel#doBeginRead方法
protected void doBeginRead() throws Exception { SelectionKey selectionKey = this.selectionKey; if (selectionKey.isValid()) { this.readPending = true; int interestOps = selectionKey.interestOps(); if ((interestOps & this.readInterestOp) == 0) { selectionKey.interestOps(interestOps | this.readInterestOp); } } }
this.readInterestOp 是反射创建的时候指定的 16 ， 也就是连接事件。 这里注册了OP_ACCEPT 事件。
至此启动过程完成了。总结：
1. 创建2个EventLoopGroup 线程池数组。 数组默认大小CPU*2， 方便chooser 选择线程池时提高性能
2. bootstrap 将boss 设置为group属性，将worker 设置为child 属性
3. 通过bind 方法启动，内部重要方法为 initAndRegister 和 dobind0 方法
initAndRegister 方法会反射创建NioServerSocketChannel及其相关的NIO对象 pipeline、 unsafe。 同时也为pipeline 初始化了head节点和tail伪节点。
doBind0 方法，最后调用JDK nio 中的ServerSocketChannelImpl的bind 方法，调用内核的bind 方法和listen 方法。完成服务器的启动，并开始监听连接事件。
2. 接收客户端请求源码跟踪
在上面ServerBootstrap 启动过程中创建了bossGroup 和 workerGroup ， 并且workerGroup注册了 16 OP_ACCEPT 事件。 接下来 NioEventLoop的run 方法会循环进行。所以入口就是NioEventLoop的run 方法。
1. io.netty.channel.nio.NioEventLoop#run 方法如下：
@Override protected void run() { for (;;) { try { switch (selectStrategy.calculateStrategy(selectNowSupplier, hasTasks())) { case SelectStrategy.CONTINUE: continue; case SelectStrategy.SELECT: select(wakenUp.getAndSet(false)); // 'wakenUp.compareAndSet(false, true)' is always evaluated // before calling 'selector.wakeup()' to reduce the wake-up // overhead. (Selector.wakeup() is an expensive operation.) // // However, there is a race condition in this approach. // The race condition is triggered when 'wakenUp' is set to // true too early. // // 'wakenUp' is set to true too early if: // 1) Selector is waken up between 'wakenUp.set(false)' and // 'selector.select(...)'. (BAD) // 2) Selector is waken up between 'selector.select(...)' and // 'if (wakenUp.get()) { ... }'. (OK) // // In the first case, 'wakenUp' is set to true and the // following 'selector.select(...)' will wake up immediately. // Until 'wakenUp' is set to false again in the next round, // 'wakenUp.compareAndSet(false, true)' will fail, and therefore // any attempt to wake up the Selector will fail, too, causing // the following 'selector.select(...)' call to block // unnecessarily. // // To fix this problem, we wake up the selector again if wakenUp // is true immediately after selector.select(...). // It is inefficient in that it wakes up the selector for both // the first case (BAD - wake-up required) and the second case // (OK - no wake-up required). if (wakenUp.get()) { selector.wakeup(); } default: // fallthrough } cancelledKeys = 0; needsToSelectAgain = false; final int ioRatio = this.ioRatio; if (ioRatio == 100) { try { processSelectedKeys(); } finally { // Ensure we always run tasks. runAllTasks(); } } else { final long ioStartTime = System.nanoTime(); try { processSelectedKeys(); } finally { // Ensure we always run tasks. final long ioTime = System.nanoTime() - ioStartTime; runAllTasks(ioTime * (100 - ioRatio) / ioRatio); } } } catch (Throwable t) { handleLoopException(t); } // Always handle shutdown even if the loop processing threw an exception. try { if (isShuttingDown()) { closeAll(); if (confirmShutdown()) { return; } } } catch (Throwable t) { handleLoopException(t); } } }
io.netty.channel.nio.NioEventLoop#selectNowSupplier如下：
private final IntSupplier selectNowSupplier = new IntSupplier() { @Override public int get() throws Exception { return selectNow(); } }; int selectNow() throws IOException { try { return selector.selectNow(); } finally { // restore wakeup state if needed if (wakenUp.get()) { selector.wakeup(); } } }
该loop做的三件事情为：
1》 有条件的等待NIO事件
2》处理NIO事件
3》处理消息队列中的任务
2. io.netty.channel.nio.NioEventLoop#processSelectedKeys 方法如下
private void processSelectedKeys() { if (selectedKeys != null) { processSelectedKeysOptimized(); } else { processSelectedKeysPlain(selector.selectedKeys()); } } private void processSelectedKeysOptimized() { for (int i = 0; i < selectedKeys.size; ++i) { final SelectionKey k = selectedKeys.keys[i]; // null out entry in the array to allow to have it GC'ed once the Channel close // See https://github.com/netty/netty/issues/2363 selectedKeys.keys[i] = null; final Object a = k.attachment(); if (a instanceof AbstractNioChannel) { processSelectedKey(k, (AbstractNioChannel) a); } else { @SuppressWarnings("unchecked") NioTask<SelectableChannel> task = (NioTask<SelectableChannel>) a; processSelectedKey(k, task); } if (needsToSelectAgain) { // null out entries in the array to allow to have it GC'ed once the Channel close // See https://github.com/netty/netty/issues/2363 selectedKeys.reset(i + 1); selectAgain(); i = -1; } } } private void processSelectedKey(SelectionKey k, AbstractNioChannel ch) { final AbstractNioChannel.NioUnsafe unsafe = ch.unsafe(); if (!k.isValid()) { final EventLoop eventLoop; try { eventLoop = ch.eventLoop(); } catch (Throwable ignored) { // If the channel implementation throws an exception because there is no event loop, we ignore this // because we are only trying to determine if ch is registered to this event loop and thus has authority // to close ch. return; } // Only close ch if ch is still registered to this EventLoop. ch could have deregistered from the event loop // and thus the SelectionKey could be cancelled as part of the deregistration process, but the channel is // still healthy and should not be closed. // See https://github.com/netty/netty/issues/5125 if (eventLoop != this || eventLoop == null) { return; } // close the channel if the key is not valid anymore unsafe.close(unsafe.voidPromise()); return; } try { int readyOps = k.readyOps(); // We first need to call finishConnect() before try to trigger a read(...) or write(...) as otherwise // the NIO JDK channel implementation may throw a NotYetConnectedException. if ((readyOps & SelectionKey.OP_CONNECT) != 0) { // remove OP_CONNECT as otherwise Selector.select(..) will always return without blocking // See https://github.com/netty/netty/issues/924 int ops = k.interestOps(); ops &= ~SelectionKey.OP_CONNECT; k.interestOps(ops); unsafe.finishConnect(); } // Process OP_WRITE first as we may be able to write some queued buffers and so free memory. if ((readyOps & SelectionKey.OP_WRITE) != 0) { // Call forceFlush which will also take care of clear the OP_WRITE once there is nothing left to write ch.unsafe().forceFlush(); } // Also check for readOps of 0 to workaround possible JDK bug which may otherwise lead // to a spin loop if ((readyOps & (SelectionKey.OP_READ | SelectionKey.OP_ACCEPT)) != 0 || readyOps == 0) { unsafe.read(); } } catch (CancelledKeyException ignored) { unsafe.close(unsafe.voidPromise()); } }
所以核心在io.netty.channel.nio.NioEventLoop#processSelectedKey(java.nio.channels.SelectionKey, io.netty.channel.nio.AbstractNioChannel) 方法。
3. io.netty.channel.nio.NioEventLoop#processSelectedKey(java.nio.channels.SelectionKey, io.netty.channel.nio.AbstractNioChannel) 方法处理逻辑
我们debugger 打断点到该方法，浏览器访问 http://localhost:8007/
线程调用链为：
查看SelectionKey 为如下：(16 为建立链接事件)
1》走代码unsafe.read(); 进行调用io.netty.channel.nio.AbstractNioMessageChannel.NioMessageUnsafe#read
@Override public void read() { assert eventLoop().inEventLoop(); final ChannelConfig config = config(); final ChannelPipeline pipeline = pipeline(); final RecvByteBufAllocator.Handle allocHandle = unsafe().recvBufAllocHandle(); allocHandle.reset(config); boolean closed = false; Throwable exception = null; try { try { do { int localRead = doReadMessages(readBuf); if (localRead == 0) { break; } if (localRead < 0) { closed = true; break; } allocHandle.incMessagesRead(localRead); } while (allocHandle.continueReading()); } catch (Throwable t) { exception = t; } int size = readBuf.size(); for (int i = 0; i < size; i ++) { readPending = false; pipeline.fireChannelRead(readBuf.get(i)); } readBuf.clear(); allocHandle.readComplete(); pipeline.fireChannelReadComplete(); if (exception != null) { closed = closeOnReadError(exception); pipeline.fireExceptionCaught(exception); } if (closed) { inputShutdown = true; if (isOpen()) { close(voidPromise()); } } } finally { // Check if there is a readPending which was not processed yet. // This could be for two reasons: // * The user called Channel.read() or ChannelHandlerContext.read() in channelRead(...) method // * The user called Channel.read() or ChannelHandlerContext.read() in channelReadComplete(...) method // // See https://github.com/netty/netty/issues/2254 if (!readPending && !config.isAutoRead()) { removeReadOp(); } } }
1.1》doReadMessages(readBuf); 调用io.netty.channel.socket.nio.NioServerSocketChannel#doReadMessages
@Override protected int doReadMessages(List<Object> buf) throws Exception { SocketChannel ch = SocketUtils.accept(javaChannel()); try { if (ch != null) { buf.add(new NioSocketChannel(this, ch)); return 1; } } catch (Throwable t) { logger.warn("Failed to create a new channel from an accepted socket.", t); try { ch.close(); } catch (Throwable t2) { logger.warn("Failed to close a socket.", t2); } } return 0; }
首先调用：io.netty.util.internal.SocketUtils#accept 实际就是调用serverSocketChannel.accept() 方法获取一个SocketChannel
public static SocketChannel accept(final ServerSocketChannel serverSocketChannel) throws IOException { try { return AccessController.doPrivileged(new PrivilegedExceptionAction<SocketChannel>() { @Override public SocketChannel run() throws IOException { return serverSocketChannel.accept(); } }); } catch (PrivilegedActionException e) { throw (IOException) e.getCause(); } }
然后包装为NioSocketChannel 对象添加到buf 容器中。
包装后的属性如下：(构造方法中进行属性的初始化)
1.2》 for 循环中pipeline.fireChannelRead(readBuf.get(i)); 激活fireChannelRead 事件
io.netty.channel.DefaultChannelPipeline#fireChannelRead
@Override public final ChannelPipeline fireChannelRead(Object msg) { AbstractChannelHandlerContext.invokeChannelRead(head, msg); return this; }
这里是一个链式的调用：
链条中有一个重要的类是在上面初始化过程中加的类： io.netty.bootstrap.ServerBootstrap.ServerBootstrapAcceptor#channelRead
@Override @SuppressWarnings("unchecked") public void channelRead(ChannelHandlerContext ctx, Object msg) { final Channel child = (Channel) msg; child.pipeline().addLast(childHandler); setChannelOptions(child, childOptions, logger); for (Entry<AttributeKey<?>, Object> e: childAttrs) { child.attr((AttributeKey<Object>) e.getKey()).set(e.getValue()); } try { childGroup.register(child).addListener(new ChannelFutureListener() { @Override public void operationComplete(ChannelFuture future) throws Exception { if (!future.isSuccess()) { forceClose(child, future.cause()); } } }); } catch (Throwable t) { forceClose(child, t); } }
这里重要的过程是msg 强转为Channel，添加childHandler到pipeline 中，也就是我们在serverBootstrap 中的childHandler 记录的handler； 设置属性； 将NioSocketChannel 注册到childGroup中的一个EventLoop 上，并添加一个监听器。所以重要的方法在：io.netty.channel.MultithreadEventLoopGroup#register(io.netty.channel.Channel) 注册方法
@Override public ChannelFuture register(Channel channel) { return next().register(channel); }
第一步：next 方法选择一个EventLoop
io.netty.util.concurrent.MultithreadEventExecutorGroup#next：
@Override public EventExecutor next() { return chooser.next(); }
io.netty.util.concurrent.DefaultEventExecutorChooserFactory.PowerOfTwoEventExecutorChooser#next：可以看到默认是轮询
@Override public EventExecutor next() { return executors[idx.getAndIncrement() & executors.length - 1]; }
第二步：调用io.netty.channel.SingleThreadEventLoop#register(io.netty.channel.Channel) 进行注册
@Override public ChannelFuture register(Channel channel) { return register(new DefaultChannelPromise(channel, this)); } @Override public ChannelFuture register(final ChannelPromise promise) { ObjectUtil.checkNotNull(promise, "promise"); promise.channel().unsafe().register(this, promise); return promise; }
紧接着调用到：io.netty.channel.AbstractChannel.AbstractUnsafe#register
@Override public final void register(EventLoop eventLoop, final ChannelPromise promise) { if (eventLoop == null) { throw new NullPointerException("eventLoop"); } if (isRegistered()) { promise.setFailure(new IllegalStateException("registered to an event loop already")); return; } if (!isCompatible(eventLoop)) { promise.setFailure( new IllegalStateException("incompatible event loop type: " + eventLoop.getClass().getName())); return; } AbstractChannel.this.eventLoop = eventLoop; if (eventLoop.inEventLoop()) { register0(promise); } else { try { eventLoop.execute(new Runnable() { @Override public void run() { register0(promise); } }); } catch (Throwable t) { logger.warn( "Force-closing a channel whose registration task was not accepted by an event loop: {}", AbstractChannel.this, t); closeForcibly(); closeFuture.setClosed(); safeSetFailure(promise, t); } } } private void register0(ChannelPromise promise) { try { // check if the channel is still open as it could be closed in the mean time when the register // call was outside of the eventLoop if (!promise.setUncancellable() || !ensureOpen(promise)) { return; } boolean firstRegistration = neverRegistered; doRegister(); neverRegistered = false; registered = true; // Ensure we call handlerAdded(...) before we actually notify the promise. This is needed as the // user may already fire events through the pipeline in the ChannelFutureListener. pipeline.invokeHandlerAddedIfNeeded(); safeSetSuccess(promise); pipeline.fireChannelRegistered(); // Only fire a channelActive if the channel has never been registered. This prevents firing // multiple channel actives if the channel is deregistered and re-registered. if (isActive()) { if (firstRegistration) { pipeline.fireChannelActive(); } else if (config().isAutoRead()) { // This channel was registered before and autoRead() is set. This means we need to begin read // again so that we process inbound data. // // See https://github.com/netty/netty/issues/4805 beginRead(); } } } catch (Throwable t) { // Close the channel directly to avoid FD leak. closeForcibly(); closeFuture.setClosed(); safeSetFailure(promise, t); } }
doRegister(); 方法注册的是一个0事件，注册完成之后调用 pipeline.fireChannelActive(); 激活事件，然后同上面启动过程一样，调用到：
io.netty.channel.nio.AbstractNioChannel#doBeginRead
@Override protected void doBeginRead() throws Exception { // Channel.read() or ChannelHandlerContext.read() was called final SelectionKey selectionKey = this.selectionKey; if (!selectionKey.isValid()) { return; } readPending = true; final int interestOps = selectionKey.interestOps(); if ((interestOps & readInterestOp) == 0) { selectionKey.interestOps(interestOps | readInterestOp); } }
如上readInterestOp 是创建时候的事件1， 也就是 OP_READ 事件
至此客户端连接已经建立成功，接下来就可以监听读事件进行读取数据。
3. 读取客户端数据：
读取的时候的线程调用栈如下：
(1) 入口可以从io.netty.channel.nio.NioEventLoop#processSelectedKey(java.nio.channels.SelectionKey, io.netty.channel.nio.AbstractNioChannel) 查看：
private void processSelectedKey(SelectionKey k, AbstractNioChannel ch) { final AbstractNioChannel.NioUnsafe unsafe = ch.unsafe(); if (!k.isValid()) { final EventLoop eventLoop; try { eventLoop = ch.eventLoop(); } catch (Throwable ignored) { // If the channel implementation throws an exception because there is no event loop, we ignore this // because we are only trying to determine if ch is registered to this event loop and thus has authority // to close ch. return; } // Only close ch if ch is still registered to this EventLoop. ch could have deregistered from the event loop // and thus the SelectionKey could be cancelled as part of the deregistration process, but the channel is // still healthy and should not be closed. // See https://github.com/netty/netty/issues/5125 if (eventLoop != this || eventLoop == null) { return; } // close the channel if the key is not valid anymore unsafe.close(unsafe.voidPromise()); return; } try { int readyOps = k.readyOps(); // We first need to call finishConnect() before try to trigger a read(...) or write(...) as otherwise // the NIO JDK channel implementation may throw a NotYetConnectedException. if ((readyOps & SelectionKey.OP_CONNECT) != 0) { // remove OP_CONNECT as otherwise Selector.select(..) will always return without blocking // See https://github.com/netty/netty/issues/924 int ops = k.interestOps(); ops &= ~SelectionKey.OP_CONNECT; k.interestOps(ops); unsafe.finishConnect(); } // Process OP_WRITE first as we may be able to write some queued buffers and so free memory. if ((readyOps & SelectionKey.OP_WRITE) != 0) { // Call forceFlush which will also take care of clear the OP_WRITE once there is nothing left to write ch.unsafe().forceFlush(); } // Also check for readOps of 0 to workaround possible JDK bug which may otherwise lead // to a spin loop if ((readyOps & (SelectionKey.OP_READ | SelectionKey.OP_ACCEPT)) != 0 || readyOps == 0) { unsafe.read(); } } catch (CancelledKeyException ignored) { unsafe.close(unsafe.voidPromise()); } }
(2)会调用到io.netty.channel.nio.AbstractNioByteChannel.NioByteUnsafe#read
@Override public final void read() { final ChannelConfig config = config(); final ChannelPipeline pipeline = pipeline(); final ByteBufAllocator allocator = config.getAllocator(); final RecvByteBufAllocator.Handle allocHandle = recvBufAllocHandle(); allocHandle.reset(config); ByteBuf byteBuf = null; boolean close = false; try { do { byteBuf = allocHandle.allocate(allocator); allocHandle.lastBytesRead(doReadBytes(byteBuf)); if (allocHandle.lastBytesRead() <= 0) { // nothing was read. release the buffer. byteBuf.release(); byteBuf = null; close = allocHandle.lastBytesRead() < 0; break; } allocHandle.incMessagesRead(1); readPending = false; pipeline.fireChannelRead(byteBuf); byteBuf = null; } while (allocHandle.continueReading()); allocHandle.readComplete(); pipeline.fireChannelReadComplete(); if (close) { closeOnRead(pipeline); } } catch (Throwable t) { handleReadException(pipeline, byteBuf, t, close, allocHandle); } finally { // Check if there is a readPending which was not processed yet. // This could be for two reasons: // * The user called Channel.read() or ChannelHandlerContext.read() in channelRead(...) method // * The user called Channel.read() or ChannelHandlerContext.read() in channelReadComplete(...) method // // See https://github.com/netty/netty/issues/2254 if (!readPending && !config.isAutoRead()) { removeReadOp(); } } }
先读取消息，读取完之后调用pipeline.fireChannelRead(byteBuf); 处理读取事件。读取到的byteBuf 如下：
可以看到对于不同的事件有不同的unsafe， 对于 16 连接事件unsafe 为io.netty.channel.nio.AbstractNioMessageChannel.NioMessageUnsafe； 对于 1 读取事件unsafe 为 io.netty.channel.socket.nio.NioSocketChannel.NioSocketChannelUnsafe
接收请求流程：接受连接-》创建一个新的nioSocketChannel(包装ServerSocketChannel.accept() 返回的类)-》注册到一个worker eventLoop上 -》 注册read 事件 -》 监听读取事件并处理