Netty源码分析-ChunkedFile和ChunkedWriteHandler

 

/*
 * Copyright 2012 The Netty Project
 *
 * The Netty Project licenses this file to you under the Apache License,
 * version 2.0 (the "License"); you may not use this file except in compliance
 * with the License. You may obtain a copy of the License at:
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations
 * under the License.
 */
package io.netty.handler.stream;

import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufAllocator;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.FileRegion;
import io.netty.util.internal.ObjectUtil;

import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;

/**
 * A {@link ChunkedInput} that fetches data from a file chunk by chunk.
 * 

 * If your operating system supports
 * zero-copy file transfer
 * such as {@code sendfile()}, you might want to use {@link FileRegion} instead.
 */
public class ChunkedFile implements ChunkedInput {

    private final RandomAccessFile file;
    private final long startOffset;
    private final long endOffset;
    private final int chunkSize;
    private long offset;

    /**
     * Creates a new instance that fetches data from the specified file.
     */
    public ChunkedFile(File file) throws IOException {
        this(file, ChunkedStream.DEFAULT_CHUNK_SIZE);
    }

    /**
     * Creates a new instance that fetches data from the specified file.
     *
     * @param chunkSize the number of bytes to fetch on each
     *                  {@link #readChunk(ChannelHandlerContext)} call
     */
    //只读方式打开文件
    public ChunkedFile(File file, int chunkSize) throws IOException {
        this(new RandomAccessFile(file, "r"), chunkSize);
    }

    /**
     * Creates a new instance that fetches data from the specified file.
     */
    public ChunkedFile(RandomAccessFile file) throws IOException {
        this(file, ChunkedStream.DEFAULT_CHUNK_SIZE);
    }

    /**
     * Creates a new instance that fetches data from the specified file.
     *
     * @param chunkSize the number of bytes to fetch on each
     *                  {@link #readChunk(ChannelHandlerContext)} call
     */
    public ChunkedFile(RandomAccessFile file, int chunkSize) throws IOException {
        this(file, 0, file.length(), chunkSize);
    }

    /**
     * Creates a new instance that fetches data from the specified file.
     *
     * @param offset the offset of the file where the transfer begins 文件读取起始位置偏移量
     * @param length the number of bytes to transfer   一共读取多少字节
     * @param chunkSize the number of bytes to fetch on each  每次循环调用readChunk读取的块大小
     *                  {@link #readChunk(ChannelHandlerContext)} call
     */
    public ChunkedFile(RandomAccessFile file, long offset, long length, int chunkSize) throws IOException {
        ObjectUtil.checkNotNull(file, "file");
        ObjectUtil.checkPositiveOrZero(offset, "offset");
        ObjectUtil.checkPositiveOrZero(length, "length");
        ObjectUtil.checkPositive(chunkSize, "chunkSize");

        this.file = file; //文件
        this.offset = startOffset = offset; //读取起始位置
        this.endOffset = offset + length; //读取结束位置
        this.chunkSize = chunkSize; //每次读取快大小

        //设置file的偏移量
        file.seek(offset);
    }

    /**
     * Returns the offset in the file where the transfer began.
     */
    public long startOffset() {
        return startOffset;
    }

    /**
     * Returns the offset in the file where the transfer will end.
     */
    public long endOffset() {
        return endOffset;
    }

    /**
     * Returns the offset in the file where the transfer is happening currently.
     */
    public long currentOffset() {
        return offset;
    }

    @Override
    public boolean isEndOfInput() throws Exception {
        //是否读取完毕
        //offset < endOffset = true 说明没读到最后
        //file.getChannel().isOpen()=true,说明文件打开状态
        //上诉俩种情况则任务没有读取完毕，二者出现任何一种情况都认为读取完毕
        return !(offset < endOffset && file.getChannel().isOpen());
    }

    //关闭底层资源
    @Override
    public void close() throws Exception {
        file.close();
    }

    @Deprecated
    @Override
    public ByteBuf readChunk(ChannelHandlerContext ctx) throws Exception {
        //读取底层文件
        return readChunk(ctx.alloc());
    }

    @Override
    public ByteBuf readChunk(ByteBufAllocator allocator) throws Exception {
        //如果已经到达结束偏移量，不需要再读取数据了
        long offset = this.offset;
        if (offset >= endOffset) {
            return null;
        }

        //看看剩余的量与chunkSize块谁更小一些
        //如果剩余的字节非常大，则每次按照chunkSize去读取，如果剩余的字节数量不够一个chunkSize了，就读实际的数量。
        int chunkSize = (int) Math.min(this.chunkSize, endOffset - offset);
        // Check if the buffer is backed by an byte array. If so we can optimize it a bit an safe a copy

        //分配ByteBuf,注意是堆内存
        ByteBuf buf = allocator.heapBuffer(chunkSize);
        boolean release = true;
        try {
            //读取固定长度chunkSize这个些个字节，如果字节不够会阻塞，如果返回EOF会抛出异常。
            file.readFully(buf.array(), buf.arrayOffset(), chunkSize);
            //重新设置ByteBuf的写索引
            buf.writerIndex(chunkSize);
            //偏移量增加chunkSize这么些字节.
            this.offset = offset + chunkSize;
            release = false;
            return buf;
        } finally {
            if (release) {
                buf.release();
            }
        }
    }

    //返回长度
    @Override
    public long length() {
        return endOffset - startOffset;
    }

    //返回处理了多少字节，用当前的offset减去最开始的startOffset
    @Override
    public long progress() {
        return offset - startOffset;
    }
}

 

/*
 * Copyright 2012 The Netty Project
 *
 * The Netty Project licenses this file to you under the Apache License,
 * version 2.0 (the "License"); you may not use this file except in compliance
 * with the License. You may obtain a copy of the License at:
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations
 * under the License.
 */
package io.netty.handler.stream;

import io.netty.buffer.ByteBufAllocator;
import io.netty.buffer.Unpooled;
import io.netty.channel.Channel;
import io.netty.channel.ChannelDuplexHandler;
import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelFutureListener;
import io.netty.channel.ChannelHandler;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelPipeline;
import io.netty.channel.ChannelProgressivePromise;
import io.netty.channel.ChannelPromise;
import io.netty.util.ReferenceCountUtil;
import io.netty.util.internal.logging.InternalLogger;
import io.netty.util.internal.logging.InternalLoggerFactory;

import java.nio.channels.ClosedChannelException;
import java.util.ArrayDeque;
import java.util.Queue;

/**
 * A {@link ChannelHandler} that adds support for writing a large data stream
 * asynchronously neither spending a lot of memory nor getting
 * {@link OutOfMemoryError}.  Large data streaming such as file
 * transfer requires complicated state management in a {@link ChannelHandler}
 * implementation.  {@link ChunkedWriteHandler} manages such complicated states
 * so that you can send a large data stream without difficulties.
 * 

 * To use {@link ChunkedWriteHandler} in your application, you have to insert
 * a new {@link ChunkedWriteHandler} instance:
 * 
 * {@link ChannelPipeline} p = ...;
 * p.addLast("streamer", new {@link ChunkedWriteHandler}());
 * p.addLast("handler", new MyHandler());
 * 
 * Once inserted, you can write a {@link ChunkedInput} so that the
 * {@link ChunkedWriteHandler} can pick it up and fetch the content of the
 * stream chunk by chunk and write the fetched chunk downstream:
 * 
 * {@link Channel} ch = ...;
 * ch.write(new {@link ChunkedFile}(new File("video.mkv"));
 * 
 *
 * 
Sending a stream which generates a chunk intermittently
 *
 * Some {@link ChunkedInput} generates a chunk on a certain event or timing.
 * Such {@link ChunkedInput} implementation often returns {@code null} on
 * {@link ChunkedInput#readChunk(ChannelHandlerContext)}, resulting in the indefinitely suspended
 * transfer.  To resume the transfer when a new chunk is available, you have to
 * call {@link #resumeTransfer()}.
 */
public class ChunkedWriteHandler extends ChannelDuplexHandler {

    private static final InternalLogger logger =
        InternalLoggerFactory.getInstance(ChunkedWriteHandler.class);

    //一个队列，用来存储需要输出的数据
    private final Queue queue = new ArrayDeque();

    //当前Context上下文
    private volatile ChannelHandlerContext ctx;

    public ChunkedWriteHandler() {
    }

    /**
     * @deprecated use {@link #ChunkedWriteHandler()}
     */
    @Deprecated
    public ChunkedWriteHandler(int maxPendingWrites) {
        if (maxPendingWrites <= 0) {
            throw new IllegalArgumentException(
                    "maxPendingWrites: " + maxPendingWrites + " (expected: > 0)");
        }
    }

    @Override
    public void handlerAdded(ChannelHandlerContext ctx) throws Exception {
        this.ctx = ctx;
    }

    /**
     * Continues to fetch the chunks from the input.
     */
    public void resumeTransfer() {
        final ChannelHandlerContext ctx = this.ctx;
        if (ctx == null) {
            return;
        }
        if (ctx.executor().inEventLoop()) {
            resumeTransfer0(ctx);
        } else {
            // let the transfer resume on the next event loop round
            ctx.executor().execute(new Runnable() {

                @Override
                public void run() {
                    resumeTransfer0(ctx);
                }
            });
        }
    }

    private void resumeTransfer0(ChannelHandlerContext ctx) {
        try {
            doFlush(ctx);
        } catch (Exception e) {
            logger.warn("Unexpected exception while sending chunks.", e);
        }
    }

    @Override
    public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception {
        //write方法没有向下传递，而是封装为PendingWrite对象，并存入队列。
        queue.add(new PendingWrite(msg, promise));
    }


    @Override
    public void flush(ChannelHandlerContext ctx) throws Exception {
        //flush方法没有向下传递，根据情况进行输出
        doFlush(ctx);
    }

    @Override
    public void channelInactive(ChannelHandlerContext ctx) throws Exception {
        //连接断开事件中，根据情况进行输出
        doFlush(ctx);
        ctx.fireChannelInactive();
    }

    @Override
    public void channelWritabilityChanged(ChannelHandlerContext ctx) throws Exception {
        if (ctx.channel().isWritable()) {
            // channel is writable again try to continue flushing
            //如果当前channel再次切换到可写状态，则继续doFlush
            doFlush(ctx);
        }
        ctx.fireChannelWritabilityChanged();
    }

    private void discard(Throwable cause) {
        for (;;) {
            //循环每一个等待输出的对象
            PendingWrite currentWrite = queue.poll();

            //如果为null说明处理完毕
            if (currentWrite == null) {
                break;
            }

            //拿到msg对象
            Object message = currentWrite.msg;

            if (message instanceof ChunkedInput) {
                //块对象处理
                ChunkedInput in = (ChunkedInput) message;
                boolean endOfInput;
                long inputLength;
                try {
                    //判断是否已经到达结尾
                    endOfInput = in.isEndOfInput();
                    //获取长度
                    inputLength = in.length();
                    //关闭资源
                    closeInput(in);
                } catch (Exception e) {
                    //关闭资源
                    closeInput(in);
                    //设置错误
                    currentWrite.fail(e);
                    if (logger.isWarnEnabled()) {
                        logger.warn(ChunkedInput.class.getSimpleName() + " failed", e);
                    }
                    continue;
                }

                //如果块资源还没输出干净
                if (!endOfInput) {
                    if (cause == null) {
                        cause = new ClosedChannelException();
                    }
                    //设置为失败状态
                    currentWrite.fail(cause);
                } else {
                    //如果已经输出完毕，则设置为成功状态
                    currentWrite.success(inputLength);
                }
            } else {
                //普通对象设置promise失败即可
                if (cause == null) {
                    cause = new ClosedChannelException();
                }
                currentWrite.fail(cause);
            }
        }
    }

    private void doFlush(final ChannelHandlerContext ctx) {
        final Channel channel = ctx.channel();
        //首先要判断channel是否连接着
        if (!channel.isActive()) {
            //处理没有输出掉的对象
            discard(null);
            return;
        }

        //是否需要调用底层flush
        boolean requiresFlush = true;

        //内存分配器
        ByteBufAllocator allocator = ctx.alloc();

        //如果channel还可以写就继续循环，如果底层队列写满这里会返回false
        while (channel.isWritable()) {
            //拿出队列头部但不移除元素
            final PendingWrite currentWrite = queue.peek();

            //如果为null则结束循环
            if (currentWrite == null) {
                break;
            }

            //如果当前元素已经完成，则从队列中移除后继续循环下一个
            if (currentWrite.promise.isDone()) {
                // This might happen e.g. in the case when a write operation
                // failed, but there're still unconsumed chunks left.
                // Most chunked input sources would stop generating chunks
                // and report end of input, but this doesn't work with any
                // source wrapped in HttpChunkedInput.
                // Note, that we're not trying to release the message/chunks
                // as this had to be done already by someone who resolved the
                // promise (using ChunkedInput.close method).
                // See https://github.com/netty/netty/issues/8700.
                queue.remove();
                continue;
            }

            //消息
            final Object pendingMessage = currentWrite.msg;

            //消息是块消息
            if (pendingMessage instanceof ChunkedInput) {
                final ChunkedInput chunks = (ChunkedInput) pendingMessage;
                boolean endOfInput;
                boolean suspend;
                Object message = null;
                try {
                    //读取一部分数据内容
                    message = chunks.readChunk(allocator);
                    //判断是否读取结束
                    endOfInput = chunks.isEndOfInput();

                    //如果消息等于null
                    if (message == null) {
                        // No need to suspend when reached at the end.
                        //如果此时消息块读取结束则endOfInput=treu，那么suspend=false，表示不需要暂停
                        //否则说明消息快还没结束，但是本次获取数据暂时没有。
                        suspend = !endOfInput;
                    } else {
                        //如果suspend不为空，则不需要暂停
                        suspend = false;
                    }
                } catch (final Throwable t) {
                    //异常处理，从队列中移除
                    queue.remove();

                    //释放msg
                    if (message != null) {
                        ReferenceCountUtil.release(message);
                    }

                    //关闭资源
                    closeInput(chunks);
                    //设置失败
                    currentWrite.fail(t);
                    //结束循环
                    break;
                }

                //如果需要暂停，那么也结束本次循环
                if (suspend) {
                    // ChunkedInput.nextChunk() returned null and it has
                    // not reached at the end of input. Let's wait until
                    // more chunks arrive. Nothing to write or notify.
                    break;
                }

                //message为null，给一个空的ByteBuf对象
                if (message == null) {
                    // If message is null write an empty ByteBuf.
                    // See https://github.com/netty/netty/issues/1671
                    message = Unpooled.EMPTY_BUFFER;
                }

                // Flush each chunk to conserve memory
                //向底层输出message
                ChannelFuture f = ctx.writeAndFlush(message);

                if (endOfInput) {//如果块数据已经结束
                    queue.remove(); //移除

                    //如果f已经输出完毕
                    if (f.isDone()) {
                        handleEndOfInputFuture(f, currentWrite);
                    } else {
                        // Register a listener which will close the input once the write is complete.
                        // This is needed because the Chunk may have some resource bound that can not
                        // be closed before its not written.
                        //
                        // See https://github.com/netty/netty/issues/303
                        //如果没有立即完成，则注册回调事件
                        f.addListener(new ChannelFutureListener() {
                            @Override
                            public void operationComplete(ChannelFuture future) {
                                handleEndOfInputFuture(future, currentWrite);
                            }
                        });
                    }
                } else { //如果块数据没写完
                    //是否需要重新开始
                    final boolean resume = !channel.isWritable();
                    if (f.isDone()) { //上一个块底层socket写入成功
                        handleFuture(f, currentWrite, resume);
                    } else {
                        //没有立即返回结果，注册事件
                        f.addListener(new ChannelFutureListener() {
                            @Override
                            public void operationComplete(ChannelFuture future) {
                                handleFuture(future, currentWrite, resume);
                            }
                        });
                    }
                }
                requiresFlush = false;
            } else {
                //不是块消息，则从队列中移除
                queue.remove();
                //向下传递后把requiresFlush设置为true
                ctx.write(pendingMessage, currentWrite.promise);
                requiresFlush = true;
            }

            //如果channel关闭了，则处理关闭逻辑，释放资源。
            if (!channel.isActive()) {
                discard(new ClosedChannelException());
                break;
            }
        }

        //如果上面没有循环，这里需要向底层传递flush方法。
        if (requiresFlush) {
            ctx.flush();
        }
    }

    private static void handleEndOfInputFuture(ChannelFuture future, PendingWrite currentWrite) {
        ChunkedInput input = (ChunkedInput) currentWrite.msg;
        if (!future.isSuccess()) { //底层socket输出失败
            closeInput(input); //关闭资源
            currentWrite.fail(future.cause()); //设置失败
        } else {
            // read state of the input in local variables before closing it
            long inputProgress = input.progress();
            long inputLength = input.length();
            closeInput(input); //关闭资源
            currentWrite.progress(inputProgress, inputLength); //设置进度
            currentWrite.success(inputLength); //设置成功
        }
    }

    private void handleFuture(ChannelFuture future, PendingWrite currentWrite, boolean resume) {
        ChunkedInput input = (ChunkedInput) currentWrite.msg;
        if (!future.isSuccess()) { //写入失败
            closeInput(input); //关闭资源
            currentWrite.fail(future.cause());//设置失败
        } else {
            //更新进度
            currentWrite.progress(input.progress(), input.length());

            //如果resume=false，说明当时channel还是可以写的，则顶层循环会继续
            //不需要在这里触发继续写的逻辑
            //否则此时再判断一下isWritable，如果可以写，则触发继续写的逻辑
            if (resume && future.channel().isWritable()) {
                resumeTransfer();
            }
        }
    }

    private static void closeInput(ChunkedInput chunks) {
        try {
            chunks.close();
        } catch (Throwable t) {
            if (logger.isWarnEnabled()) {
                logger.warn("Failed to close a chunked input.", t);
            }
        }
    }

    //封装原始消息
    private static final class PendingWrite {
        final Object msg; //等待输出的消息对象
        final ChannelPromise promise; //任务句柄

        PendingWrite(Object msg, ChannelPromise promise) {
            this.msg = msg;
            this.promise = promise;
        }

        //设置失败
        void fail(Throwable cause) {
            ReferenceCountUtil.release(msg);
            promise.tryFailure(cause);
        }

        //设置成功
        void success(long total) {
            if (promise.isDone()) {
                // No need to notify the progress or fulfill the promise because it's done already.
                return;
            }
            //更新进度
            progress(total, total);
            promise.trySuccess();
        }

        //更新进度
        void progress(long progress, long total) {
            if (promise instanceof ChannelProgressivePromise) {
                ((ChannelProgressivePromise) promise).tryProgress(progress, total);
            }
        }
    }
}