基于netty4的文件下载

在使用过程中发现这种方式下载文件，是在将文件加载至内存后再往服务端发送，会随之文件的大小而占用内存。我测试中下载2G的文件，发现内存升高，下载10G文件时无法下载。

客户端：

public class DownLoadClient {
    private StringBuffer resultBuffer = new StringBuffer();
    private EventLoopGroup group = null;
    private HttpDataFactory factory = null;

    private ChannelFuture future = null;

    private Object waitObject = new Object();

    private String downFileName = null;

    public DownLoadClient(String host, int port) throws Exception {
        this.group = new NioEventLoopGroup();
        this.factory = new DefaultHttpDataFactory(DefaultHttpDataFactory.MINSIZE);

        Bootstrap b = new Bootstrap();
        b.option(ChannelOption.TCP_NODELAY, true);
        b.option(ChannelOption.SO_RCVBUF, 1048576*200);
        b.option(ChannelOption.SO_KEEPALIVE, true);

        b.group(group).channel(NioSocketChannel.class);
        b.handler(new DownLoadClientIntializer());

        this.future = b.connect(host, port).sync();
    }

    public void downLoadFile(String fileName) {

        if(fileName == null || "".equals(fileName) || fileName.indexOf(".") == -1) {
            System.out.println("下载的文件名未正确指定...");
            return;
        }
        this.downFileName = fileName;
        try {
            HttpRequest request = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, fileName);

            Channel channel = this.future.channel();
            if(channel.isActive() && channel.isWritable()) {
                channel.writeAndFlush(request);
            }
            channel.closeFuture().sync();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    public void shutdownClient() {
        // 等待数据的传输通道关闭
        group.shutdownGracefully();
        factory.cleanAllHttpDatas();
    }

    public boolean isCompleted() {
        while(waitObject != null) {
            //当通道处于开通和活动时，处于等待
        }
        if(resultBuffer.length() > 0) {
            if("200".equals(resultBuffer.toString())) {
                resultBuffer.setLength(0);
                return true;
            }
        }

        return false;
    }

    private class DownLoadClientIntializer extends ChannelInitializer {
        @Override
        protected void initChannel(SocketChannel ch) throws Exception {
            ChannelPipeline pipeline = ch.pipeline();

            pipeline.addLast("decoder", new HttpResponseDecoder());
            pipeline.addLast("encoder", new HttpRequestEncoder());
            pipeline.addLast("chunkedWriter", new ChunkedWriteHandler());

            pipeline.addLast("dispatcher", new DownLoadClientHandler());
        }
    }

    private class DownLoadClientHandler extends SimpleChannelInboundHandler {
        private boolean readingChunks = false;
        private File downloadFile = null;
        private FileOutputStream fOutputStream = null;

        private int succCode = 200;

        protected void channelRead0(ChannelHandlerContext ctx, HttpObject msg)
                throws Exception {
            if (msg instanceof HttpResponse) {
                HttpResponse response = (HttpResponse) msg;

                succCode = response.getStatus().code();

                if (succCode == 200) {
                    setDownLoadFile();
                    readingChunks = true;
                }
            }

            if (msg instanceof HttpContent) {
                HttpContent chunk = (HttpContent) msg;
                if (chunk instanceof LastHttpContent) {
                    readingChunks = false;
                }

                ByteBuf buffer = chunk.content();
                byte[] dst = new byte[buffer.readableBytes()];
                if(succCode == 200) {
                    while(buffer.isReadable()) {
                        buffer.readBytes(dst);
                        fOutputStream.write(dst);
                    }

                    if (null != fOutputStream) {
                        fOutputStream.flush();
                    }
                }
            }

            if (!readingChunks) {
                if (null != fOutputStream) {
                    fOutputStream.flush();
                    fOutputStream.close();

                    downloadFile = null;
                    fOutputStream = null;

                    resultBuffer.append(succCode);
                }
                ctx.channel().close();
            }
        }

        private void setDownLoadFile() throws Exception {
            if(null == fOutputStream) {
                downloadFile = DBTools.getDownlaodFile(downFileName);
                if(!downloadFile.exists()) {
                    downloadFile.createNewFile();
                }
                fOutputStream = new FileOutputStream(downloadFile);
            }
        }

        @Override
        public void channelInactive(ChannelHandlerContext ctx) throws Exception {
            waitObject = null;
        }

        @Override
        public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause)
                throws Exception {
            resultBuffer.setLength(0);
            resultBuffer.append(500);
            System.out.println("管道异常：" + cause.getMessage());
            cause.printStackTrace();
            ctx.channel().close();
        }
    }
}

 

服务端：

public class DBServer extends Thread {

    //单实例
    private static DBServer dbServer = null;

    //定时调度的周期实例
    private static Scheduler sched = null;

    private EventLoopGroup bossGroup = null;
    private EventLoopGroup workerGroup = null;
    //创建实例
    public static DBServer newBuild() {
        if(dbServer == null) {
            dbServer = new DBServer();
        }
        return dbServer;
    }

    public void run() {
        try {
            startServer();
        } catch(Exception e) {
            System.out.println("数据服务启动出现异常："+e.toString());
            e.printStackTrace();
        }
    }

    private void startServer() throws Exception {
        bossGroup = new NioEventLoopGroup();
        workerGroup = new NioEventLoopGroup();

        try {
            ServerBootstrap b = new ServerBootstrap();

            b.group(bossGroup, workerGroup);

            b.option(ChannelOption.TCP_NODELAY, true);
            b.option(ChannelOption.SO_TIMEOUT, 60000);
            b.option(ChannelOption.SO_SNDBUF, 1048576*200);

            b.option(ChannelOption.SO_KEEPALIVE, true);

            b.channel(NioServerSocketChannel.class);
            b.childHandler(new DBServerInitializer());

            // 服务器绑定端口监听
            ChannelFuture f = b.bind(DBConfig.curHost.getIp(), DBConfig.curHost.getPort()).sync();

            System.out.println("数据服务："+DBConfig.curHost.getServerHost()+"启动完成...");
            // 监听服务器关闭监听
            f.channel().closeFuture().sync();
        } finally {
            bossGroup.shutdownGracefully();
            workerGroup.shutdownGracefully();
        }
    }
}

 

public class DBServerHandler extends SimpleChannelInboundHandler {

    private static final HttpDataFactory factory = new DefaultHttpDataFactory(DefaultHttpDataFactory.MINSIZE);

    private String uri = null;

    private HttpRequest request = null;

    private HttpPostRequestDecoder decoder;

    //message、download、upload
    private String type = "message";

    public static final String HTTP_DATE_FORMAT = "EEE, dd MMM yyyy HH:mm:ss zzz";
    public static final String HTTP_DATE_GMT_TIMEZONE = "GMT";
    public static final int HTTP_CACHE_SECONDS = 60;

    static {
        DiskFileUpload.baseDirectory = DBConfig.curHost.getZipPath();
    }

    @Override
    public void channelRead0(ChannelHandlerContext ctx, HttpObject msg) throws Exception {
        if (msg instanceof HttpRequest) {
            request = (HttpRequest) msg;

            uri = sanitizeUri(request.getUri());

            //文件下载
            if (request.getMethod() == HttpMethod.GET) {
                type = "download";
                File file = DBTools.getDownlaodFile(uri);
                if (file.isHidden() || !file.exists() || !file.isFile()) {
                    writeResponse(ctx.channel(), HttpResponseStatus.INTERNAL_SERVER_ERROR, "下载的文件不存在...");
                    return;
                }

                // 随机文件读取，这种方式速度快
                RandomAccessFile raf = null;
                try {
                    raf = new RandomAccessFile(file, "r");
                } catch(FileNotFoundException e) {
                    writeResponse(ctx.channel(), HttpResponseStatus.INTERNAL_SERVER_ERROR, e.toString());
                    return;
                }

                writeDownLoadResponse(ctx, raf, file);
            }
    }

        if (decoder != null && msg instanceof HttpContent) {
            HttpContent chunk = (HttpContent) msg;

            try {
                decoder.offer(chunk);
            } catch (Exception e) {
                e.printStackTrace();
                writeResponse(ctx.channel(), HttpResponseStatus.INTERNAL_SERVER_ERROR, e.toString());
                ctx.channel().close();
                return;
            }

            readHttpDataChunkByChunk();

            if (chunk instanceof LastHttpContent) {
                writeResponse(ctx.channel(), HttpResponseStatus.OK, "");
                reset();
                return;
            }
        }
    }

    private String sanitizeUri(String uri) {
        try {
            uri = URLDecoder.decode(uri, "UTF-8");
        } catch(UnsupportedEncodingException e) {
            try {
                uri = URLDecoder.decode(uri, "ISO-8859-1");
            } catch(UnsupportedEncodingException e1) {
                throw new Error();
            }
        }

        return uri;
    }

    private void reset() {
        request = null;

        //销毁decoder释放所有的资源
        decoder.destroy();

        decoder = null;
    }

    /**
     * 通过chunk读取request，获取chunk数据
     * @throws IOException
     */
    private void readHttpDataChunkByChunk() throws IOException {
        try {
            while (decoder.hasNext()) {

                InterfaceHttpData data = decoder.next();
                if (data != null) {
                    try {
                        writeHttpData(data);
                    } finally {
                        data.release();
                    }
                }
            }
        } catch (EndOfDataDecoderException e1) {
            System.out.println("end chunk");
        }
    }

    private void writeHttpData(InterfaceHttpData data) throws IOException {
        if (data.getHttpDataType() == HttpDataType.FileUpload) {
            FileUpload fileUpload = (FileUpload) data;
            if (fileUpload.isCompleted()) {

                StringBuffer fileNameBuf = new StringBuffer();
                fileNameBuf.append(DiskFileUpload.baseDirectory)
                           .append(uri);

                fileUpload.renameTo(new File(fileNameBuf.toString()));
            }
        } else if (data.getHttpDataType() == HttpDataType.Attribute) {
            Attribute attribute = (Attribute) data;
            if(CommonParam.DOWNLOAD_COLLECTION.equals(attribute.getName())) {
                SynchMessageWatcher.newBuild().getMsgQueue().add(attribute.getValue());
            }
        }
    }

    private void writeDownLoadResponse(ChannelHandlerContext ctx, RandomAccessFile raf, File file) throws Exception {
        long fileLength = raf.length();

        //判断是否关闭请求响应连接
        boolean close = HttpHeaders.Values.CLOSE.equalsIgnoreCase(request.headers().get(CONNECTION))
                || request.getProtocolVersion().equals(HttpVersion.HTTP_1_0)
                && !HttpHeaders.Values.KEEP_ALIVE.equalsIgnoreCase(request.headers().get(CONNECTION));

        HttpResponse response = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
        HttpHeaders.setContentLength(response, fileLength);

        setContentHeader(response, file);

        if (!close) {
            response.headers().set(CONNECTION, HttpHeaders.Values.KEEP_ALIVE);
        }

        ctx.write(response);
        System.out.println("读取大小："+fileLength);

        final FileRegion region = new DefaultFileRegion(raf.getChannel(), 0, 1000);
        ChannelFuture writeFuture = ctx.write(region, ctx.newProgressivePromise());
        writeFuture.addListener(new ChannelProgressiveFutureListener() {
            public void operationProgressed(ChannelProgressiveFuture future, long progress, long total) {
                if (total < 0) {
                    System.err.println(future.channel() + " Transfer progress: " + progress);
                } else {
                    System.err.println(future.channel() + " Transfer progress: " + progress + " / " + total);
                }
            }

            public void operationComplete(ChannelProgressiveFuture future) {
            }
        });

        ChannelFuture lastContentFuture = ctx.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT);
        if(close) {
            raf.close();
            lastContentFuture.addListener(ChannelFutureListener.CLOSE);
        }
    }

    private static void setContentHeader(HttpResponse response, File file) {
        MimetypesFileTypeMap mimeTypesMap = new MimetypesFileTypeMap();
        response.headers().set(CONTENT_TYPE, mimeTypesMap.getContentType(file.getPath()));

        SimpleDateFormat dateFormatter = new SimpleDateFormat(HTTP_DATE_FORMAT, Locale.US);
        dateFormatter.setTimeZone(TimeZone.getTimeZone(HTTP_DATE_GMT_TIMEZONE));

        // Date header
        Calendar time = new GregorianCalendar();
        response.headers().set(DATE, dateFormatter.format(time.getTime()));

        // Add cache headers
        time.add(Calendar.SECOND, HTTP_CACHE_SECONDS);
        response.headers().set(EXPIRES, dateFormatter.format(time.getTime()));
        response.headers().set(CACHE_CONTROL, "private, max-age=" + HTTP_CACHE_SECONDS);
        response.headers().set(LAST_MODIFIED, dateFormatter.format(new Date(file.lastModified())));
    }

    private void writeResponse(Channel channel, HttpResponseStatus httpResponseStatus, String returnMsg) {
        String resultStr = "节点【"+DBConfig.curHost.getServerHost()+"】";
        if(httpResponseStatus.code() == HttpResponseStatus.OK.code()) {
            resultStr += "正常接收";
            if("message".equals(type)) {
                resultStr += "字符串。";
            } else if("upload".equals(type)) {
                resultStr += "上传文件。";
            } else if("download".equals(type)) {
                resultStr += "下载文件名。";
            }
        } else if(httpResponseStatus.code() == HttpResponseStatus.INTERNAL_SERVER_ERROR.code()) {
            resultStr += "接收";
            if("message".equals(type)) {
                resultStr += "字符串";
            } else if("upload".equals(type)) {
                resultStr += "上传文件";
            } else if("download".equals(type)) {
                resultStr += "下载文件名";
            }
            resultStr += "的过程中出现异常："+returnMsg;
        }
        //将请求响应的内容转换成ChannelBuffer.
        ByteBuf buf = copiedBuffer(resultStr, CharsetUtil.UTF_8);

        //判断是否关闭请求响应连接
        boolean close = HttpHeaders.Values.CLOSE.equalsIgnoreCase(request.headers().get(CONNECTION))
                || request.getProtocolVersion().equals(HttpVersion.HTTP_1_0)
                && !HttpHeaders.Values.KEEP_ALIVE.equalsIgnoreCase(request.headers().get(CONNECTION));

        //构建请求响应对象
        FullHttpResponse response = new DefaultFullHttpResponse(
                HttpVersion.HTTP_1_1, httpResponseStatus, buf);
        response.headers().set(CONTENT_TYPE, "text/plain; charset=UTF-8");

        if (!close) {
            //若该请求响应是最后的响应，则在响应头中没有必要添加'Content-Length'
            response.headers().set(CONTENT_LENGTH, buf.readableBytes());
        }

        //发送请求响应
        ChannelFuture future = channel.writeAndFlush(response);
        //发送请求响应操作结束后关闭连接
        if (close) {
            future.addListener(ChannelFutureListener.CLOSE);
        }
    }

    @Override
    public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
        cause.getCause().printStackTrace();
        writeResponse(ctx.channel(), HttpResponseStatus.INTERNAL_SERVER_ERROR, "数据文件通过过程中出现异常："+cause.getMessage().toString());
        ctx.channel().close();
    }
}

 

public class DBServerInitializer extends ChannelInitializer {

    @Override
    public void initChannel(SocketChannel ch) {
        ChannelPipeline pipeline = ch.pipeline();

        pipeline.addLast("decoder", new HttpRequestDecoder());
        pipeline.addLast("encoder", new HttpResponseEncoder());

        pipeline.addLast("deflater", new HttpContentCompressor());

        pipeline.addLast("handler", new DBServerHandler());
    }
}