文件下载输出--零拷贝
- Linux探秘之用户态与内核态
- JVM致命错误日志(hs_err_pid.log)分析
###问题
在并发情况下，从远程下载文件，copy到outPutStream中，再删除文件，服务会挂掉。
#
# A fatal error has been detected by the Java Runtime Environment:
#
# SIGBUS (0x7) at pc=0x00007fecb8373050, pid=36878, tid=0x00007fec7ce67700
#
# JRE version: Java(TM) SE Runtime Environment (8.0_162-b12) (build 1.8.0_162-b12)
# Java VM: Java HotSpot(TM) 64-Bit Server VM (25.162-b12 mixed mode linux-amd64 compressed oops)
# Problematic frame:
# V [libjvm.so+0x803050]
#
# Failed to write core dump. Core dumps have been disabled. To enable core dumping, try "ulimit -c unlimited" before starting Java again
#
# If you would like to submit a bug report, please visit:
# http://bugreport.java.com/bugreport/crash.jsp
#
--------------- T H R E A D ---------------
Current thread (0x00007fecb0d5d800): JavaThread "http-nio-8810-exec-2" daemon [_thread_in_vm, id=36925, stack(0x00007fec7cd67000,0x00007fec7ce68000)]
siginfo: si_signo: 7 (SIGBUS), si_code: 2 (BUS_ADRERR), si_addr: 0x00007fec5e323726
Registers:
RAX=0x00007fec5e32471e, RBX=0x00007fecb0d5d800, RCX=0x00000000efeb0b78, RDX=0xfffffffffffffe04
RSP=0x00007fec7ce614a8, RBP=0x00007fec7ce61500, RSI=0x00000000efeafb80, RDI=0x00007fec5e323726
R8 =0x0000000000000800, R9 =0x0000000000000010, R10=0x00007feca175eaa7, R11=0x00000000efeafb70
R12=0x0000000000000000, R13=0x00007fec7ce61530, R14=0x00007fecb8b3bc40, R15=0x00007fecb8b40a00
RIP=0x00007fecb8373050, EFLAGS=0x0000000000010282, CSGSFS=0x0000000000000033, ERR=0x0000000000000004
TRAPNO=0x000000000000000e
Top of Stack: (sp=0x00007fec7ce614a8)
0x00007fec7ce614a8: 00007fecb86126ff 0000000000000010
0x00007fec7ce614b8: 00007fec5e323726 00007fecb0d5d800
0x00007fec7ce614c8: 00000000eff378a8 ffffffffffffffff
0x00007fec7ce614d8: 0000000000001000 0000000000000000
0x00007fec7ce614e8: 0000000000000010 0000000000000000
0x00007fec7ce614f8: 00007fecb0d5d800 00007fec7ce61550
0x00007fec7ce61508: 00007feca175eb21 0000000000001000
0x00007fec7ce61518: 00000000c42b9a58 0000000000000000
0x00007fec7ce61528: 00007fec5eeadc30 00000000efeafb70
0x00007fec7ce61538: 00007fecb0d5d800 0000000000000000
0x00007fec7ce61548: 00000000000000f4 00000000efeafa28
0x00007fec7ce61558: 00007feca25e3bac 0000000100855858
0x00007fec7ce61568: 00007fec5ee73d68 00007fec7ce615b0
0x00007fec7ce61578: 00007fecb81b488f 00007fecb0d5d800
0x00007fec7ce61588: 00007fecb0d5e190 00007fec7ce61740
0x00007fec7ce61598: 00007fec5ee73d68 00007fec7ce61740
0x00007fec7ce615a8: 00007fec7ce61730 00007fec5e323726
0x00007fec7ce615b8: 0000000000001000 0000000000001000
0x00007fec7ce615c8: 0000000000000010 00000000efeafb70
0x00007fec7ce615d8: 00007fecb0d5e060 00000000efeafa28
0x00007fec7ce615e8: 00007feca25e31b4 00007fec7ce61650
0x00007fec7ce615f8: 00007fecb7e50f7b 00007fec7ce62000
0x00007fec7ce61608: 0000000000000000 0000000000000000
0x00007fec7ce61618: 00007fecb0d5d858 000000000002eb89
0x00007fec7ce61628: 0000000000000004 00007fec7ce61650
0x00007fec7ce61638: 0000000000000c00 0000000000000000
0x00007fec7ce61648: 0000000000000000 00007fec7ce61600
0x00007fec7ce61658: 00007fecb0d5d800 00007fec00000001
0x00007fec7ce61668: 0000000100000208 00007fec7ce61740
0x00007fec7ce61678: 00000000efeafb08 00001000003cd726
0x00007fec7ce61688: 0000000000000000 00000000efff1ae0
0x00007fec7ce61698: 00007fec7ce61740 00007fec7ce61730
Instructions: (pc=0x00007fecb8373050)
0x00007fecb8373030: 08 89 71 08 48 83 c1 04 49 f7 c0 01 00 00 00 74
0x00007fecb8373040: 0a 66 42 8b 74 47 fe 66 89 71 08 c3 0f 1f 40 00
0x00007fecb8373050: 48 8b 74 d0 e8 48 89 74 d1 e8 48 8b 74 d0 f0 48
0x00007fecb8373060: 89 74 d1 f0 48 8b 74 d0 f8 48 89 74 d1 f8 48 8b
Register to memory mapping:
RAX=0x00007fec5e32471e is an unknown value
RBX=0x00007fecb0d5d800 is a thread
RCX=0x00000000efeb0b78 is an oop
没有打印stack异常信息（暂时没有找到方法解决）。
#
# A fatal error has been detected by the Java Runtime Environment:
#
# SIGBUS (0x7) at pc=0x00007f634fdb90b0, pid=5924, tid=140064101504768
#
# JRE version: Java(TM) SE Runtime Environment (7.0_45-b18) (build 1.7.0_45-b18)
# Java VM: Java HotSpot(TM) 64-Bit Server VM (24.45-b08 mixed mode linux-amd64 compressed oops)
# Problematic frame:
# V [libjvm.so+0x73e0b0] _Copy_conjoint_jshorts_atomic+0x60
#
# Core dump written. Default location: /app/core or core.5924
#
# If you would like to submit a bug report, please visit:
# http://bugreport.sun.com/bugreport/crash.jsp
#
--------------- T H R E A D ---------------
Current thread (0x000000000240b800): JavaThread "http-nio-18080-exec-88" daemon [_thread_in_vm, id=6120, stack(0x00007f6337001000,0x00007f6337042000)]
siginfo:si_signo=SIGBUS: si_errno=0, si_code=2 (BUS_ADRERR), si_addr=0x00007f63367bf38e
Registers:
RAX=0x00007f63367c0386, RBX=0x000000000240b800, RCX=0x00000007dc043928, RDX=0xfffffffffffffe04
RSP=0x00007f633703ee38, RBP=0x00007f633703ee90, RSI=0x00000007dc042930, RDI=0x00007f63367bf38e
R8 =0x0000000000000800, R9 =0x0000000000000010, R10=0x00007f634501270c, R11=0x00007f634ff22a90
R12=0x0000000000000000, R13=0x00007f633703ef68, R14=0x0000000000000010, R15=0x00007f63367bf38e
RIP=0x00007f634fdb90b0, EFLAGS=0x0000000000010282, CSGSFS=0x0000000000000033, ERR=0x0000000000000004
TRAPNO=0x000000000000000e
Top of Stack: (sp=0x00007f633703ee38)
0x00007f633703ee38: 00007f634ffe6ec2 000000000240b800
0x00007f633703ee48: 000027440240b800 000000000240b800
0x00007f633703ee58: 00007f634ccc13b0 0000000000000002
0x00007f633703ee68: 00000006b00ea120 0000000000000000
0x00007f633703ee78: 00000006b00ea120 00007f633703ef88
0x00007f633703ee88: 000000000240b800 00007f633703ef28
0x00007f633703ee98: 00007f6345012738 0000000000001000
0x00007f633703eea8: 000000000240c708 0000000000000000
0x00007f633703eeb8: 000000000240c6f8 000000000240c6f8
0x00007f633703eec8: 00007f634cc9d110 0000000000000010
0x00007f633703eed8: 000000000240c6f8 000000000240b800
0x00007f633703eee8: 00007f633703eee8 0000000000000000
0x00007f633703eef8: 00007f633703ef88 00000006b00ecd50
0x00007f633703ef08: 0000000000000000 00000006b00ea120
0x00007f633703ef18: 0000000000000000 00007f633703ef48
0x00007f633703ef28: 00007f633703efd0 00007f6345006058
0x00007f633703ef38: 0000000000000000 00007f634500ecd8
0x00007f633703ef48: 0000000000001000 00007f633703efa0
0x00007f633703ef58: 0000000000000010 000000000240c708
0x00007f633703ef68: 00000007dc042920 00007f63367bf38e
0x00007f633703ef78: 0000000000000000 0000000000000000
0x00007f633703ef88: 00000006c0000810 00007f633703ef90
0x00007f633703ef98: 00000006b0176b12 00007f633703f040
0x00007f633703efa8: 00000006b0177880 0000000000000000
0x00007f633703efb8: 00000006b0176b40 00007f633703ef48
0x00007f633703efc8: 00007f633703f000 00007f633703f088
0x00007f633703efd8: 00007f6345006058 0000000000001000
0x00007f633703efe8: 0000000000000000 0000000000000010
0x00007f633703eff8: 0000000000000000 0000000000001000
0x00007f633703f008: 0000000000001000 0000000000000000
0x00007f633703f018: 0000000000000000 0000000000000010
0x00007f633703f028: 00007f6345006175 00000007dc042920
Instructions: (pc=0x00007f634fdb90b0)
0x00007f634fdb9090: 08 89 71 08 48 83 c1 04 49 f7 c0 01 00 00 00 74
0x00007f634fdb90a0: 0a 66 42 8b 74 47 fe 66 89 71 08 c3 0f 1f 40 00
0x00007f634fdb90b0: 48 8b 74 d0 e8 48 89 74 d1 e8 48 8b 74 d0 f0 48
0x00007f634fdb90c0: 89 74 d1 f0 48 8b 74 d0 f8 48 89 74 d1 f8 48 8b
Register to memory mapping:
RAX=0x00007f63367c0386 is an unknown value
RBX=0x000000000240b800 is a thread
RCX=0x00000007dc043928 is an unknown value
RDX=0xfffffffffffffe04 is an unknown value
RSP=0x00007f633703ee38 is pointing into the stack for thread: 0x000000000240b800
RBP=0x00007f633703ee90 is pointing into the stack for thread: 0x000000000240b800
RSI=0x00000007dc042930 is an unknown value
RDI=0x00007f63367bf38e is an unknown value
R8 =0x0000000000000800 is an unknown value
R9 =0x0000000000000010 is an unknown value
R10=0x00007f634501270c is at code_begin+620 in an Interpreter codelet
method entry point (kind = native) [0x00007f63450124a0, 0x00007f6345012d00] 2144 bytes
R11=0x00007f634ff22a90: in /app/jdk1.7.0_45/jre/lib/amd64/server/libjvm.so at 0x00007f634f67b000
R12=0x0000000000000000 is an unknown value
R13=0x00007f633703ef68 is pointing into the stack for thread: 0x000000000240b800
R14=0x0000000000000010 is an unknown value
R15=0x00007f63367bf38e is an unknown value
Stack: [0x00007f6337001000,0x00007f6337042000], sp=0x00007f633703ee38, free space=247k
Native frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code)
V [libjvm.so+0x73e0b0] _Copy_conjoint_jshorts_atomic+0x60
j sun.misc.Unsafe.copyMemory(Ljava/lang/Object;JLjava/lang/Object;JJ)V+0
j java.nio.Bits.copyToArray(JLjava/lang/Object;JJJ)V+42
j java.nio.DirectByteBuffer.get([BII)Ljava/nio/ByteBuffer;+98
j com.itextpdf.text.io.ByteBufferRandomAccessSource.get(J[BII)I+66
j com.itextpdf.text.io.MappedChannelRandomAccessSource.get(J[BII)I+27
j com.itextpdf.text.io.FileChannelRandomAccessSource.get(J[BII)I+10
Java frames: (J=compiled Java code, j=interpreted, Vv=VM code)
j sun.misc.Unsafe.copyMemory(Ljava/lang/Object;JLjava/lang/Object;JJ)V+0
j java.nio.Bits.copyToArray(JLjava/lang/Object;JJJ)V+42
j java.nio.DirectByteBuffer.get([BII)Ljava/nio/ByteBuffer;+98
j com.itextpdf.text.io.ByteBufferRandomAccessSource.get(J[BII)I+66
j com.itextpdf.text.io.MappedChannelRandomAccessSource.get(J[BII)I+27
j com.itextpdf.text.io.FileChannelRandomAccessSource.get(J[BII)I+10
J com.itextpdf.text.pdf.RandomAccessFileOrArray.read()I
j com.itextpdf.text.pdf.PRTokeniser.nextToken()Z+6
j com.itextpdf.text.pdf.PRTokeniser.nextValidToken()V+10
j com.itextpdf.text.pdf.PdfReader.readXrefSection()Lcom/itextpdf/text/pdf/PdfDictionary;+254
j com.itextpdf.text.pdf.PdfReader.readXref()V+162
j com.itextpdf.text.pdf.PdfReader.readPdf()V+26
j com.itextpdf.text.pdf.PdfReader.(Lcom/itextpdf/text/io/RandomAccessSource;Z[BLjava/security/cert/Certificate;Ljava/security/Key;Ljava/lang/String;Lcom/itextpdf/text/pdf/security/ExternalDecryptionProcess;Z)V+155
j com.itextpdf.text.pdf.PdfReader.(Ljava/lang/String;[BZ)V+29
j com.itextpdf.text.pdf.PdfReader.(Ljava/lang/String;[B)V+4
j com.itextpdf.text.pdf.PdfReader.(Ljava/lang/String;)V+6
j com.dimeng.p2p.user.servlets.financing.agreement.CompNotGuaranteeDeal.createDownLoadLink(Ljavax/servlet/http/HttpServletRequest;Ljavax/servlet/http/HttpServletResponse;Lcom/dimeng/framework/http/servlet/Controller;Ljava/lang/String;Lcom/dimeng/p2p/S51/entities/T5140;Lcom/dimeng/framework/config/ConfigureProvider;)V+43
j com.dimeng.p2p.user.servlets.financing.agreement.CompNotGuaranteeDeal.searchAgm(Ljavax/servlet/http/HttpServletRequest;Ljavax/servlet/http/HttpServletResponse;Lcom/dimeng/framework/service/ServiceSession;Lcom/dimeng/framework/config/ConfigureProvider;)V+114
j com.dimeng.p2p.user.servlets.financing.agreement.CompNotGuaranteeDeal.processPost(Ljavax/servlet/http/HttpServletRequest;Ljavax/servlet/http/HttpServletResponse;Lcom/dimeng/framework/service/ServiceSession;)V+201
j com.dimeng.framework.http.servlet.AbstractServlet.doPost(Ljavax/servlet/http/HttpServletRequest;Ljavax/servlet/http/HttpServletResponse;)V+182
j javax.servlet.http.HttpServlet.service(Ljavax/servlet/http/HttpServletRequest;Ljavax/servlet/http/HttpServletResponse;)V+149
j javax.servlet.http.HttpServlet.service(Ljavax/servlet/ServletRequest;Ljavax/servlet/ServletResponse;)V+30
j org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(Ljavax/servlet/ServletRequest;Ljavax/servlet/ServletResponse;)V+446
j org.apache.catalina.core.ApplicationFilterChain.doFilter(Ljavax/servlet/ServletRequest;Ljavax/servlet/ServletResponse;)V+101
j org.apache.tomcat.websocket.server.WsFilter.doFilter(Ljavax/servlet/ServletRequest;Ljavax/servlet/ServletResponse;Ljavax/servlet/FilterChain;)V+21
j org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(Ljavax/servlet/ServletRequest;Ljavax/servlet/ServletResponse;)V+151
j org.apache.catalina.core.ApplicationFilterChain.doFilter(Ljavax/servlet/ServletRequest;Ljavax/servlet/ServletResponse;)V+101
j com.dimeng.p2p.user.filter.TraceIdFilter.doFilter(Ljavax/servlet/ServletRequest;Ljavax/servlet/ServletResponse;Ljavax/servlet/FilterChain;)V+147
j org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(Ljavax/servlet/ServletRequest;Ljavax/servlet/ServletResponse;)V+151
j org.apache.catalina.core.ApplicationFilterChain.doFilter(Ljavax/servlet/ServletRequest;Ljavax/servlet/ServletResponse;)V+101
j org.apache.catalina.core.StandardWrapperValve.invoke(Lorg/apache/catalina/connector/Request;Lorg/apache/catalina/connector/Response;)V+783
j org.apache.catalina.core.StandardContextValve.invoke(Lorg/apache/catalina/connector/Request;Lorg/apache/catalina/connector/Response;)V+166
j org.apache.catalina.authenticator.AuthenticatorBase.invoke(Lorg/apache/catalina/connector/Request;Lorg/apache/catalina/connector/Response;)V+449
j org.apache.catalina.core.StandardHostValve.invoke(Lorg/apache/catalina/connector/Request;Lorg/apache/catalina/connector/Response;)V+148
j org.apache.catalina.valves.ErrorReportValve.invoke(Lorg/apache/catalina/connector/Request;Lorg/apache/catalina/connector/Response;)V+6
j org.apache.catalina.core.StandardEngineValve.invoke(Lorg/apache/catalina/connector/Request;Lorg/apache/catalina/connector/Response;)V+71
j org.apache.catalina.connector.CoyoteAdapter.service(Lorg/apache/coyote/Request;Lorg/apache/coyote/Response;)V+196
j org.apache.coyote.http11.AbstractHttp11Processor.process(Lorg/apache/tomcat/util/net/SocketWrapper;)Lorg/apache/tomcat/util/net/AbstractEndpoint$Handler$SocketState;+632
j org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(Lorg/apache/tomcat/util/net/SocketWrapper;Lorg/apache/tomcat/util/net/SocketStatus;)Lorg/apache/tomcat/util/net/AbstractEndpoint$Handler$SocketState;+239
j org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(Ljava/nio/channels/SelectionKey;Lorg/apache/tomcat/util/net/NioEndpoint$KeyAttachment;)V+140
j org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.run()V+94
j java.util.concurrent.ThreadPoolExecutor.runWorker(Ljava/util/concurrent/ThreadPoolExecutor$Worker;)V+95
j java.util.concurrent.ThreadPoolExecutor$Worker.run()V+5
j org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run()V+4
j java.lang.Thread.run()V+11
v ~StubRoutines::call_stub
测试代码
@RequestMapping(value = "/api/agm/view1")
@ApiOperation("从fastdfs下载一个文件 并输出，输出完成后删除文件")
public void agmView1(HttpServletResponse response) {
// int viewCount = VIEW_COUNT.getAndIncrement();
// log.info("总浏览次数: " + viewCount);
String fileId = "group1/M00/07/B4/rB0D9Vzt7waAOmUKAD3q5hmRBIg203.pdf";
File agreementFile;
try {
agreementFile = YyfaxFileUtils.downloadFile(fileId);
}catch (Exception e){
log.error("exception: " + e.getMessage());
return;
}
PdfReader reader = null;
PdfStamper stamper = null;
// 读取文件，添加链接/文字/签章
try {
reader = new PdfReader(agreementFile.getPath());
stamper = new PdfStamper(reader, response.getOutputStream());
} catch (Exception e){
log.error("生成合同失败，失败id：", e);
} finally{
if (stamper != null) {
try {
stamper.close();
} catch (Exception e) {
log.error("stamper", e);
}
}
if (reader != null) {
try {
reader.close();
} catch (Exception e) {
log.error("reader", e);
}
}
YyfaxFileUtils.deleteFile(fileId);
}
}
@RequestMapping(value = "/api/agm/view2")
@ApiOperation("从fastdfs下载一个文件 并输出，输出完成后不删除文件")
public void agmView2(HttpServletResponse response) {
String fileId = "group1/M00/07/B4/rB0D9Vzt7waAOmUKAD3q5hmRBIg203.pdf";
File agreementFile;
try {
agreementFile = YyfaxFileUtils.downloadFile(fileId);
}catch (Exception e){
log.error("exception: " + e.getMessage());
return;
}
PdfReader reader = null;
PdfStamper stamper = null;
// 读取文件，添加链接/文字/签章
try {
reader = new PdfReader(agreementFile.getPath());
stamper = new PdfStamper(reader, response.getOutputStream());
} catch (Exception e){
log.error("生成合同失败，失败id：", e);
} finally{
if (stamper != null) {
try {
stamper.close();
} catch (Exception e) {
log.error("stamper", e);
}
}
if (reader != null) {
try {
reader.close();
} catch (Exception e) {
log.error("reader", e);
}
}
// YyfaxFileUtils.deleteFile(fileId);
}
}
@RequestMapping(value = "/api/agm/view3")
@ApiOperation("从fastdfs下载文件流 并输出")
public void agmView3(HttpServletResponse response) {
String fileId = "group1/M00/07/B4/rB0D9Vzt7waAOmUKAD3q5hmRBIg203.pdf";
PdfReader reader = null;
PdfStamper stamper = null;
// 读取文件，添加链接/文字/签章
try {
reader = new PdfReader(YyfaxFileUtils.downloadStream(fileId));
stamper = new PdfStamper(reader, response.getOutputStream());
} catch (Exception e){
log.error("生成合同失败，失败id：", e);
} finally{
if (stamper != null) {
try {
stamper.close();
} catch (Exception e) {
log.error("stamper", e);
}
}
if (reader != null) {
try {
reader.close();
} catch (Exception e) {
log.error("reader", e);
}
}
}
}
@RequestMapping(value = "/api/agm/view4")
@ApiOperation("随机获取本地文件 并输出")
public void agmView4(HttpServletResponse response) {
PdfReader reader = null;
PdfStamper stamper = null;
// 读取文件，添加链接/文字/签章
try {
File dir=new File(path);
File[] files = dir.listFiles();
Random rand = new Random();
File file = files[rand.nextInt(files.length)];
reader = new PdfReader(file.getAbsolutePath());
stamper = new PdfStamper(reader, response.getOutputStream());
} catch (Exception e){
log.error("生成合同失败，失败id：", e);
} finally{
if (stamper != null) {
try {
stamper.close();
} catch (Exception e) {
log.error("stamper", e);
}
}
if (reader != null) {
try {
reader.close();
} catch (Exception e) {
log.error("reader", e);
}
}
}
}
@RequestMapping(value = "/api/agm/view5")
@ApiOperation("获取本地一个文件 并输出")
public void agmView5(HttpServletResponse response) {
PdfReader reader = null;
PdfStamper stamper = null;
// 读取文件，添加链接/文字/签章
try {
reader = new PdfReader(fileName);
stamper = new PdfStamper(reader, response.getOutputStream());
} catch (Exception e){
log.error("生成合同失败，失败id：", e);
} finally{
if (stamper != null) {
try {
stamper.close();
} catch (Exception e) {
log.error("stamper", e);
}
}
if (reader != null) {
try {
reader.close();
} catch (Exception e) {
log.error("reader", e);
}
}
}
}
###测试结果
1./api/agm/view1从fastdfs下载一个文件 并输出，输出完成后删除文件 50个1S压蹦了 服务直接挂掉 hs_err_pid41374.log
2./api/agm/view2从fastdfs下载一个文件 并输出，输出完成后不删除文件 压蹦了 服务直接挂掉 hs_err_pid41871.log
3./api/agm/view3 从fastdfs下载文件流 并输出 50个1S 没有问题
4./api/agm/view4 随机获取本地文件 并输出 50个1S 没有问题
5. /api/agm/view5 获取本地一个文件 并输出 50个1S 没有问题
结论
- 大致认为是在并发情况下载同一个文件，再删除这个文件出现的问题。
- itextPDF操作涉及到堆外内存，unsafe方法。
- 解决方法：改为直接下载文件流。原则上尽量避免直接操作文件，文件操作很多都是同步操作。