JVM实践-记录OOM分析(十一)

说明

线上出现异常，但是查看节点状态正常，因为使用了容器,挂掉了会重启。因为jvm配置了-XX：HeapDumpOnOutOfMemoryError 参数，出现OMM就会将当时线程和jvm内存情况转存起来可参考:jmv参数配置

排查

1.将运维发过来的hprof导入到 vm 可参考 直通车

 

3.我们去当时的线程日志查看

4.搜索13219这个线程id找到线程查看堆栈信息

 5.分析堆栈找到业务代码 分析问题原因

具体看标红处 发现是导入功能问题 导致oom

"http-nio-8080-exec-11" daemon prio=5 tid=13219 RUNNABLE
    at java.lang.OutOfMemoryError.<init>(OutOfMemoryError.java:48)
    at org.apache.xmlbeans.impl.store.CharUtil.allocate(CharUtil.java:397)
       local variable: org.apache.xmlbeans.impl.store.CharUtil#2
    at org.apache.xmlbeans.impl.store.CharUtil.saveChars(CharUtil.java:506)
       local variable: org.apache.xmlbeans.impl.store.CharUtil#2
       local variable: java.lang.String#622892
       local variable: char[]#2750421
    at org.apache.xmlbeans.impl.store.Cur$CurLoadContext.text(Cur.java:2927)
       local variable: org.apache.xmlbeans.impl.store.Cur$CurLoadContext#1
    at org.apache.xmlbeans.impl.store.Cur$CurLoadContext.attr(Cur.java:3048)
       local variable: org.apache.xmlbeans.impl.store.Cur$CurLoadContext#1
       local variable: java.lang.String#622892
       local variable: org.apache.xmlbeans.impl.store.Xobj$AttrXobj#2119297
    at org.apache.xmlbeans.impl.store.Locale.loadNode(Locale.java:1440)
       local variable: org.apache.xmlbeans.impl.store.Locale#3
       local variable: com.sun.org.apache.xerces.internal.dom.DeferredElementNSImpl#1062106
       local variable: org.apache.xmlbeans.impl.store.Cur$CurLoadContext#1
       local variable: com.sun.org.apache.xerces.internal.dom.AttributeMap#1061610
    at org.apache.xmlbeans.impl.store.Locale.loadNodeChildren(Locale.java:1403)
       local variable: org.apache.xmlbeans.impl.store.Locale#3
       local variable: org.apache.xmlbeans.impl.store.Cur$CurLoadContext#1
       local variable: com.sun.org.apache.xerces.internal.dom.DeferredElementNSImpl#1062106
    at org.apache.xmlbeans.impl.store.Locale.loadNode(Locale.java:1445)
       local variable: org.apache.xmlbeans.impl.store.Cur$CurLoadContext#1
    at org.apache.xmlbeans.impl.store.Locale.loadNodeChildren(Locale.java:1403)
       local variable: org.apache.xmlbeans.impl.store.Locale#3
       local variable: org.apache.xmlbeans.impl.store.Cur$CurLoadContext#1
       local variable: com.sun.org.apache.xerces.internal.dom.DeferredElementNSImpl#243777
    at org.apache.xmlbeans.impl.store.Locale.loadNode(Locale.java:1445)
       local variable: org.apache.xmlbeans.impl.store.Cur$CurLoadContext#1
    at org.apache.xmlbeans.impl.store.Locale.loadNodeChildren(Locale.java:1403)
       local variable: org.apache.xmlbeans.impl.store.Locale#3
       local variable: org.apache.xmlbeans.impl.store.Cur$CurLoadContext#1
       local variable: com.sun.org.apache.xerces.internal.dom.DeferredElementNSImpl#1
    at org.apache.xmlbeans.impl.store.Locale.loadNode(Locale.java:1445)
       local variable: org.apache.xmlbeans.impl.store.Cur$CurLoadContext#1
    at org.apache.xmlbeans.impl.store.Locale.parseToXmlObject(Locale.java:1385)
       local variable: org.apache.xmlbeans.impl.store.Locale#3
       local variable: com.sun.org.apache.xerces.internal.dom.DeferredElementNSImpl#2
       local variable: org.apache.xmlbeans.impl.schema.SchemaTypeImpl#3
       local variable: org.apache.xmlbeans.XmlOptions#3
       local variable: org.apache.xmlbeans.impl.store.Cur$CurLoadContext#1
    at org.apache.xmlbeans.impl.store.Locale.parseToXmlObject(Locale.java:1370)
       local variable: org.apache.xmlbeans.impl.schema.SchemaTypeLoaderImpl#1
       local variable: com.sun.org.apache.xerces.internal.dom.DeferredElementNSImpl#2
       local variable: org.apache.xmlbeans.impl.schema.SchemaTypeImpl#3
       local variable: org.apache.xmlbeans.XmlOptions#3
       local variable: org.apache.xmlbeans.impl.store.Locale#3
       local variable: org.apache.xmlbeans.impl.store.Locale#3
    at org.apache.xmlbeans.impl.schema.SchemaTypeLoaderBase.parse(SchemaTypeLoaderBase.java:370)
       local variable: org.apache.xmlbeans.impl.schema.SchemaTypeLoaderImpl#1
       local variable: com.sun.org.apache.xerces.internal.dom.DeferredElementNSImpl#2
       local variable: org.apache.xmlbeans.impl.schema.SchemaTypeImpl#3
       local variable: org.apache.xmlbeans.XmlOptions#3
    at org.apache.poi.POIXMLTypeLoader.parse(POIXMLTypeLoader.java:164)
       local variable: java.io.ByteArrayInputStream#1
       local variable: org.apache.xmlbeans.impl.schema.SchemaTypeImpl#3
       local variable: org.apache.xmlbeans.XmlOptions#3
       local variable: com.sun.org.apache.xerces.internal.dom.DeferredDocumentImpl#50
    at org.openxmlformats.schemas.spreadsheetml.x2006.main.WorksheetDocument$Factory.parse(<unresolved string 0x0>)
       local variable: java.io.ByteArrayInputStream#1
       local variable: org.apache.xmlbeans.XmlOptions#3
    at org.apache.poi.xssf.usermodel.XSSFSheet.read(XSSFSheet.java:226)
       local variable: org.apache.poi.xssf.usermodel.XSSFSheet#1
       local variable: java.io.ByteArrayInputStream#1
    at org.apache.poi.xssf.usermodel.XSSFSheet.onDocumentRead(XSSFSheet.java:218)
       local variable: org.apache.poi.xssf.usermodel.XSSFSheet#1
    at org.apache.poi.xssf.usermodel.XSSFWorkbook.parseSheet(XSSFWorkbook.java:443)
       local variable: org.apache.poi.xssf.usermodel.XSSFWorkbook#1
       local variable: java.util.HashMap#17405
       local variable: org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTSheetImpl#1
       local variable: org.apache.poi.xssf.usermodel.XSSFSheet#1
    at org.apache.poi.xssf.usermodel.XSSFWorkbook.onDocumentRead(XSSFWorkbook.java:408)
       local variable: org.apache.poi.xssf.usermodel.XSSFWorkbook#1
       local variable: org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.WorkbookDocumentImpl#1
       local variable: org.apache.poi.xssf.model.ThemesTable#1
       local variable: java.util.HashMap#17405
       local variable: java.util.HashMap#17406
       local variable: org.openxmlformats.schemas.spreadsheetml.x2006.main.CTSheet[]#1
       local variable: org.openxmlformats.schemas.spreadsheetml.x2006.main.impl.CTSheetImpl#1
    at org.apache.poi.POIXMLDocument.load(POIXMLDocument.java:169)
       local variable: java.util.HashMap#17407
    at org.apache.poi.xssf.usermodel.XSSFWorkbook.<init>(XSSFWorkbook.java:300)
       local variable: org.apache.poi.xssf.usermodel.XSSFWorkbook#1
    at com.biaoguoworks.commons.utils.ExcelUtils.readExcel(ExcelUtils.java:204)
       local variable: class com.biaoguoworks.scm.logistics.dto.sorting.task.SkuSortingImportDTO
       local variable: java.lang.String#254630
       local variable: java.util.ArrayList#63983
       local variable: java.io.FileInputStream#8
    at com.biaoguoworks.scm.logistics.controller.demeter.web.SortingTagController.sortingImport(SortingTagController.java:168)
       local variable: com.biaoguoworks.scm.logistics.controller.demeter.web.SortingTagController#1
       local variable: com.biaoguoworks.scm.logistics.vo.sorting.SkuSortingImportVO#1
    at com.biaoguoworks.scm.logistics.controller.demeter.web.SortingTagController$$FastClassBySpringCGLIB$$9eba9d0d.invoke(<generated>)
    at org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:218)
       local variable: org.springframework.cglib.proxy.MethodProxy#4788
    at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:771)
    at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:163)
    at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:749)
       local variable: org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation#1
    at org.springframework.validation.beanvalidation.MethodValidationInterceptor.invoke(MethodValidationInterceptor.java:119)
       local variable: org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation#1
       local variable: java.lang.Class[]#5445
       local variable: org.hibernate.validator.internal.engine.ValidatorImpl#1
       local variable: java.lang.reflect.Method#91741
    at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:186)
    at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.proceed(CglibAopProxy.java:749)
       local variable: org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation#1
    at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:691)
       local variable: com.biaoguoworks.scm.logistics.controller.demeter.web.SortingTagController$$EnhancerBySpringCGLIB$$fbb0efe6#1
       local variable: java.lang.reflect.Method#91741
       local variable: com.biaoguoworks.scm.logistics.controller.demeter.web.SortingTagController#1
       local variable: org.springframework.aop.target.SingletonTargetSource#57
    at com.biaoguoworks.scm.logistics.controller.demeter.web.SortingTagController$$EnhancerBySpringCGLIB$$fbb0efe6.sortingImport(<generated>)
       local variable: com.biaoguoworks.scm.logistics.controller.demeter.web.SortingTagController$$EnhancerBySpringCGLIB$$fbb0efe6#1
       local variable: com.biaoguoworks.scm.logistics.vo.sorting.SkuSortingImportVO#1
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:498)
    at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:190)
       local variable: org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod#1
       local variable: java.lang.Object[]#89529
    at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:138)
    at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:105)
       local variable: org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod#1
       local variable: org.springframework.web.context.request.ServletWebRequest#1
       local variable: org.springframework.web.method.support.ModelAndViewContainer#1
    at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:878)
       local variable: org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter#1
       local variable: org.springframework.web.context.request.ServletWebRequest#1
       local variable: org.springframework.web.method.annotation.ModelFactory#1
       local variable: org.springframework.web.method.support.ModelAndViewContainer#1
       local variable: org.springframework.web.context.request.async.WebAsyncManager#1
    at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:792)
       local variable: org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter#1
       local variable: org.apache.catalina.connector.ResponseFacade#5
       local variable: org.springframework.web.method.HandlerMethod#378
    at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
    at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1040)
       local variable: org.springframework.web.servlet.DispatcherServlet#1
       local variable: org.apache.catalina.connector.ResponseFacade#5
       local variable: org.springframework.web.multipart.support.StandardMultipartHttpServletRequest#1
       local variable: org.springframework.web.servlet.HandlerExecutionChain#1
       local variable: org.springframework.web.context.request.async.WebAsyncManager#1
    at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:943)
       local variable: org.springframework.web.servlet.DispatcherServlet#1
       local variable: org.apache.catalina.connector.RequestFacade#5
    at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
       local variable: org.springframework.web.servlet.DispatcherServlet#1
       local variable: org.apache.catalina.connector.RequestFacade#5
       local variable: org.apache.catalina.connector.ResponseFacade#5
       local variable: org.springframework.context.i18n.SimpleLocaleContext#1
       local variable: org.springframework.web.context.request.ServletRequestAttributes#2
       local variable: org.springframework.web.context.request.ServletRequestAttributes#1
       local variable: org.springframework.web.context.request.async.WebAsyncManager#1
    at org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:909)
    at javax.servlet.http.HttpServlet.service(HttpServlet.java:652)
    at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
    at javax.servlet.http.HttpServlet.service(HttpServlet.java:733)
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:231)
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
    at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:53)
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
    at com.github.xiaoymin.knife4j.spring.filter.SecurityBasicAuthFilter.doFilter(SecurityBasicAuthFilter.java:90)
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
    at com.github.xiaoymin.knife4j.spring.filter.ProductionSecurityFilter.doFilter(ProductionSecurityFilter.java:53)
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
    at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
       local variable: org.springframework.boot.web.servlet.filter.OrderedRequestContextFilter#1
       local variable: org.apache.catalina.connector.RequestFacade#5
       local variable: org.springframework.web.context.request.ServletRequestAttributes#2
    at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119)
       local variable: org.apache.catalina.connector.RequestFacade#5
       local variable: java.lang.String#254631
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
    at org.springframework.web.filter.FormContentFilter.doFilterInternal(FormContentFilter.java:93)
    at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119)
       local variable: org.apache.catalina.connector.RequestFacade#5
       local variable: java.lang.String#254632
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
    at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
    at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119)
       local variable: org.apache.catalina.connector.RequestFacade#5
       local variable: java.lang.String#254633
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
    at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:202)
       local variable: org.apache.catalina.core.StandardWrapperValve#2
       local variable: org.apache.catalina.connector.Request#5
       local variable: org.apache.catalina.connector.Response#5
       local variable: org.apache.catalina.core.StandardWrapper#2
       local variable: org.springframework.web.servlet.DispatcherServlet#1
       local variable: org.springframework.boot.web.embedded.tomcat.TomcatEmbeddedContext#1
       local variable: org.apache.catalina.core.ApplicationFilterChain#5
       local variable: org.apache.catalina.core.StandardWrapper#2
    at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:96)
    at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:541)
    at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:139)
       local variable: org.apache.catalina.core.StandardHostValve#1
       local variable: org.apache.catalina.connector.Request#5
       local variable: org.apache.catalina.connector.Response#5
       local variable: org.springframework.boot.web.embedded.tomcat.TomcatEmbeddedContext#1
    at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:92)
       local variable: org.apache.catalina.valves.ErrorReportValve#1
       local variable: org.apache.catalina.connector.Request#5
       local variable: org.apache.catalina.connector.Response#5
    at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:74)
    at org.apache.catalina.valves.RemoteIpValve.invoke(RemoteIpValve.java:747)
       local variable: org.apache.catalina.valves.RemoteIpValve#1
       local variable: org.apache.catalina.connector.Request#5
       local variable: java.lang.String#254634
       local variable: java.lang.String#254634
       local variable: java.lang.String#18896
       local variable: java.lang.String#254635
       local variable: java.lang.String#254637
       local variable: java.lang.String#254636
    at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:343)
       local variable: org.apache.catalina.connector.CoyoteAdapter#1
       local variable: org.apache.coyote.Request#5
       local variable: org.apache.coyote.Response#5
       local variable: org.apache.catalina.connector.Request#5
       local variable: org.apache.catalina.connector.Response#5
    at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:373)
       local variable: org.apache.coyote.http11.Http11Processor#5
       local variable: org.apache.tomcat.util.net.NioEndpoint$NioSocketWrapper#7
       local variable: org.apache.coyote.RequestInfo#5
    at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:65)
       local variable: org.apache.coyote.http11.Http11Processor#5
       local variable: org.apache.tomcat.util.net.NioEndpoint$NioSocketWrapper#7
       local variable: org.apache.tomcat.util.net.SocketEvent#6
    at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:868)
       local variable: org.apache.coyote.AbstractProtocol$ConnectionHandler#1
       local variable: org.apache.tomcat.util.net.NioEndpoint$NioSocketWrapper#7
       local variable: org.apache.tomcat.util.net.SocketEvent#6
       local variable: org.apache.coyote.http11.Http11Processor#5
    at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1589)
       local variable: org.apache.tomcat.util.net.NioEndpoint$SocketProcessor#4
       local variable: org.apache.tomcat.util.net.NioChannel#17
       local variable: org.apache.tomcat.util.net.NioEndpoint$Poller#1
    at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
       local variable: org.apache.tomcat.util.net.NioEndpoint$NioSocketWrapper#7
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
       local variable: org.apache.tomcat.util.threads.ThreadPoolExecutor#1
       local variable: java.util.concurrent.ThreadPoolExecutor$Worker#63
       local variable: org.apache.tomcat.util.threads.TaskThread#14
       local variable: org.apache.tomcat.util.net.NioEndpoint$SocketProcessor#4
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
    at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
       local variable: org.apache.tomcat.util.threads.TaskThread$WrappingRunnable#14
    at java.lang.Thread.run(Thread.java:748)