main ERROR Unable to invoke factory method in class class org.apache.hadoop.hive.ql.log.HushableRand

报错信息

main ERROR Unable to invoke factory method in class class org.apache.hadoop.hive.ql.log.HushableRandomAccessFileAppender for element HushableMutableRandomAccess. java.lang.reflect.InvocationTargetException
 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
 at java.lang.reflect.Method.invoke(Method.java:498)
 at org.apache.logging.log4j.core.config.plugins.util.PluginBuilder.build(PluginBuilder.java:132)
 at org.apache.logging.log4j.core.config.AbstractConfiguration.createPluginObject(AbstractConfiguration.java:952)
 at org.apache.logging.log4j.core.config.AbstractConfiguration.createConfiguration(AbstractConfiguration.java:892)
 at org.apache.logging.log4j.core.appender.routing.RoutingAppender.createAppender(RoutingAppender.java:271)
 at org.apache.logging.log4j.core.appender.routing.RoutingAppender.getControl(RoutingAppender.java:255)
 at org.apache.logging.log4j.core.appender.routing.RoutingAppender.append(RoutingAppender.java:225)
 at org.apache.logging.log4j.core.config.AppenderControl.tryCallAppender(AppenderControl.java:156)
 at org.apache.logging.log4j.core.config.AppenderControl.callAppender0(AppenderControl.java:129)
 at org.apache.logging.log4j.core.config.AppenderControl.callAppenderPreventRecursion(AppenderControl.java:120)
 at org.apache.logging.log4j.core.config.AppenderControl.callAppender(AppenderControl.java:84)
 at org.apache.logging.log4j.core.config.LoggerConfig.callAppenders(LoggerConfig.java:448)
 at org.apache.logging.log4j.core.config.LoggerConfig.processLogEvent(LoggerConfig.java:433)
 at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:417)
 at org.apache.logging.log4j.core.config.LoggerConfig.log(LoggerConfig.java:403)
 at org.apache.logging.log4j.core.config.AwaitCompletionReliabilityStrategy.log(AwaitCompletionReliabilityStrategy.java:63)
 at org.apache.logging.log4j.core.Logger.logMessage(Logger.java:146)
 at org.apache.logging.log4j.spi.AbstractLogger.logMessageSafely(AbstractLogger.java:2091)
 at org.apache.logging.log4j.spi.AbstractLogger.logMessage(AbstractLogger.java:1993)
 at org.apache.logging.log4j.spi.AbstractLogger.logIfEnabled(AbstractLogger.java:1852)
 at org.apache.logging.slf4j.Log4jLogger.info(Log4jLogger.java:179)
 at org.apache.hadoop.hive.ql.exec.Operator.initialize(Operator.java:326)
 at org.apache.hadoop.hive.ql.exec.Operator.initialize(Operator.java:546)
 at org.apache.hadoop.hive.ql.exec.Operator.initializeChildren(Operator.java:498)
 at org.apache.hadoop.hive.ql.exec.Operator.initialize(Operator.java:368)
 at org.apache.hadoop.hive.ql.exec.mr.MapredLocalTask.initializeOperators(MapredLocalTask.java:514)
 at org.apache.hadoop.hive.ql.exec.mr.MapredLocalTask.startForward(MapredLocalTask.java:418)
 at org.apache.hadoop.hive.ql.exec.mr.MapredLocalTask.executeInProcess(MapredLocalTask.java:393)
 at org.apache.hadoop.hive.ql.exec.mr.ExecDriver.main(ExecDriver.java:766)
 at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
 at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
 at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
 at java.lang.reflect.Method.invoke(Method.java:498)
 at org.apache.hadoop.util.RunJar.run(RunJar.java:313)
 at org.apache.hadoop.util.RunJar.main(RunJar.java:227)
Caused by: java.lang.IllegalStateException: ManagerFactory [org.apache.logging.log4j.core.appender.RandomAccessFileManager$RandomAccessFileManagerFactory@23591a2c] unable to create manager for [/var/log/hive/operation_logs/d92d6df1-cfd8-4a57-9164-f2b9b9cd723c/hdfs_20181121150202_ad6757f7-1e6d-4d57-8938-f7c08d882953] with data [org.apache.logging.log4j.core.appender.RandomAccessFileManager$FactoryData@352a2660]
 at org.apache.logging.log4j.core.appender.AbstractManager.getManager(AbstractManager.java:114)
 at org.apache.logging.log4j.core.appender.OutputStreamManager.getManager(OutputStreamManager.java:114)
 at org.apache.logging.log4j.core.appender.RandomAccessFileManager.getFileManager(RandomAccessFileManager.java:74)
 at org.apache.hadoop.hive.ql.log.HushableRandomAccessFileAppender.createAppender(HushableRandomAccessFileAppender.java:178)
 ... 38 more

产生原因

mr将数据量小的表识别成了大表，数据量大的识别成小表，导致将数据量大的表加入到内存，导致程序异常.

解决办法

set hive.execution.engine=mr;
set hive.mapjoin.smalltable.filesize=55000000;
set hive.auto.convert.join = false;  #取消小表加载至内存中

转载自https://blog.csdn.net/weixin_45744450/article/details/111150144