hadoop运行任务出错，Hive Runtime Error while processing row

Error: java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing row {“loan_money_time”:“2021-01-10 14:54:02”,“loan_date”:“2021-01-10”,“loan_month”:“2021-01”,“product_type”:“–”,“channel_name”:“—”,“loan_way”:“–”,“credit_rlevel”:null,“first_loan”:“首借”,“user_group”:null,“client_id”:111111111,“province”:null,“diqu”:null,“id”:111222211,“loan_phase”:12,“due_state”:null,“expected_repay_date”:null,“actual_repay_date”:null,“onloan_state”:null,“rate”:“0.3528”,“rate_year”:“35.28%”,“total_amount”:“23000.0”,“onloan_amount”:null,“ondelay_day”:null,“ondelay_phase”:null,“maxdelay_day”:null,“maxexpre_day”:null,“onloan_interest”:null,“repaid_interest”:null,“repaid_penalty”:null,“close_date”:null,“acct_limit”:null,“ondelay_day2”:null,“ondelay_phase2”:null} at org.apache.hadoop.hive.ql.exec.mr.ExecMapper.map(ExecMapper.java:179) at org.apache.hadoop.mapred.MapRunner.run(MapRunner.java:54) at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:459) at org.apache.hadoop.mapred.MapTask.run(MapTask.java:343) at org.apache.hadoop.mapred.YarnChildKaTeX parse error: Expected '}', got 'EOF' at end of input: …xec.MapOperatorMapOpCtx.forward(MapOperator.java:157) at org.apache.hadoop.hive.ql.exec.MapOperator.process(MapOperator.java:497) … 9 more Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: java.lang.ArrayIndexOutOfBoundsException at org.apache.hadoop.hive.ql.exec.ReduceSinkOperator.processOp(ReduceSinkOperator.java:405) at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:815) at org.apache.hadoop.hive.ql.exec.GroupByOperator.forward(GroupByOperator.java:1007) at org.apache.hadoop.hive.ql.exec.GroupByOperator.processAggr(GroupByOperator.java:818) at org.apache.hadoop.hive.ql.exec.GroupByOperator.processKey(GroupByOperator.java:692) at org.apache.hadoop.hive.ql.exec.GroupByOperator.processOp(GroupByOperator.java:755) at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:815) at org.apache.hadoop.hive.ql.exec.CommonJoinOperator.internalForward(CommonJoinOperator.java:638) at org.apache.hadoop.hive.ql.exec.CommonJoinOperator.genAllOneUniqueJoinObject(CommonJoinOperator.java:670) at org.apache.hadoop.hive.ql.exec.CommonJoinOperator.checkAndGenObject(CommonJoinOperator.java:748) at org.apache.hadoop.hive.ql.exec.MapJoinOperator.processOp(MapJoinOperator.java:306) … 17 more Caused by: java.lang.ArrayIndexOutOfBoundsException at org.apache.hadoop.mapred.MapTask$MapOutputBuffer$Buffer.write(MapTask.java:1460) at org.apache.hadoop.mapred.MapTask$MapOutputBuffer$Buffer.write(MapTask.java:1356) at java.io.DataOutputStream.writeInt(DataOutputStream.java:197) at org.apache.hadoop.io.BytesWritable.write(BytesWritable.java:188) at org.apache.hadoop.io.serializer.WritableSerialization$WritableSerializer.serialize(WritableSerialization.java:98)atorg.apache.hadoop.io.serializer.WritableSerialization$WritableSerializer.serialize(WritableSerialization.java:82) at org.apache.hadoop.mapred.MapTask$MapOutputBuffer.collect(MapTask.java:1156)atorg.apache.hadoop.mapred.MapTask$OldOutputCollector.collect(MapTask.java:616) at org.apache.hadoop.hive.ql.exec.ReduceSinkOperator.collect(ReduceSinkOperator.java:542) at org.apache.hadoop.hive.ql.exec.ReduceSinkOperator.processOp(ReduceSinkOperator.java:400) … 27 more
数据已经脱敏。
很恼火，任务运行1个多小时报错，很浪费时间，我们设置一些配置参数，再试试。
mapset="set mapred.max.split.size=256000000;
set mapred.min.split.size.per.node=128000000;
set mapred.min.split.size.per.rack=128000000;
set hive.exec.reducers.bytes.per.reducer=1073741824; – 每个reduce处理的数据量,默认1GB
set hive.input.format=org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
set hive.merge.mapfiles=true;
set hive.merge.mapredfiles=true;
set hive.map.aggr=true;
set hive.merge.size.per.task=256000000;
set hive.merge.smallfiles.avgsize=128000000;
set hive.exec.dynamic.partition.mode=nonstrict;
set hive.exec.dynamic.partition=true;
set hive.exec.max.dynamic.partitions.pernode=600000;
set hive.new.job.grouping.set.cardinality=6000;
set hive.groupby.skewindata=False;
set mapreduce.map.memory.mb=10240;
set mapreduce.reduce.memory.mb=10240;
set hive.auto.convert.join=true;
set mapred.reduce.tasks=150;
set mapreduce.job.jvm.numtasks=15;
set hive.vectorized.execution.enabled=false;