SPARK SQL 求助帖
1
INSERT OVERWRITE TABLE SJZL.DWD_PRD_IDRY_MONTH PARTITION
(PROV_ID, MONTH_ID)
SELECT LATN_ID,
PRVNCE_ID AS STD_PRVNCE_CD,
LATN_ID AS STD_LATN_CD,
PROD_INST_ID,
IDRY_APP_ID,
IDRY_APP_NM,
IDRY_APP_CATE_CD,
EFF_DATE,
EXP_DATE,
IDRY_STATUS,
RETRANS_NBR,
SEIAL_NBR,
BATCH_NBR,
from_unixtime(unix_timestamp(), ''yyyy/MM/dd HH:mm:ss''),
PRVNCE_ID,
MONTH_ID
FROM SJZL.DWS_PRD_IDRY_MONTH A1
WHERE MONTH_ID = ''201902''
AND PRVNCE_ID = ''811''
', 'application_1546863598863_14357797', '1', '811')
mysql connect success !
insert log successful
RuntimeError: Error: Traceback (most recent call last):
File “p_dwd_prd_idry_month_01.py”, line 111, in p_dwd_prd_idry_month
sqlContext.sql(sql_01)
File “/data1/yarn/local/usercache/sjzl/appcache/application_1546863598863_14357797/container_1546863598863_14357797_01_000001/pyspark.zip/pyspark/sql/context.py”, line 580, in sql
return DataFrame(self._ssql_ctx.sql(sqlQuery), self)
File “/data1/yarn/local/usercache/sjzl/appcache/application_1546863598863_14357797/container_1546863598863_14357797_01_000001/py4j-0.9-src.zip/py4j/java_gateway.py”, line 813, in call
answer, self.gateway_client, self.target_id, self.name)
File “/data1/yarn/local/usercache/sjzl/appcache/application_1546863598863_14357797/container_1546863598863_14357797_01_000001/pyspark.zip/pyspark/sql/utils.py”, line 45, in deco
return f(*a, **kw)
File “/data1/yarn/local/usercache/sjzl/appcache/application_1546863598863_14357797/container_1546863598863_14357797_01_000001/py4j-0.9-src.zip/py4j/protocol.py”, line 308, in get_return_value
format(target_id, “.”, name), value)
Py4JJavaError: An error occurred while calling o41.sql.
: org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 0.0 failed 4 times, most recent failure: Lost task 0.3 in stage 0.0 (TID 3, A5-402-NF5280M4-2017-562.BIGDATA.CHINATELECOM.CN): org.apache.hadoop.fs.viewfs.NotInMountpointException: getDefaultReplication on empty path is invalid
at org.apache.hadoop.fs.viewfs.ViewFileSystem.getDefaultReplication(ViewFileSystem.java:593)
at org.apache.hadoop.hive.ql.io.orc.WriterImpl.getStream(WriterImpl.java:2103)
at org.apache.hadoop.hive.ql.io.orc.WriterImpl.flushStripe(WriterImpl.java:2120)
at org.apache.hadoop.hive.ql.io.orc.WriterImpl.checkMemory(WriterImpl.java:352)
at org.apache.hadoop.hive.ql.io.orc.MemoryManager.notifyWriters(MemoryManager.java:168)
at org.apache.hadoop.hive.ql.io.orc.MemoryManager.addedRow(MemoryManager.java:157)
at org.apache.hadoop.hive.ql.io.orc.WriterImpl.addRow(WriterImpl.java:2413)
at org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat O r c R e c o r d W r i t e r . w r i t e ( O r c O u t p u t F o r m a t . j a v a : 86 ) a t o r g . a p a c h e . s p a r k . s q l . h i v e . e x e c u t i o n . I n s e r t I n t o H i v e T a b l e OrcRecordWriter.write(OrcOutputFormat.java:86) at org.apache.spark.sql.hive.execution.InsertIntoHiveTable OrcRecordWriter.write(OrcOutputFormat.java:86)atorg.apache.spark.sql.hive.execution.InsertIntoHiveTable a n o n f u n anonfun anonfunorg a p a c h e apache apachespark s q l sql sqlhive e x e c u t i o n execution executionInsertIntoHiveTableKaTeX parse error: Can't use function '$' in math mode at position 12: writeToFile$̲1$1.apply(Inser…anonfun o r g org orgapache s p a r k spark sparksql h i v e hive hiveexecution I n s e r t I n t o H i v e T a b l e InsertIntoHiveTable InsertIntoHiveTable$writeToFile$1 1. a p p l y ( I n s e r t I n t o H i v e T a b l e . s c a l a : 104 ) a t s c a l a . c o l l e c t i o n . I t e r a t o r 1.apply(InsertIntoHiveTable.scala:104) at scala.collection.Iterator 1.apply(InsertIntoHiveTable.scala:104)atscala.collection.Iteratorclass.foreach(Iterator.scala:727)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1157)
at org.apache.spark.sql.hive.execution.InsertIntoHiveTable.org a p a c h e apache apachespark s q l sql sqlhive e x e c u t i o n execution executionInsertIntoHiveTableKaTeX parse error: Can't use function '$' in math mode at position 12: writeToFile$̲1(InsertIntoHiv…anonfun$saveAsHiveFile 3. a p p l y ( I n s e r t I n t o H i v e T a b l e . s c a l a : 84 ) a t o r g . a p a c h e . s p a r k . s q l . h i v e . e x e c u t i o n . I n s e r t I n t o H i v e T a b l e 3.apply(InsertIntoHiveTable.scala:84) at org.apache.spark.sql.hive.execution.InsertIntoHiveTable 3.apply(InsertIntoHiveTable.scala:84)atorg.apache.spark.sql.hive.execution.InsertIntoHiveTable a n o n f u n anonfun anonfunsaveAsHiveFile 3. a p p l y ( I n s e r t I n t o H i v e T a b l e . s c a l a : 84 ) a t o r g . a p a c h e . s p a r k . s c h e d u l e r . R e s u l t T a s k . r u n T a s k ( R e s u l t T a s k . s c a l a : 66 ) a t o r g . a p a c h e . s p a r k . s c h e d u l e r . T a s k . r u n ( T a s k . s c a l a : 89 ) a t o r g . a p a c h e . s p a r k . e x e c u t o r . E x e c u t o r 3.apply(InsertIntoHiveTable.scala:84) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) at org.apache.spark.scheduler.Task.run(Task.scala:89) at org.apache.spark.executor.Executor 3.apply(InsertIntoHiveTable.scala:84)atorg.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66)atorg.apache.spark.scheduler.Task.run(Task.scala:89)atorg.apache.spark.executor.ExecutorTaskRunner.run(Executor.scala:227)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:745)
Driver stacktrace:
at org.apache.spark.scheduler.DAGScheduler.org a p a c h e apache apachespark s c h e d u l e r scheduler schedulerDAGScheduler f a i l J o b A n d I n d e p e n d e n t S t a g e s ( D A G S c h e d u l e r . s c a l a : 1431 ) a t o r g . a p a c h e . s p a r k . s c h e d u l e r . D A G S c h e d u l e r failJobAndIndependentStages(DAGScheduler.scala:1431) at org.apache.spark.scheduler.DAGScheduler failJobAndIndependentStages(DAGScheduler.scala:1431)atorg.apache.spark.scheduler.DAGScheduleranonfun$abortStage 1. a p p l y ( D A G S c h e d u l e r . s c a l a : 1419 ) a t o r g . a p a c h e . s p a r k . s c h e d u l e r . D A G S c h e d u l e r 1.apply(DAGScheduler.scala:1419) at org.apache.spark.scheduler.DAGScheduler 1.apply(DAGScheduler.scala:1419)atorg.apache.spark.scheduler.DAGScheduler a n o n f u n anonfun anonfunabortStage 1. a p p l y ( D A G S c h e d u l e r . s c a l a : 1418 ) a t s c a l a . c o l l e c t i o n . m u t a b l e . R e s i z a b l e A r r a y 1.apply(DAGScheduler.scala:1418) at scala.collection.mutable.ResizableArray 1.apply(DAGScheduler.scala:1418)atscala.collection.mutable.ResizableArrayclass.foreach(ResizableArray.scala:59)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47)
at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1418)
at org.apache.spark.scheduler.DAGSchedulerKaTeX parse error: Can't use function '$' in math mode at position 8: anonfun$̲handleTaskSetFa…anonfun$handleTaskSetFailed 1. a p p l y ( D A G S c h e d u l e r . s c a l a : 799 ) a t s c a l a . O p t i o n . f o r e a c h ( O p t i o n . s c a l a : 236 ) a t o r g . a p a c h e . s p a r k . s c h e d u l e r . D A G S c h e d u l e r . h a n d l e T a s k S e t F a i l e d ( D A G S c h e d u l e r . s c a l a : 799 ) a t o r g . a p a c h e . s p a r k . s c h e d u l e r . D A G S c h e d u l e r E v e n t P r o c e s s L o o p . d o O n R e c e i v e ( D A G S c h e d u l e r . s c a l a : 1640 ) a t o r g . a p a c h e . s p a r k . s c h e d u l e r . D A G S c h e d u l e r E v e n t P r o c e s s L o o p . o n R e c e i v e ( D A G S c h e d u l e r . s c a l a : 1599 ) a t o r g . a p a c h e . s p a r k . s c h e d u l e r . D A G S c h e d u l e r E v e n t P r o c e s s L o o p . o n R e c e i v e ( D A G S c h e d u l e r . s c a l a : 1588 ) a t o r g . a p a c h e . s p a r k . u t i l . E v e n t L o o p 1.apply(DAGScheduler.scala:799) at scala.Option.foreach(Option.scala:236) at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:799) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1640) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1599) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1588) at org.apache.spark.util.EventLoop 1.apply(DAGScheduler.scala:799)atscala.Option.foreach(Option.scala:236)atorg.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:799)atorg.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1640)atorg.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1599)atorg.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1588)atorg.apache.spark.util.EventLoop$anon 1. r u n ( E v e n t L o o p . s c a l a : 48 ) a t o r g . a p a c h e . s p a r k . s c h e d u l e r . D A G S c h e d u l e r . r u n J o b ( D A G S c h e d u l e r . s c a l a : 620 ) a t o r g . a p a c h e . s p a r k . S p a r k C o n t e x t . r u n J o b ( S p a r k C o n t e x t . s c a l a : 1832 ) a t o r g . a p a c h e . s p a r k . S p a r k C o n t e x t . r u n J o b ( S p a r k C o n t e x t . s c a l a : 1845 ) a t o r g . a p a c h e . s p a r k . S p a r k C o n t e x t . r u n J o b ( S p a r k C o n t e x t . s c a l a : 1922 ) a t o r g . a p a c h e . s p a r k . s q l . h i v e . e x e c u t i o n . I n s e r t I n t o H i v e T a b l e . s a v e A s H i v e F i l e ( I n s e r t I n t o H i v e T a b l e . s c a l a : 84 ) a t o r g . a p a c h e . s p a r k . s q l . h i v e . e x e c u t i o n . I n s e r t I n t o H i v e T a b l e . s i d e E f f e c t R e s u l t 1.run(EventLoop.scala:48) at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:620) at org.apache.spark.SparkContext.runJob(SparkContext.scala:1832) at org.apache.spark.SparkContext.runJob(SparkContext.scala:1845) at org.apache.spark.SparkContext.runJob(SparkContext.scala:1922) at org.apache.spark.sql.hive.execution.InsertIntoHiveTable.saveAsHiveFile(InsertIntoHiveTable.scala:84) at org.apache.spark.sql.hive.execution.InsertIntoHiveTable.sideEffectResult 1.run(EventLoop.scala:48)atorg.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:620)atorg.apache.spark.SparkContext.runJob(SparkContext.scala:1832)atorg.apache.spark.SparkContext.runJob(SparkContext.scala:1845)atorg.apache.spark.SparkContext.runJob(SparkContext.scala:1922)atorg.apache.spark.sql.hive.execution.InsertIntoHiveTable.saveAsHiveFile(InsertIntoHiveTable.scala:84)atorg.apache.spark.sql.hive.execution.InsertIntoHiveTable.sideEffectResultlzycompute(InsertIntoHiveTable.scala:201)
at org.apache.spark.sql.hive.execution.InsertIntoHiveTable.sideEffectResult(InsertIntoHiveTable.scala:127)
at org.apache.spark.sql.hive.execution.InsertIntoHiveTable.doExecute(InsertIntoHiveTable.scala:276)
at org.apache.spark.sql.execution.SparkPlanKaTeX parse error: Can't use function '$' in math mode at position 8: anonfun$̲execute$5.apply…anonfun$execute 5. a p p l y ( S p a r k P l a n . s c a l a : 130 ) a t o r g . a p a c h e . s p a r k . r d d . R D D O p e r a t i o n S c o p e 5.apply(SparkPlan.scala:130) at org.apache.spark.rdd.RDDOperationScope 5.apply(SparkPlan.scala:130)atorg.apache.spark.rdd.RDDOperationScope.withScope(RDDOperationScope.scala:150)
at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:130)
at org.apache.spark.sql.execution.QueryExecution.toRdd l z y c o m p u t e ( Q u e r y E x e c u t i o n . s c a l a : 55 ) a t o r g . a p a c h e . s p a r k . s q l . e x e c u t i o n . Q u e r y E x e c u t i o n . t o R d d ( Q u e r y E x e c u t i o n . s c a l a : 55 ) a t o r g . a p a c h e . s p a r k . s q l . D a t a F r a m e . < i n i t > ( D a t a F r a m e . s c a l a : 145 ) a t o r g . a p a c h e . s p a r k . s q l . D a t a F r a m e . < i n i t > ( D a t a F r a m e . s c a l a : 130 ) a t o r g . a p a c h e . s p a r k . s q l . D a t a F r a m e lzycompute(QueryExecution.scala:55) at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:55) at org.apache.spark.sql.DataFrame.<init>(DataFrame.scala:145) at org.apache.spark.sql.DataFrame.<init>(DataFrame.scala:130) at org.apache.spark.sql.DataFrame lzycompute(QueryExecution.scala:55)atorg.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:55)atorg.apache.spark.sql.DataFrame.<init>(DataFrame.scala:145)atorg.apache.spark.sql.DataFrame.<init>(DataFrame.scala:130)atorg.apache.spark.sql.DataFrame.apply(DataFrame.scala:52)
at org.apache.spark.sql.SQLContext.sql(SQLContext.scala:817)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:381)
at py4j.Gateway.invoke(Gateway.java:259)
at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
at py4j.commands.CallCommand.execute(CallCommand.java:79)
at py4j.GatewayConnection.run(GatewayConnection.java:209)
at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.hadoop.fs.viewfs.NotInMountpointException: getDefaultReplication on empty path is invalid
at org.apache.hadoop.fs.viewfs.ViewFileSystem.getDefaultReplication(ViewFileSystem.java:593)
at org.apache.hadoop.hive.ql.io.orc.WriterImpl.getStream(WriterImpl.java:2103)
at org.apache.hadoop.hive.ql.io.orc.WriterImpl.flushStripe(WriterImpl.java:2120)
at org.apache.hadoop.hive.ql.io.orc.WriterImpl.checkMemory(WriterImpl.java:352)
at org.apache.hadoop.hive.ql.io.orc.MemoryManager.notifyWriters(MemoryManager.java:168)
at org.apache.hadoop.hive.ql.io.orc.MemoryManager.addedRow(MemoryManager.java:157)
at org.apache.hadoop.hive.ql.io.orc.WriterImpl.addRow(WriterImpl.java:2413)
at org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat O r c R e c o r d W r i t e r . w r i t e ( O r c O u t p u t F o r m a t . j a v a : 86 ) a t o r g . a p a c h e . s p a r k . s q l . h i v e . e x e c u t i o n . I n s e r t I n t o H i v e T a b l e OrcRecordWriter.write(OrcOutputFormat.java:86) at org.apache.spark.sql.hive.execution.InsertIntoHiveTable OrcRecordWriter.write(OrcOutputFormat.java:86)atorg.apache.spark.sql.hive.execution.InsertIntoHiveTable a n o n f u n anonfun anonfunorg a p a c h e apache apachespark s q l sql sqlhive e x e c u t i o n execution executionInsertIntoHiveTableKaTeX parse error: Can't use function '$' in math mode at position 12: writeToFile$̲1$1.apply(Inser…anonfun o r g org orgapache s p a r k spark sparksql h i v e hive hiveexecution I n s e r t I n t o H i v e T a b l e InsertIntoHiveTable InsertIntoHiveTable$writeToFile$1 1. a p p l y ( I n s e r t I n t o H i v e T a b l e . s c a l a : 104 ) a t s c a l a . c o l l e c t i o n . I t e r a t o r 1.apply(InsertIntoHiveTable.scala:104) at scala.collection.Iterator 1.apply(InsertIntoHiveTable.scala:104)atscala.collection.Iteratorclass.foreach(Iterator.scala:727)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1157)
at org.apache.spark.sql.hive.execution.InsertIntoHiveTable.org a p a c h e apache apachespark s q l sql sqlhive e x e c u t i o n execution executionInsertIntoHiveTableKaTeX parse error: Can't use function '$' in math mode at position 12: writeToFile$̲1(InsertIntoHiv…anonfun$saveAsHiveFile 3. a p p l y ( I n s e r t I n t o H i v e T a b l e . s c a l a : 84 ) a t o r g . a p a c h e . s p a r k . s q l . h i v e . e x e c u t i o n . I n s e r t I n t o H i v e T a b l e 3.apply(InsertIntoHiveTable.scala:84) at org.apache.spark.sql.hive.execution.InsertIntoHiveTable 3.apply(InsertIntoHiveTable.scala:84)atorg.apache.spark.sql.hive.execution.InsertIntoHiveTable a n o n f u n anonfun anonfunsaveAsHiveFile 3. a p p l y ( I n s e r t I n t o H i v e T a b l e . s c a l a : 84 ) a t o r g . a p a c h e . s p a r k . s c h e d u l e r . R e s u l t T a s k . r u n T a s k ( R e s u l t T a s k . s c a l a : 66 ) a t o r g . a p a c h e . s p a r k . s c h e d u l e r . T a s k . r u n ( T a s k . s c a l a : 89 ) a t o r g . a p a c h e . s p a r k . e x e c u t o r . E x e c u t o r 3.apply(InsertIntoHiveTable.scala:84) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66) at org.apache.spark.scheduler.Task.run(Task.scala:89) at org.apache.spark.executor.Executor 3.apply(InsertIntoHiveTable.scala:84)atorg.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:66)atorg.apache.spark.scheduler.Task.run(Task.scala:89)atorg.apache.spark.executor.ExecutorTaskRunner.run(Executor.scala:227)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
… 1 more
2
INSERT OVERWRITE TABLE eb_name PARTITION (DAY_ID = '20190611')
SELECT itv_prod_type,
itv_prod_name,
remark FROM tb_name ;
Error: java.lang.RuntimeException: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing writable org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable@2085fb29
at org.apache.hadoop.hive.ql.exec.mr.ExecMapper.map(ExecMapper.java:179)
at org.apache.hadoop.mapred.MapRunner.run(MapRunner.java:54)
at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:459)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:343)
at org.apache.hadoop.mapred.YarnChild 2. r u n ( Y a r n C h i l d . j a v a : 164 ) a t j a v a . s e c u r i t y . A c c e s s C o n t r o l l e r . d o P r i v i l e g e d ( N a t i v e M e t h o d ) a t j a v a x . s e c u r i t y . a u t h . S u b j e c t . d o A s ( S u b j e c t . j a v a : 415 ) a t o r g . a p a c h e . h a d o o p . s e c u r i t y . U s e r G r o u p I n f o r m a t i o n . d o A s ( U s e r G r o u p I n f o r m a t i o n . j a v a : 1917 ) a t o r g . a p a c h e . h a d o o p . m a p r e d . Y a r n C h i l d . m a i n ( Y a r n C h i l d . j a v a : 158 ) C a u s e d b y : o r g . a p a c h e . h a d o o p . h i v e . q l . m e t a d a t a . H i v e E x c e p t i o n : H i v e R u n t i m e E r r o r w h i l e p r o c e s s i n g w r i t a b l e o r g . a p a c h e . h a d o o p . h i v e . s e r d e 2. c o l u m n a r . B y t e s R e f A r r a y W r i t a b l e @ 2085 f b 29 a t o r g . a p a c h e . h a d o o p . h i v e . q l . e x e c . M a p O p e r a t o r . p r o c e s s ( M a p O p e r a t o r . j a v a : 505 ) a t o r g . a p a c h e . h a d o o p . h i v e . q l . e x e c . m r . E x e c M a p p e r . m a p ( E x e c M a p p e r . j a v a : 170 ) . . . 8 m o r e C a u s e d b y : j a v a . l a n g . C l a s s C a s t E x c e p t i o n : o r g . a p a c h e . h a d o o p . h i v e . s e r d e 2. c o l u m n a r . B y t e s R e f A r r a y W r i t a b l e c a n n o t b e c a s t t o o r g . a p a c h e . h a d o o p . i o . B i n a r y C o m p a r a b l e a t o r g . a p a c h e . h a d o o p . h i v e . s e r d e 2. l a z y . L a z y S i m p l e S e r D e . d o D e s e r i a l i z e ( L a z y S i m p l e S e r D e . j a v a : 162 ) a t o r g . a p a c h e . h a d o o p . h i v e . s e r d e 2. A b s t r a c t E n c o d i n g A w a r e S e r D e . d e s e r i a l i z e ( A b s t r a c t E n c o d i n g A w a r e S e r D e . j a v a : 76 ) a t o r g . a p a c h e . h a d o o p . h i v e . q l . e x e c . M a p O p e r a t o r 2.run(YarnChild.java:164) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1917) at org.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158) Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing writable org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable@2085fb29 at org.apache.hadoop.hive.ql.exec.MapOperator.process(MapOperator.java:505) at org.apache.hadoop.hive.ql.exec.mr.ExecMapper.map(ExecMapper.java:170) ... 8 more Caused by: java.lang.ClassCastException: org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable cannot be cast to org.apache.hadoop.io.BinaryComparable at org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.doDeserialize(LazySimpleSerDe.java:162) at org.apache.hadoop.hive.serde2.AbstractEncodingAwareSerDe.deserialize(AbstractEncodingAwareSerDe.java:76) at org.apache.hadoop.hive.ql.exec.MapOperator 2.run(YarnChild.java:164)atjava.security.AccessController.doPrivileged(NativeMethod)atjavax.security.auth.Subject.doAs(Subject.java:415)atorg.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1917)atorg.apache.hadoop.mapred.YarnChild.main(YarnChild.java:158)Causedby:org.apache.hadoop.hive.ql.metadata.HiveException:HiveRuntimeErrorwhileprocessingwritableorg.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritable@2085fb29atorg.apache.hadoop.hive.ql.exec.MapOperator.process(MapOperator.java:505)atorg.apache.hadoop.hive.ql.exec.mr.ExecMapper.map(ExecMapper.java:170)...8moreCausedby:java.lang.ClassCastException:org.apache.hadoop.hive.serde2.columnar.BytesRefArrayWritablecannotbecasttoorg.apache.hadoop.io.BinaryComparableatorg.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.doDeserialize(LazySimpleSerDe.java:162)atorg.apache.hadoop.hive.serde2.AbstractEncodingAwareSerDe.deserialize(AbstractEncodingAwareSerDe.java:76)atorg.apache.hadoop.hive.ql.exec.MapOperatorMapOpCtx.readRow(MapOperator.java:136)
at org.apache.hadoop.hive.ql.exec.MapOperator$MapOpCtx.access$200(MapOperator.java:100)
at org.apache.hadoop.hive.ql.exec.MapOperator.process(MapOperator.java:496)
… 9 more