spark整合kafka打包运行错误处理

scala> val kafkaDS= KafkaUtils.createDirectStream[String,String,StringDecoder,StringDecoder](ssc,kafkaParams,topics).map(_._2)


java.lang.NoClassDefFoundError: com/yammer/metrics/Metrics
  at kafka.metrics.KafkaMetricsGroup$class.newTimer(KafkaMetricsGroup.scala:85)
  at kafka.consumer.FetchRequestAndResponseMetrics.newTimer(FetchRequestAndResponseStats.scala:26)
  at kafka.consumer.FetchRequestAndResponseMetrics.(FetchRequestAndResponseStats.scala:35)
  at kafka.consumer.FetchRequestAndResponseStats.(FetchRequestAndResponseStats.scala:46)
  at kafka.consumer.FetchRequestAndResponseStatsRegistry$$anonfun$2.apply(FetchRequestAndResponseStats.scala:59)
  at kafka.consumer.FetchRequestAndResponseStatsRegistry$$anonfun$2.apply(FetchRequestAndResponseStats.scala:59)
  at kafka.utils.Pool.getAndMaybePut(Pool.scala:61)
  at kafka.consumer.FetchRequestAndResponseStatsRegistry$.getFetchRequestAndResponseStats(FetchRequestAndResponseStats.scala:63)
  at kafka.consumer.SimpleConsumer.(SimpleConsumer.scala:39)
  at org.apache.spark.streaming.kafka.KafkaCluster.connect(KafkaCluster.scala:59)
  at org.apache.spark.streaming.kafka.KafkaCluster$$anonfun$org$apache$spark$streaming$kafka$KafkaCluster$$withBrokers$1.apply(KafkaCluster.scala:364)
  at org.apache.spark.streaming.kafka.KafkaCluster$$anonfun$org$apache$spark$streaming$kafka$KafkaCluster$$withBrokers$1.apply(KafkaCluster.scala:361)
  at scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33)
  at scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:35)
  at org.apache.spark.streaming.kafka.KafkaCluster.org$apache$spark$streaming$kafka$KafkaCluster$$withBrokers(KafkaCluster.scala:361)
  at org.apache.spark.streaming.kafka.KafkaCluster.getPartitionMetadata(KafkaCluster.scala:132)
  at org.apache.spark.streaming.kafka.KafkaCluster.getPartitions(KafkaCluster.scala:119)
  at org.apache.spark.streaming.kafka.KafkaUtils$.getFromOffsets(KafkaUtils.scala:211)
  at org.apache.spark.streaming.kafka.KafkaUtils$.createDirectStream(KafkaUtils.scala:484)

  ... 49 elided


解决:

加上 --jars metrics-core-2.2.0.jar

raini@biyuzhe:~/ideaProject/Spark-MLlib2.0$ spark-shell --jars /home/raini/app/libs/kafka082/spark-streaming-kafka-0-8_2.11-2.1.1.jar,/home/raini/app/libs/kafka082/kafka_2.11-0.8.2.2.jar

,/home/raini/app/libs/kafka082/kafka-clients-0.8.2.2.jar,/home/raini/app/libs/kafka082/metrics-core-2.2.0.jar



你可能感兴趣的:(Kafka)