【八】storm+kafka整合
官网参考0.8.X的kafka
maven的pom文件
4.0.0
com.sid.bigdata
storm
0.0.1
jar
storm
http://maven.apache.org
UTF-8
1.1.1
2.6.0
org.apache.storm
storm-core
${storm.version}
org.slf4j
log4j-over-slf4j
commons-io
commons-io
2.4
org.apache.hadoop
hadoop-client
${hadoop.version}
log4j
log4j
org.apache.storm
storm-redis
${storm.version}
org.apache.storm
storm-jdbc
${storm.version}
org.apache.storm
storm-hdfs
${storm.version}
org.apache.hadoop
hadoop-client
org.apache.hadoop
hadoop-auth
mysql
mysql-connector-java
5.1.31
org.apache.storm
storm-kafka
${storm.version}
org.apache.kafka
kafka_2.11
0.9.0.0
org.apache.kafka
kafka-clients
0.9.0.0
org.apache.curator
curator-client
2.12.0
项目
Topology代码
package integration.kafka;
import java.util.UUID;
import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
import org.apache.storm.kafka.BrokerHosts;
import org.apache.storm.kafka.KafkaSpout;
import org.apache.storm.kafka.SpoutConfig;
import org.apache.storm.kafka.StringScheme;
import org.apache.storm.kafka.ZkHosts;
import org.apache.storm.spout.SchemeAsMultiScheme;
import org.apache.storm.topology.TopologyBuilder;
/**
* @author liyijie
* @date 2018年7月9日下午8:21:23
* @email [email protected]
* @remark
* @version
*/
public class StormKafkaTopology {
public static void main(String[] args) {
TopologyBuilder tb = new TopologyBuilder();
//kafka使用的zk地址
BrokerHosts hosts = new ZkHosts("node1:2181");
//2参数，指定kafka的topic 3参数指定ZK的一个根目录，是kafkaspout读取数据的位置信息（offset）
SpoutConfig spoutConfig = new SpoutConfig(hosts, "storm_topic", "/" + "storm_topic", UUID.randomUUID().toString());
spoutConfig.scheme = new SchemeAsMultiScheme(new StringScheme());
KafkaSpout kafkaSpout = new KafkaSpout(spoutConfig);
String spoutId=KafkaSpout.class.getName();
tb.setSpout(spoutId, kafkaSpout);
tb.setBolt("LogProcessBolt", new LogProcessBolt()).shuffleGrouping(spoutId);
LocalCluster cluster = new LocalCluster();
cluster.submitTopology("StormKafkaTopology", new Config(), tb.createTopology());
}
}
bolt代码
package integration.kafka;
import java.lang.reflect.Field;
import java.util.Map;
import org.apache.storm.spout.MultiScheme;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichBolt;
import org.apache.storm.tuple.Tuple;
import org.apache.storm.tuple.TupleImpl;
/**
* @author liyijie
* @date 2018年7月9日下午10:28:44
* @email [email protected]
* @remark
* @version
*/
public class LogProcessBolt extends BaseRichBolt{
private OutputCollector collector;
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
this.collector = collector;
}
public void execute(Tuple input) {
try{
String binaryByField = input.getString(0);
System.out.println("rec value is "+binaryByField);
this.collector.ack(input);
}catch(Exception e){
this.collector.fail(input);
}
}
public void declareOutputFields(OutputFieldsDeclarer declarer) {
}
}
storm整合kafka运行报错
1.
java.lang.NoClassDefFoundError: org/apache/curator/shaded/com/google/common/cache/CacheBuilder
at org.apache.curator.framework.imps.NamespaceWatcherMap.(NamespaceWatcherMap.java:31) ~[curator-framework-2.12.0.jar:?]
at org.apache.curator.framework.imps.CuratorFrameworkImpl.(CuratorFrameworkImpl.java:81) ~[curator-framework-2.12.0.jar:?]
at org.apache.curator.framework.CuratorFrameworkFactory$Builder.build(CuratorFrameworkFactory.java:145) ~[curator-framework-2.12.0.jar:?]
at org.apache.curator.framework.CuratorFrameworkFactory.newClient(CuratorFrameworkFactory.java:100) ~[curator-framework-2.12.0.jar:?]
at org.apache.storm.kafka.ZkState.newCurator(ZkState.java:45) ~[storm-kafka-1.1.1.jar:1.1.1]
at org.apache.storm.kafka.ZkState.(ZkState.java:61) ~[storm-kafka-1.1.1.jar:1.1.1]
at org.apache.storm.kafka.KafkaSpout.open(KafkaSpout.java:76) ~[storm-kafka-1.1.1.jar:1.1.1]
at org.apache.storm.daemon.executor$fn__4962$fn__4977.invoke(executor.clj:602) ~[storm-core-1.1.1.jar:1.1.1]
at org.apache.storm.util$async_loop$fn__557.invoke(util.clj:482) [storm-core-1.1.1.jar:1.1.1]
at clojure.lang.AFn.run(AFn.java:22) [clojure-1.7.0.jar:?]
at java.lang.Thread.run(Thread.java:745) [?:1.8.0_91]
Caused by: java.lang.ClassNotFoundException: org.apache.curator.shaded.com.google.common.cache.CacheBuilder
at java.net.URLClassLoader.findClass(URLClassLoader.java:381) ~[?:1.8.0_91]
at java.lang.ClassLoader.loadClass(ClassLoader.java:424) ~[?:1.8.0_91]
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331) ~[?:1.8.0_91]
at java.lang.ClassLoader.loadClass(ClassLoader.java:357) ~[?:1.8.0_91]
... 11 more 解决办法
这是差jar包，在pom中引入依赖jar包
org.apache.curator
curator-client
2.12.0
2.
SLF4J: Detected both log4j-over-slf4j.jar AND slf4j-log4j12.jar on the class path, preempting StackOverflowError.
SLF4J: See also http://www.slf4j.org/codes.html#log4jDelegationLoop for more details.
82691 [Thread-20-org.apache.storm.kafka.KafkaSpout-executor[3 3]] ERROR o.a.s.util - Async loop died!
java.lang.NoClassDefFoundError: Could not initialize class org.apache.log4j.Log4jLoggerFactory
at org.apache.log4j.Logger.getLogger(Logger.java:39) ~[log4j-over-slf4j-1.6.6.jar:1.6.6]
at kafka.utils.Logging$class.logger(Logging.scala:24) ~[kafka_2.11-0.9.0.0.jar:?]
at kafka.consumer.SimpleConsumer.logger$lzycompute(SimpleConsumer.scala:35) ~[kafka_2.11-0.9.0.0.jar:?]
at kafka.consumer.SimpleConsumer.logger(SimpleConsumer.scala:35) ~[kafka_2.11-0.9.0.0.jar:?]
at kafka.utils.Logging$class.info(Logging.scala:67) ~[kafka_2.11-0.9.0.0.jar:?]
at kafka.consumer.SimpleConsumer.info(SimpleConsumer.scala:35) ~[kafka_2.11-0.9.0.0.jar:?]
at kafka.consumer.SimpleConsumer.liftedTree1$1(SimpleConsumer.scala:94) ~[kafka_2.11-0.9.0.0.jar:?]
at kafka.consumer.SimpleConsumer.kafka$consumer$SimpleConsumer$$sendRequest(SimpleConsumer.scala:83) ~[kafka_2.11-0.9.0.0.jar:?]
at kafka.consumer.SimpleConsumer.getOffsetsBefore(SimpleConsumer.scala:149) ~[kafka_2.11-0.9.0.0.jar:?]
at kafka.javaapi.consumer.SimpleConsumer.getOffsetsBefore(SimpleConsumer.scala:79) ~[kafka_2.11-0.9.0.0.jar:?]
at org.apache.storm.kafka.KafkaUtils.getOffset(KafkaUtils.java:81) ~[storm-kafka-1.1.1.jar:1.1.1]
at org.apache.storm.kafka.KafkaUtils.getOffset(KafkaUtils.java:71) ~[storm-kafka-1.1.1.jar:1.1.1]
at org.apache.storm.kafka.PartitionManager.(PartitionManager.java:135) ~[storm-kafka-1.1.1.jar:1.1.1]
at org.apache.storm.kafka.ZkCoordinator.refresh(ZkCoordinator.java:108) ~[storm-kafka-1.1.1.jar:1.1.1]
at org.apache.storm.kafka.ZkCoordinator.getMyManagedPartitions(ZkCoordinator.java:69) ~[storm-kafka-1.1.1.jar:1.1.1]
at org.apache.storm.kafka.KafkaSpout.nextTuple(KafkaSpout.java:130) ~[storm-kafka-1.1.1.jar:1.1.1]
at org.apache.storm.daemon.executor$fn__4962$fn__4977$fn__5008.invoke(executor.clj:646) ~[storm-core-1.1.1.jar:1.1.1]
at org.apache.storm.util$async_loop$fn__557.invoke(util.clj:484) [storm-core-1.1.1.jar:1.1.1]
at clojure.lang.AFn.run(AFn.java:22) [clojure-1.7.0.jar:?]
at java.lang.Thread.run(Thread.java:745) [?:1.8.0_91]
82692 [Thread-20-org.apache.storm.kafka.KafkaSpout-executor[3 3]] ERROR o.a.s.d.executor - 解决方法
修改pom，这是log4j和slf4j冲突了
3.
java.lang.NoSuchMethodError: org.apache.kafka.common.network.NetworkSend.(Ljava/lang/String;[Ljava/nio/ByteBuffer;)V
at kafka.network.RequestOrResponseSend.(RequestOrResponseSend.scala:41) ~[kafka_2.11-0.9.0.0.jar:?]
at kafka.network.RequestOrResponseSend.(RequestOrResponseSend.scala:44) ~[kafka_2.11-0.9.0.0.jar:?]
at kafka.network.BlockingChannel.send(BlockingChannel.scala:112) ~[kafka_2.11-0.9.0.0.jar:?]
at kafka.consumer.SimpleConsumer.liftedTree1$1(SimpleConsumer.scala:98) ~[kafka_2.11-0.9.0.0.jar:?]
at kafka.consumer.SimpleConsumer.kafka$consumer$SimpleConsumer$$sendRequest(SimpleConsumer.scala:83) ~[kafka_2.11-0.9.0.0.jar:?]
at kafka.consumer.SimpleConsumer.getOffsetsBefore(SimpleConsumer.scala:149) ~[kafka_2.11-0.9.0.0.jar:?]
at kafka.javaapi.consumer.SimpleConsumer.getOffsetsBefore(SimpleConsumer.scala:79) ~[kafka_2.11-0.9.0.0.jar:?]
at org.apache.storm.kafka.KafkaUtils.getOffset(KafkaUtils.java:81) ~[storm-kafka-1.1.1.jar:1.1.1]
at org.apache.storm.kafka.KafkaUtils.getOffset(KafkaUtils.java:71) ~[storm-kafka-1.1.1.jar:1.1.1]
at org.apache.storm.kafka.PartitionManager.(PartitionManager.java:135) ~[storm-kafka-1.1.1.jar:1.1.1]
at org.apache.storm.kafka.ZkCoordinator.refresh(ZkCoordinator.java:108) ~[storm-kafka-1.1.1.jar:1.1.1]
at org.apache.storm.kafka.ZkCoordinator.getMyManagedPartitions(ZkCoordinator.java:69) ~[storm-kafka-1.1.1.jar:1.1.1]
at org.apache.storm.kafka.KafkaSpout.nextTuple(KafkaSpout.java:130) ~[storm-kafka-1.1.1.jar:1.1.1]
at org.apache.storm.daemon.executor$fn__4962$fn__4977$fn__5008.invoke(executor.clj:646) ~[storm-core-1.1.1.jar:1.1.1]
at org.apache.storm.util$async_loop$fn__557.invoke(util.clj:484) [storm-core-1.1.1.jar:1.1.1]
at clojure.lang.AFn.run(AFn.java:22) [clojure-1.7.0.jar:?]
at java.lang.Thread.run(Thread.java:745) [?:1.8.0_91] 解决办法添加kafka-client依赖
org.apache.kafka
kafka-clients
0.9.0.0