import java.util.Map; import java.util.Random; import backtype.storm.spout.SpoutOutputCollector; import backtype.storm.task.TopologyContext; import backtype.storm.topology.OutputFieldsDeclarer; import backtype.storm.topology.base.BaseRichSpout; import backtype.storm.tuple.Fields; import backtype.storm.tuple.Values; /** * Spout起到和外界沟通的作用,他可以从一个数据库中按照某种规则取数据,也可以从分布式队列中取任务 */ public class SimpleSpout extends BaseRichSpout { // 用来发射数据的工具类 private SpoutOutputCollector collector; private static String[] info = new String[] { "storm", "hadoop", "flume" }; Random random = new Random(); public SimpleSpout() { System.out.println("-------SimpleSpout()-------"); } /** * 定义字段id,该id在简单模式下没有用处,但在按照字段分组的模式下有很大的用处。 * 该declarer变量有很大作用,还可以调用declarer.declareStream() * 来定义stramId,该id可以用来定义更加复杂的流拓扑结构 */ @Override public void declareOutputFields(OutputFieldsDeclarer declarer) { System.out.println("-------SimpleSpout declareOutputFields()-------"); declarer.declare(new Fields("source")); // collector.emit(new Values(msg));参数要对应 } /** * 初始化collector */ public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { System.out.println("-------SimpleSpout open()-------"); this.collector = collector; } @Override public void activate() { System.out.println("-------SimpleSpout activate()-------"); super.activate(); } /** * 在SpoutTracker类中被调用,每调用一次就可以向storm集群中发射一条数据(一个tuple元组),该方法会被不停的调用 */ @Override public void nextTuple() { System.out.println("-------SimpleSpout nextTuple()-------"); try { String msg = info[random.nextInt(info.length)]; // 调用发射方法 collector.emit(new Values(msg)); // 模拟等待1000ms Thread.sleep(1000); } catch (InterruptedException e) { e.printStackTrace(); } } }
import java.util.Map; import backtype.storm.task.TopologyContext; import backtype.storm.topology.BasicOutputCollector; import backtype.storm.topology.OutputFieldsDeclarer; import backtype.storm.topology.base.BaseBasicBolt; import backtype.storm.tuple.Tuple; /** * 接收喷发节点(Spout)发送的数据进行简单的处理后,发射出去。 * */ @SuppressWarnings("serial") public class SimpleBolt extends BaseBasicBolt { public SimpleBolt() { System.out.println("-------SimpleBolt()-------"); } public void declareOutputFields(OutputFieldsDeclarer declarer) { System.out.println("-------SimpleBolt declareOutputFields()-------"); // declarer.declare(new Fields("info")); } @Override public void prepare(Map stormConf, TopologyContext context) { System.out.println("-------SimpleBolt prepare()-------"); super.prepare(stormConf, context); } public void execute(Tuple input, BasicOutputCollector collector) { System.out.println("-------SimpleBolt execute()-------"); try { String msg = input.getString(0); if (msg != null) { System.out.println("-------" + msg); // System.out.println("****msg="+msg+"-----"+Thread.activeCount()); // collector.emit(new Values(msg + " msg is processed!")); } } catch (Exception e) { e.printStackTrace(); } } }
import backtype.storm.Config; import backtype.storm.LocalCluster; import backtype.storm.StormSubmitter; import backtype.storm.topology.TopologyBuilder; /** * 定义了一个简单的topology,包括一个数据喷发节点spout和一个数据处理节点bolt。 */ public class SimpleTopology { public static void main(String[] args) { try { // 实例化TopologyBuilder类。 TopologyBuilder topologyBuilder = new TopologyBuilder(); // 设置喷发节点并分配并发数,该并发数将会控制该对象在集群中的线程数。 topologyBuilder.setSpout("SimpleSpout", new SimpleSpout(), 1); // 设置数据处理节点并分配并发数。指定该节点接收喷发节点的策略为随机方式。 topologyBuilder.setBolt("SimpleBolt", new SimpleBolt(), 2) .shuffleGrouping("SimpleSpout"); Config config = new Config(); config.setDebug(false); if (args != null && args.length > 0) { config.setNumWorkers(1); StormSubmitter.submitTopology(args[0], config, topologyBuilder.createTopology()); } else { // 这里是本地模式下运行的启动代码。 config.setMaxTaskParallelism(1); LocalCluster cluster = new LocalCluster(); cluster.submitTopology("simple", config, topologyBuilder.createTopology()); } } catch (Exception e) { e.printStackTrace(); } } }
-------SimpleSpout()------- -------SimpleBolt()------- -------SimpleBolt declareOutputFields()------- -------SimpleSpout declareOutputFields()------- -------SimpleSpout open()------- -------SimpleSpout activate()------- -------SimpleSpout nextTuple()------- -------SimpleBolt prepare()------- -------SimpleBolt execute()------- -------flume -------SimpleSpout nextTuple()------- -------SimpleBolt execute()------- -------storm