storm-workcount例子

pom.xml文件内容:

xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

4.0.0

com.xu.eshop

storm-helloworld

0.0.1-SNAPSHOT

jar

storm-helloworld

storm-helloworld

UTF-8

UTF-8

1.8

junit

junit

4.10

test

org.apache.storm

storm-core

1.1.0

commons-collections

commons-collections

3.2.1

src/main/java

src/test/java

org.apache.maven.plugins

maven-shade-plugin

true

*:*

META-INF/*.SF

META-INF/*.sf

META-INF/*.DSA

META-INF/*.dsa

META-INF/*.RSA

META-INF/*.rsa

META-INF/*.EC

META-INF/*.ec

META-INF/MSFTSIG.SF

META-INF/MSFTSIG.RSA

package

shade

org.codehaus.mojo

exec-maven-plugin

1.2.1

exec

java

true

false

compile

com.xu.eshop.storm.WordCountTopology

WordCountTopology.java代码:

package com.xu.eshop.storm;

import org.apache.storm.Config;

import org.apache.storm.LocalCluster;

import org.apache.storm.StormSubmitter;

import org.apache.storm.spout.SpoutOutputCollector;

import org.apache.storm.task.OutputCollector;

import org.apache.storm.task.TopologyContext;

import org.apache.storm.topology.OutputFieldsDeclarer;

import org.apache.storm.topology.TopologyBuilder;

import org.apache.storm.topology.base.BaseRichBolt;

import org.apache.storm.topology.base.BaseRichSpout;

import org.apache.storm.tuple.Fields;

import org.apache.storm.tuple.Tuple;

import org.apache.storm.tuple.Values;

import org.apache.storm.utils.Utils;

import org.slf4j.Logger;

import org.slf4j.LoggerFactory;

import java.util.HashMap;

import java.util.Map;

import java.util.Random;

public class WordCountTopology {

    public static class RandomSentenceSpout extends BaseRichSpout {

        private static final Logger LOGGER = LoggerFactory.getLogger(RandomSentenceSpout.class);

        private Random random;

        private SpoutOutputCollector collector;

        public void open(Map map, TopologyContext topologyContext, SpoutOutputCollector collector) {

            this.collector = collector;

            this.random = new Random();

        }

        public void nextTuple() {

            Utils.sleep(100);

            String[] sentences = new String[]{"the cow jumped over the moon", "an apple a day keeps the doctor away",

                    "four score and seven years ago", "snow white and the seven dwarfs", "i am at two with nature"};

            final String sentence = sentences[random.nextInt(sentences.length)];

            LOGGER.info("发射句子:" + sentence);

            this.collector.emit(new Values(sentence));

        }

        public void declareOutputFields(OutputFieldsDeclarer declarer) {

            declarer.declare(new Fields("sentence"));

        }

    }

    public static class SplitSentence extends BaseRichBolt {

        private OutputCollector collector;

        public void prepare(Map conf, TopologyContext context, OutputCollector collector) {

            this.collector = collector;

        }

        public void execute(Tuple tuple) {

            String sentence = tuple.getStringByField("sentence");

            String[] words = sentence.split(" ");

            for(String word : words) {

                this.collector.emit(new Values(word));

            }

        }

        public void declareOutputFields(OutputFieldsDeclarer declarer) {

            declarer.declare(new Fields("word"));

        }

    }

    public static class WordCount extends BaseRichBolt {

        private static final Logger LOGGER = LoggerFactory.getLogger(WordCount.class);

        private OutputCollector collector;

        private Map wordCounts = new HashMap();

        public void prepare(Map conf, TopologyContext context, OutputCollector collector) {

            this.collector = collector;

        }

        public void execute(Tuple tuple) {

            String word = tuple.getStringByField("word");

            Long count = this.wordCounts.get(word);

            if(count == null) {

                count = 0L;

            }

            count ++;

            this.wordCounts.put(word, count);

            LOGGER.info("单词计数" + word + "出现的次数是" + count);

            this.collector.emit(new Values(word, count));

        }

        public void declareOutputFields(OutputFieldsDeclarer declarer) {

            declarer.declare(new Fields("word", "count"));

        }

    }

    public static void main(String[] args) {

        TopologyBuilder builder = new TopologyBuilder();

        builder.setSpout("RandomSentence", new RandomSentenceSpout(), 5);

        builder.setBolt("SplitSentence", new SplitSentence(), 5)

                .setNumTasks(10)

                .shuffleGrouping("RandomSentence");

        builder.setBolt("WordCount", new WordCount(), 10)

                .setNumTasks(20)

                .fieldsGrouping("SplitSentence", new Fields("word"));

        Config config = new Config();

        config.setDebug(false);

        if(args != null && args.length > 0) {

            config.setNumWorkers(3);

            try {

                StormSubmitter.submitTopology(args[0], config, builder.createTopology());

            } catch (Exception e) {

                e.printStackTrace();

            }

        } else {

            config.setMaxTaskParallelism(20);

            LocalCluster cluster = new LocalCluster();

            cluster.submitTopology("WordCountTopology", config, builder.createTopology());

            Utils.sleep(60000);

            cluster.shutdown();

        }

    }

}

你可能感兴趣的:(storm-workcount例子)