package com.curiousby.baoyou.cn.storm;
import java.util.UUID;
import org.apache.storm.hdfs.bolt.HdfsBolt;
import org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat;
import org.apache.storm.hdfs.bolt.format.DelimitedRecordFormat;
import org.apache.storm.hdfs.bolt.format.FileNameFormat;
import org.apache.storm.hdfs.bolt.format.RecordFormat;
import org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy;
import org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy;
import org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy.TimeUnit;
import org.apache.storm.hdfs.bolt.sync.CountSyncPolicy;
import org.apache.storm.hdfs.bolt.sync.SyncPolicy;
import storm.kafka.BrokerHosts;
import storm.kafka.KafkaSpout;
import storm.kafka.SpoutConfig;
import storm.kafka.StringScheme;
import storm.kafka.ZkHosts;
import backtype.storm.Config;
import backtype.storm.LocalCluster;
import backtype.storm.StormSubmitter;
import backtype.storm.generated.AlreadyAliveException;
import backtype.storm.generated.InvalidTopologyException;
import backtype.storm.spout.SchemeAsMultiScheme;
import backtype.storm.topology.TopologyBuilder;
import backtype.storm.tuple.Fields;
/**
* @see com.curiousby.baoyou.cn.storm.TerminalInfosAnalysisTopology
* @Type TerminalInfosAnalysisTopology.java
* @Desc
* @author cmcc-B100036
* @date 2016年12月15日 下午4:54:50
* @version
*/
public class TerminalInfosAnalysisTopology {
private static String topicName = "baoy-topic";
private static String zkRoot = "/kafka" ;
public static void main(String[] args) {
BrokerHosts hosts = new ZkHosts(
"172.23.27.120:2181,172.23.27.115:2181,172.23.27.116:2181/kafka");
SpoutConfig spoutConfig = new SpoutConfig(hosts, topicName, zkRoot, UUID.randomUUID().toString());
spoutConfig.forceFromStart= false;
spoutConfig.scheme = new SchemeAsMultiScheme(new StringScheme());
//spoutConfig.socketTimeoutMs=60;
KafkaSpout kafkaSpout = new KafkaSpout(spoutConfig);
RecordFormat format = new DelimitedRecordFormat().withFieldDelimiter("\r\n");
SyncPolicy syncPolicy = new CountSyncPolicy(2);
FileRotationPolicy rotationPolicy = new TimedRotationPolicy(1.0f, TimeUnit.HOURS);
FileNameFormat fileNameFormat = new DefaultFileNameFormat().withPath("/user/hadoop/storm/").withPrefix("terminalInfo_").withExtension(".log");
HdfsBolt hdfsBolt = new HdfsBolt()
.withFsUrl("hdfs://172.23.27.120:9000/")
.withFileNameFormat(fileNameFormat).withRecordFormat(format)
.withRotationPolicy(rotationPolicy).withSyncPolicy(syncPolicy);
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout("kafkaSpout", kafkaSpout);
builder.setBolt("terminalInfosAnalysisIsValidBolt", new TerminalInfosAnalysisIsValidBolt(),1).shuffleGrouping("kafkaSpout");
builder.setBolt("terminalInfosAnalysisRedisBolt", new TerminalInfosAnalysisRedisBolt(),1).shuffleGrouping("terminalInfosAnalysisIsValidBolt");
builder.setBolt("terminalInfosAnalysisHdfsReportBolt", new TerminalInfosAnalysisHdfsReportBolt(),1).shuffleGrouping("terminalInfosAnalysisIsValidBolt");
builder.setBolt("terminalInfo", hdfsBolt,1).fieldsGrouping("terminalInfosAnalysisHdfsReportBolt",new Fields("hdfs-terminalinfo"));
// builder.setBolt("terminalInfosAnalysisHdfsBolt", new TerminalInfosAnalysisHdfsBolt(),1).shuffleGrouping("terminalInfosAnalysisIsValidBolt");
Config conf = new Config();
conf.setDebug(true);
if (args != null && args.length > 0) {
conf.setNumWorkers(1);
try {
StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createTopology());
} catch (AlreadyAliveException e) {
e.printStackTrace();
} catch (InvalidTopologyException e) {
e.printStackTrace();
}
} else {
conf.setMaxSpoutPending(1);
LocalCluster cluster = new LocalCluster();
cluster.submitTopology("terminalInfosAnalysisTopology", conf, builder.createTopology());
}
}
}
public class TerminalInfosAnalysisIsValidBolt extends BaseRichBolt {
private Logger logger =LoggerFactory.getLogger(TerminalInfosAnalysisIsValidBolt.class);
private OutputCollector collector;
@Override
public void execute(Tuple tuple) {
System.out.println(tuple.size());
logger.info("============================TerminalInfosAnalysisIsValidBolt execute===============================");
for (int i = 0; i < tuple.size(); i++) {
JSONObject formate = TerminalInfos.formate(tuple.getString(i));
TerminalInfos entity = new TerminalInfos();
entity.formate(formate);
if (entity != null && entity.isValid()) {
System.out.println(entity);
collector.emit(tuple, new Values(entity));
collector.ack(tuple);
}
}
}
@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
this.collector = collector;
}
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
declarer.declare(new Fields("after_isvalid"));
}
}
public class TerminalInfosAnalysisRedisBolt extends BaseRichBolt {
private Logger logger =LoggerFactory.getLogger(TerminalInfosAnalysisRedisBolt.class);
private OutputCollector collector;
JedisPool pool;
@Override
public void execute(Tuple tuple) {
Jedis jedis = pool.getResource();
logger.info("============================TerminalInfosAnalysisRedisBolt execute===============================");
for (int i = 0; i < tuple.size(); i++) {
TerminalInfos entity = (TerminalInfos) tuple.getValue(i);
TerminalInfoHeader tih = entity.getTerminalInfoHeader();
String key = tih.getAppId()+"-"+tih.getDeviceToken();
String value = jedis.get(key);
if (value == null || "".equals(value)) {
//
jedis.set( key, JSON.toJSONString(tih));
// insert es all infos
}else{
//update es lastupdatetime
}
}
}
@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
logger.info("============================redis prepare===============================");
this.collector = collector;
JedisPoolConfig config = new JedisPoolConfig();
config.setMaxActive(1000);
config.setMaxIdle(50);
config.setMaxWait(1000l);
config.setTestOnBorrow(false);
this.pool = new JedisPool(config, "172.23.27.120", 6379);
}
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
}
}
public class TerminalInfosAnalysisHdfsReportBolt extends BaseRichBolt {
private Logger logger =LoggerFactory.getLogger(TerminalInfosAnalysisHdfsReportBolt.class);
private OutputCollector collector;
@Override
public void execute(Tuple tuple) {
logger.info("============================TerminalInfosAnalysisHdfsReportBolt execute===============================");
for (int i = 0; i < tuple.size(); i++) {
TerminalInfos entity = (TerminalInfos) tuple.getValue(i);
TerminalInfoHeader tih = entity.getTerminalInfoHeader();
StringBuffer sb = new StringBuffer();
sb.append(tih.getAppId()).append(",");
sb.append(tih.getDeviceMac()).append(",");
sb.append(tih.getDeviceId()).append(",");
sb.append(tih.getDeviceToken()).append(",");
sb.append(tih.getDeviceImsi()).append(",");
sb.append(tih.getDeviceModel()).append(",");
sb.append(tih.getDeviceManufacture()).append(",");
sb.append(tih.getChannel()).append(",");
sb.append(tih.getAppKey()).append(",");
sb.append(tih.getUserId()).append(",");
sb.append(tih.getAppVersion()).append(",");
sb.append(tih.getVersionCode()).append(",");
sb.append(tih.getSdkType()).append(",");
sb.append(tih.getOs()).append(",");
sb.append(tih.getCountry()).append(",");
sb.append(tih.getLanguage()).append(",");
sb.append(tih.getTimezone()).append(",");
sb.append(tih.getResolution()).append(",");
sb.append(tih.getAccess()).append(",");
sb.append(tih.getAccessSubtype()).append(",");
sb.append(tih.getCarrier()).append(",");
sb.append(tih.getCpu());
collector.emit(tuple, new Values("hdfs-terminalinfo",sb.toString()));
collector.ack(tuple);
}
}
@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
this.collector = collector;
}
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
declarer.declare(new Fields("hdfs-terminalinfo", "record"));
}
}
4.0.0 com.curiousby.baoy.cn KafkaStormJavaDemo 0.0.1-SNAPSHOT jar SpringKafkaStormDemo http://maven.apache.org 4.2.5.RELEASE 1.7 1.2.17 clojars.org http://clojars.org/repo org.apache.storm storm-core 0.9.4 provided org.slf4j log4j-over-slf4j org.slf4j slf4j-log4j12 org.apache.storm storm-kafka 0.9.4 org.slf4j slf4j-log4j12 org.apache.storm storm-hdfs 0.9.4 org.slf4j slf4j-log4j12 org.apache.kafka kafka_2.10 0.8.2.1 org.apache.zookeeper zookeeper log4j log4j org.slf4j slf4j-log4j12 com.googlecode.json-simple json-simple 1.1.1 org.codehaus.jackson jackson-mapper-asl 1.9.13 com.alibaba fastjson 1.1.23 org.json json 20160810 com.fasterxml.jackson.core jackson-core 2.2.3 com.fasterxml.jackson.core jackson-annotations 2.2.3 com.fasterxml.jackson.core jackson-databind 2.2.3 log4j log4j ${log4j.version} redis.clients jedis 2.2.0 jdk.tools jdk.tools 1.7 system ${JAVA_HOME}/lib/tools.jar SpringKafkaStormDemo org.apache.maven.plugins maven-compiler-plugin 3.3 org.codehaus.plexus plexus-compiler-javac 2.5 1.7 1.7 UTF-8 ${java.home}/lib/rt.jar:${java.home}/lib/jce.jar org.apache.maven.plugins maven-shade-plugin 1.4 false ${project.artifactId}_TerminalInfosAnalysisTopology_main_start true package shade com.curiousby.baoyou.cn.storm.TerminalInfosAnalysisTopology maven-assembly-plugin false ${project.artifactId}_main_start jar-with-dependencies com.curiousby.baoyou.cn.storm.TerminalInfosAnalysisTopology
捐助开发者
在兴趣的驱动下,写一个免费的东西，有欣喜，也还有汗水，希望你喜欢我的作品，同时也能支持一下。 当然，有钱捧个钱场（右上角的爱心标志，支持支付宝和PayPal捐助），没钱捧个人场，谢谢各位。
谢谢您的赞助，我会做的更好！
