storm

hdfs topology:


package com.ssc.arqe.hadoop.topology;

import org.apache.storm.hdfs.bolt.HdfsBolt;
import org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat;
import org.apache.storm.hdfs.bolt.format.FileNameFormat;
import org.apache.storm.hdfs.bolt.format.HistoricalJobDataFormat;
import org.apache.storm.hdfs.bolt.format.NDSJobRecordFormat;
import org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy;
import org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy;
import org.apache.storm.hdfs.bolt.sync.CountSyncPolicy;
import org.apache.storm.hdfs.bolt.sync.SyncPolicy;

import storm.kafka.BrokerHosts;
import storm.kafka.KafkaSpout;
import storm.kafka.SpoutConfig;
import storm.kafka.StringScheme;
import storm.kafka.ZkHosts;
import backtype.storm.Config;
import backtype.storm.StormSubmitter;
import backtype.storm.generated.AlreadyAliveException;
import backtype.storm.generated.InvalidTopologyException;
import backtype.storm.spout.SchemeAsMultiScheme;
import backtype.storm.topology.TopologyBuilder;

import com.ssc.arqe.hadoop.bolt.JobStatusBolt;

public class NDSStatusTopology {

public static void main(String[] args) throws AlreadyAliveException,
InvalidTopologyException {
// TODO Auto-generated method stub

String hadoop_env = args[0]; // hadoop environment, like

System.out.println("hadoop_env:" + hadoop_env);
String path = args[1];
System.out.println("path:" + path);
String topic = args[2];
System.out.println("kafka_topic:" + topic);
String zkHost = args[3];
System.out.println("zkHost_port:" + zkHost);


String kafka_id = args[4];
System.out.println("consumer groupID:" + kafka_id); // consumer group id
// sync the filesystem after every 1k tuples
SyncPolicy syncPolicy = new CountSyncPolicy(2);

// rotate files when they reach 5MB
FileRotationPolicy rotationPolicy = new TimedRotationPolicy(1.0f,
TimedRotationPolicy.TimeUnit.DAYS);

FileNameFormat fileNameFormat = new DefaultFileNameFormat().withPath(
path).withExtension(".txt");

// JobStatusHandler handler=new JobStatusHandler(hadoop_env, path);
BrokerHosts brokerHosts = new ZkHosts(zkHost);

SpoutConfig kafkaConfig = new SpoutConfig(brokerHosts, topic, "",
kafka_id);
kafkaConfig.scheme = new SchemeAsMultiScheme(new StringScheme());

HdfsBolt bolt = new HdfsBolt().withFsUrl(args[0])
.withRecordFormat(new HistoricalJobDataFormat())
.withFileNameFormat(fileNameFormat)
.withRotationPolicy(rotationPolicy).withSyncPolicy(syncPolicy);

TopologyBuilder builder = new TopologyBuilder();
builder.setSpout("GetNDSJobStatusMessage", new KafkaSpout(kafkaConfig),
2);
builder.setBolt("SendNDSJobStatusMessage", bolt, 2).shuffleGrouping(
"GetNDSJobStatusMessage");
Config conf = new Config();

conf.setDebug(true);

conf.setNumWorkers(2); // set workers to running topology
StormSubmitter.submitTopologyWithProgressBar("NDSJobStatusTopology",
conf, builder.createTopology());

}

}