分享一下spark streaming与flume集成的scala代码。

文章来自:http://www.cnblogs.com/hark0623/p/4172462.html  转发请注明

object LogicHandle { def main(args: Array[String]) {
//添加这个不会报执行错误 val path = new File(".").getCanonicalPath() System.getProperties().put("hadoop.home.dir", path); new File("./bin").mkdirs(); new File("./bin/winutils.exe").createNewFile(); //val sparkConf = new SparkConf().setAppName("SensorRealTime").setMaster("local[2]") val sparkConf = new SparkConf().setAppName("SensorRealTime") val ssc = new StreamingContext(sparkConf, Seconds(20)) val hostname = "localhost" val port = 2345 val storageLevel = StorageLevel.MEMORY_ONLY val flumeStream = FlumeUtils.createStream(ssc, hostname, port, storageLevel) val lhc = new LogicHandleClass(); //日志格式化模板 val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); val sdfHour = new SimpleDateFormat("HH"); val sdfMinute = new SimpleDateFormat("mm") //存储数据的hash对象 key/value存储 根据文档规则,使用各统计指标的key/value var redisMap = new HashMap[String, String]
  
     flumeStream.foreachRDD(rdd
=> { val events = rdd.collect() //println("event count:" + events.length) var i = 1 for (event <- events) { val sensorInfo = new String(event.event.getBody.array()) //单行记录 //单行记录格式化 val arrayFileds = sensorInfo.split(",") if (arrayFileds.length == 6) { val shopId = arrayFileds(0) //店内编号 val floorId = shopId.substring(0, 5) //楼层编号 val mac = arrayFileds(1) val ts = arrayFileds(2).toLong //时间戳 val time = sdf.format(ts * 1000) var hour = sdfHour.format(ts * 1000) var minute = sdfMinute.format(ts * 1000) var allMinute = hour.toInt * 60 + minute.toInt val x = arrayFileds(3) val y = arrayFileds(4) val level = arrayFileds(5) //后边就是我的业务代码了,省略了 } } //存储至redis中 lhc.SetAll(redisMap) }) ssc.start() ssc.awaitTermination() } }

 

你可能感兴趣的:(Stream)