flume实时接收kafka消息并保存至HDFS

#-------agent------


# 定义sources
agent.sources = kafkaSource


# 定义channels
agent.channels = memoryChannel


# 定义sinks
agent.sinks = hdfsSink
agent.sources.kafkaSource.channels = memoryChannel
agent.sinks.hdfsSink.channel = memoryChannel


#-------- kafkaSource-----------------
agent.sources.kafkaSource.type = org.apache.flume.source.kafka.KafkaSource
agent.sources.kafkaSource.zookeeperConnect = h2:2181,h3:2181,h4:2181,h8:2181,h9:2181,h10:2181
agent.sources.kafkaSource.topic = T20161031
#agent.sources.kafkaSource.groupId = Group_20161031
agent.sources.kafkaSource.kafka.consumer.timeout.ms = 1000


#------- memoryChannel-------------------------
agent.channels.memoryChannel.type = memory
agent.channels.memoryChannel.capacity=10000
agent.channels.memoryChannel.transactionCapacity=1000


#---------hdfsSink ------------------
agent.sinks.hdfsSink.type = hdfs
agent.sinks.hdfsSink.hdfs.path = hdfs://h4:9000/user/test/kafka2HdfsByFlume
agent.sinks.hdfsSink.hdfs.writeFormat = Text
agent.sinks.hdfsSink.hdfs.fileType = DataStream