Flume自定义Source

讲解一下如何自定义一个Flume的Source，很简单，下面是一个自定义Source，从Kafka中读取数据的Demo.

package death.flume;

import com.google.common.base.Preconditions;
import org.apache.flume.Context;
import org.apache.flume.EventDeliveryException;
import org.apache.flume.PollableSource;
import org.apache.flume.conf.Configurable;
import org.apache.flume.event.EventBuilder;
import org.apache.flume.source.AbstractSource;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;

import java.nio.charset.Charset;
import java.util.Collections;
import java.util.Properties;

// 继承AbstractSource抽象类，并实现Configurable接口
public class FlumeSourceDemo extends AbstractSource implements Configurable, PollableSource {

    private String KAFKA_SERVER;

    private String KAFKA_TOPIC;

    private String GROUP_ID;

    private String KEY_DES;

    private String VALUE_DES;

    private Properties props;

	// Source的数据处理逻辑
    @Override
    public Status process() throws EventDeliveryException {
        try {
                KafkaConsumer<String, String> consumer = new KafkaConsumer<String, String>(props);

                consumer.subscribe(Collections.singleton(KAFKA_TOPIC));

                ConsumerRecords<String, String> records = consumer.poll(1000);

                for (ConsumerRecord<String, String> record : records) {
                    System.out.println(record.value());
                    this.getChannelProcessor().processEvent(EventBuilder.withBody(record.value(), Charset.forName("UTF-8")));
                }
            return Status.READY;

        }catch (Exception e){
            e.printStackTrace();
            return Status.BACKOFF;
        }
    }

	//该方法用于Source的初始化
    @Override
    public void configure(Context context) {

        KAFKA_SERVER = context.getString("kafka_server");
        Preconditions.checkNotNull(KAFKA_SERVER, "kafka_server must be set");

        KAFKA_TOPIC = context.getString("kafka_topic");
        Preconditions.checkNotNull(KAFKA_TOPIC, "kafka_topic must be set");

        GROUP_ID = context.getString("group_id");
        Preconditions.checkNotNull(GROUP_ID, "group_id must be set");

        KEY_DES = context.getString("key_des");
        Preconditions.checkNotNull(KEY_DES, "key_des must be set");

        VALUE_DES = context.getString("value_des");
        Preconditions.checkNotNull(VALUE_DES, "value_des must be set");

    }

	// 该方法在Source启动时调用
    @Override
    public synchronized void start() {
        super.start();

        props = new Properties();
        props.setProperty("bootstrap.servers", KAFKA_SERVER);
        props.setProperty("group.id", GROUP_ID);
        props.setProperty("key.deserializer", KEY_DES);
        props.setProperty("value.deserializer", VALUE_DES);
    }

	// 该方法在Source停止时调用
    @Override
    public synchronized void stop() {
        super.stop();
    }
}