Flink kafka 数据转成自己需要的实体类

1，主代码：

 

package application.storm;

import entry.ConsumerDeserializationSchema;
import entry.TopicInfoTest;
import org.apache.flink.api.java.utils.ParameterTool;
import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer010;
import org.slf4j.LoggerFactory;

import java.util.Properties;

 
public class FlinkTest {
    private static org.slf4j.Logger logger = LoggerFactory.getLogger(FlinkTest.class);


    public static void main(String[] args) throws Exception {


        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        args = new String[]{"--input-topic", "dianyou_wxgz3", "--bootstrap.servers", "node2.hadoop:9091,node3.hadoop:9091,node2.hadoop:9092,node3.hadoop:9092",
                "--zookeeper.connect", "node1.hadoop:2181,node2.hadoop:2181,node3.hadoop:2181", "--group.id", "cc1"};
        logger.error("************进入了程序**************");

        ParameterTool parameterTool = ParameterTool.fromArgs(args);
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);

        Properties pros = parameterTool.getProperties();


        //todo 指定输入数据为kafka topic
        DataStream kafkaDstream_test = env.addSource(new FlinkKafkaConsumer010(
                        "dianyou_wxgz3",
                        new ConsumerDeserializationSchema(TopicInfoTest.class),
//                new MySchema(),
                        pros).setStartFromGroupOffsets()

        ).setParallelism(4);
        kafkaDstream_test.print();
 



        env.execute("startExecute");
    }


}

2，实体类与自定义格式

package entry;

import org.apache.flink.api.common.serialization.DeserializationSchema;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.typeutils.TypeExtractor;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.CharBuffer;
import java.nio.charset.Charset;
import java.nio.charset.CharsetDecoder;

public class ConsumerDeserializationSchema implements DeserializationSchema {
    private Class clazz;

    public ConsumerDeserializationSchema(Class clazz) {
        this.clazz = clazz;
    }

    @Override
    public TopicInfoTest deserialize(byte[] message) throws IOException {
        ByteBuffer buffer = ByteBuffer.wrap(message).order(ByteOrder.LITTLE_ENDIAN);

        String mess = byteBuffertoString(buffer);
        //封装为POJO类

        System.out.println("mess = " + mess);
        return new TopicInfoTest(mess);
    }

    @Override
    public boolean isEndOfStream(TopicInfoTest t) {
        return false;
    }

    @Override
    public TypeInformation getProducedType() {
        return TypeExtractor.getForClass(clazz);
    }

    public static String byteBuffertoString(ByteBuffer buffer) {
        Charset charset = null;
        CharsetDecoder decoder = null;
        CharBuffer charBuffer = null;
        try {
            charset = Charset.forName("UTF-8");
            decoder = charset.newDecoder();
            // charBuffer = decoder.decode(buffer);//用这个的话，只能输出来一次结果，第二次显示为空
            charBuffer = decoder.decode(buffer.asReadOnlyBuffer());
            return charBuffer.toString();
        } catch (Exception ex) {
            ex.printStackTrace();
            return "";
        }
    }
}

 

package entry;


import java.io.Serializable;

public class TopicInfoTest implements Serializable {



    private String topicName;



    public String getTopicName() {
        return topicName;
    }

    public void setTopicName(String topicName) {
        this.topicName = topicName;
    }

    public TopicInfoTest(String topicName) {
        this.topicName = topicName;
    }

    @Override
    public String toString() {
        return "TopicInfoTest{" +
                "topicName='" + topicName + '\'' +
                '}';
    }
}

 

3，如果写成泛型：

 

package entry;

import com.alibaba.fastjson.JSON;
import org.apache.flink.api.common.serialization.DeserializationSchema;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.typeutils.TypeExtractor;

import java.io.IOException;

public class ConsumerDeserializationSchema implements DeserializationSchema {
    private Class clazz;

    public ConsumerDeserializationSchema(Class clazz) {
        this.clazz = clazz;
    }

    @Override
    public T deserialize(byte[] bytes) throws IOException {
        //确保 new String(bytes) 是json 格式，如果不是，请自行解析
        System.out.println("bytes = " + new String(bytes));

        System.out.println("json-parser"+JSON.parseObject(new String(bytes), clazz));


         return JSON.parseObject(new String(bytes), clazz);
    }

    @Override
    public boolean isEndOfStream(T t) {
        return false;
    }

    @Override
    public TypeInformation getProducedType() {
        return TypeExtractor.getForClass(clazz);
    }
}

 测试数据为：

 {"topicName":"1"}

打印结果：