Flink - 之Kafka Avro序列化格式转Json序列化格式（⭐⭐⭐）
Flink - 之Kafka Avro序列化格式转Json序列化格式（⭐⭐⭐）
参考网址：apache avro官网
1 背景
TODO 后续补充，晚间12点了不想搞了~
2 mvn依赖
<dependency>
<groupId>org.apache.flinkgroupId>
<artifactId>flink-avroartifactId>
<version>1.12.2version>
dependency>
<dependency>
<groupId>org.apache.avrogroupId>
<artifactId>avroartifactId>
<version>1.10.2version>
dependency>
build>
plugins>
<plugin>
<groupId>org.apache.avrogroupId>
<artifactId>avro-maven-pluginartifactId>
<version>1.10.2version>
<executions>
<execution>
<phase>generate-sourcesphase>
<goals>
<goal>schemagoal>
goals>
<configuration>
<sourceDirectory>${project.basedir}/src/main/avro/sourceDirectory>
<outputDirectory>${project.basedir}/src/main/java/outputDirectory>
configuration>
execution>
executions>
plugin>
<plugin>
<groupId>org.apache.maven.pluginsgroupId>
<artifactId>maven-compiler-pluginartifactId>
<configuration>
<source>1.8source>
<target>1.8target>
configuration>
plugin>
plugins>
build>
后续会通过Idea自动编译Schema生成Java Bean
3 自动编译Avro的schema生成对应的JavaBean
3.1 定义Avro Schema
- 注意，文件名称必须为
.avsc结尾
{"namespace": "com.shufang.flinkapp.avro.bean", # 类似于包名
"type": "record", # schema为消息类型，还有其它的基础类型、复杂类型如map、array
"name": "User", # 类名
"fields": [ # 创建的类的属性s
{"name": "name", "type": "string"},
{"name": "favorite_number", "type": ["int", "null"]},
{"name": "favorite_color", "type": ["string", "null"]}
]
}
3.2 通过maven手动编译schema，生成bean
// 1、最终结合maven插件配置 && schema中的namespace生成java bean User
src/main/java/com/shufang/flinkapp/avro/bean/User.java
// 2、注意：这个avsc的路径必须与插件中指定的位置一样
/src/main/avro/user.avsc
/**
* Autogenerated by Avro
* DO NOT EDIT DIRECTLY
*/
package com.shufang.flinkapp.avro.bean;
import org.apache.avro.generic.GenericArray;
import org.apache.avro.specific.SpecificData;
import org.apache.avro.util.Utf8;
import org.apache.avro.message.BinaryMessageEncoder;
import org.apache.avro.message.BinaryMessageDecoder;
import org.apache.avro.message.SchemaStore;
@org.apache.avro.specific.AvroGenerated
public class User extends org.apache.avro.specific.SpecificRecordBase implements org.apache.avro.specific.SpecificRecord {
private static final long serialVersionUID = -2112192159657953386L;
public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"com.shufang.flinkapp.avro.bean\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}");
public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; }
private static SpecificData MODEL$ = new SpecificData();
private static final BinaryMessageEncoder<User> ENCODER =
new BinaryMessageEncoder<User>(MODEL$, SCHEMA$);
private static final BinaryMessageDecoder<User> DECODER =
new BinaryMessageDecoder<User>(MODEL$, SCHEMA$);
/**
* Return the BinaryMessageEncoder instance used by this class.
* @return the message encoder used by this class
*/
public static BinaryMessageEncoder<User> getEncoder() {
return ENCODER;
}
/**
* Return the BinaryMessageDecoder instance used by this class.
* @return the message decoder used by this class
*/
public static BinaryMessageDecoder<User> getDecoder() {
return DECODER;
}
/**
* Create a new BinaryMessageDecoder instance for this class that uses the specified {@link SchemaStore}.
* @param resolver a {@link SchemaStore} used to find schemas by fingerprint
* @return a BinaryMessageDecoder instance for this class backed by the given SchemaStore
*/
public static BinaryMessageDecoder<User> createDecoder(SchemaStore resolver) {
return new BinaryMessageDecoder<User>(MODEL$, SCHEMA$, resolver);
}
/**
* Serializes this User to a ByteBuffer.
* @return a buffer holding the serialized data for this instance
* @throws java.io.IOException if this instance could not be serialized
*/
public java.nio.ByteBuffer toByteBuffer() throws java.io.IOException {
return ENCODER.encode(this);
}
/**
* Deserializes a User from a ByteBuffer.
* @param b a byte buffer holding serialized data for an instance of this class
* @return a User instance decoded from the given buffer
* @throws java.io.IOException if the given bytes could not be deserialized into an instance of this class
*/
public static User fromByteBuffer(
java.nio.ByteBuffer b) throws java.io.IOException {
return DECODER.decode(b);
}
private java.lang.CharSequence name;
private java.lang.Integer favorite_number;
private java.lang.CharSequence favorite_color;
/**
* Default constructor. Note that this does not initialize fields
* to their default values from the schema. If that is desired then
* one should use newBuilder().
*/
public User() {}
/**
* All-args constructor.
* @param name The new value for name
* @param favorite_number The new value for favorite_number
* @param favorite_color The new value for favorite_color
*/
public User(java.lang.CharSequence name, java.lang.Integer favorite_number, java.lang.CharSequence favorite_color) {
this.name = name;
this.favorite_number = favorite_number;
this.favorite_color = favorite_color;
}
public org.apache.avro.specific.SpecificData getSpecificData() { return MODEL$; }
public org.apache.avro.Schema getSchema() { return SCHEMA$; }
// Used by DatumWriter. Applications should not call.
public java.lang.Object get(int field$) {
switch (field$) {
case 0: return name;
case 1: return favorite_number;
case 2: return favorite_color;
default: throw new IndexOutOfBoundsException("Invalid index: " + field$);
}
}
// Used by DatumReader. Applications should not call.
@SuppressWarnings(value="unchecked")
public void put(int field$, java.lang.Object value$) {
switch (field$) {
case 0: name = (java.lang.CharSequence)value$; break;
case 1: favorite_number = (java.lang.Integer)value$; break;
case 2: favorite_color = (java.lang.CharSequence)value$; break;
default: throw new IndexOutOfBoundsException("Invalid index: " + field$);
}
}
/**
* Gets the value of the 'name' field.
* @return The value of the 'name' field.
*/
public java.lang.CharSequence getName() {
return name;
}
/**
* Sets the value of the 'name' field.
* @param value the value to set.
*/
public void setName(java.lang.CharSequence value) {
this.name = value;
}
/**
* Gets the value of the 'favorite_number' field.
* @return The value of the 'favorite_number' field.
*/
public java.lang.Integer getFavoriteNumber() {
return favorite_number;
}
/**
* Sets the value of the 'favorite_number' field.
* @param value the value to set.
*/
public void setFavoriteNumber(java.lang.Integer value) {
this.favorite_number = value;
}
/**
* Gets the value of the 'favorite_color' field.
* @return The value of the 'favorite_color' field.
*/
public java.lang.CharSequence getFavoriteColor() {
return favorite_color;
}
/**
* Sets the value of the 'favorite_color' field.
* @param value the value to set.
*/
public void setFavoriteColor(java.lang.CharSequence value) {
this.favorite_color = value;
}
/**
* Creates a new User RecordBuilder.
* @return A new User RecordBuilder
*/
public static com.shufang.flinkapp.avro.bean.User.Builder newBuilder() {
return new com.shufang.flinkapp.avro.bean.User.Builder();
}
/**
* Creates a new User RecordBuilder by copying an existing Builder.
* @param other The existing builder to copy.
* @return A new User RecordBuilder
*/
public static com.shufang.flinkapp.avro.bean.User.Builder newBuilder(com.shufang.flinkapp.avro.bean.User.Builder other) {
if (other == null) {
return new com.shufang.flinkapp.avro.bean.User.Builder();
} else {
return new com.shufang.flinkapp.avro.bean.User.Builder(other);
}
}
/**
* Creates a new User RecordBuilder by copying an existing User instance.
* @param other The existing instance to copy.
* @return A new User RecordBuilder
*/
public static com.shufang.flinkapp.avro.bean.User.Builder newBuilder(com.shufang.flinkapp.avro.bean.User other) {
if (other == null) {
return new com.shufang.flinkapp.avro.bean.User.Builder();
} else {
return new com.shufang.flinkapp.avro.bean.User.Builder(other);
}
}
/**
* RecordBuilder for User instances.
*/
@org.apache.avro.specific.AvroGenerated
public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase<User>
implements org.apache.avro.data.RecordBuilder<User> {
private java.lang.CharSequence name;
private java.lang.Integer favorite_number;
private java.lang.CharSequence favorite_color;
/** Creates a new Builder */
private Builder() {
super(SCHEMA$);
}
/**
* Creates a Builder by copying an existing Builder.
* @param other The existing Builder to copy.
*/
private Builder(com.shufang.flinkapp.avro.bean.User.Builder other) {
super(other);
if (isValidValue(fields()[0], other.name)) {
this.name = data().deepCopy(fields()[0].schema(), other.name);
fieldSetFlags()[0] = other.fieldSetFlags()[0];
}
if (isValidValue(fields()[1], other.favorite_number)) {
this.favorite_number = data().deepCopy(fields()[1].schema(), other.favorite_number);
fieldSetFlags()[1] = other.fieldSetFlags()[1];
}
if (isValidValue(fields()[2], other.favorite_color)) {
this.favorite_color = data().deepCopy(fields()[2].schema(), other.favorite_color);
fieldSetFlags()[2] = other.fieldSetFlags()[2];
}
}
/**
* Creates a Builder by copying an existing User instance
* @param other The existing instance to copy.
*/
private Builder(com.shufang.flinkapp.avro.bean.User other) {
super(SCHEMA$);
if (isValidValue(fields()[0], other.name)) {
this.name = data().deepCopy(fields()[0].schema(), other.name);
fieldSetFlags()[0] = true;
}
if (isValidValue(fields()[1], other.favorite_number)) {
this.favorite_number = data().deepCopy(fields()[1].schema(), other.favorite_number);
fieldSetFlags()[1] = true;
}
if (isValidValue(fields()[2], other.favorite_color)) {
this.favorite_color = data().deepCopy(fields()[2].schema(), other.favorite_color);
fieldSetFlags()[2] = true;
}
}
/**
* Gets the value of the 'name' field.
* @return The value.
*/
public java.lang.CharSequence getName() {
return name;
}
/**
* Sets the value of the 'name' field.
* @param value The value of 'name'.
* @return This builder.
*/
public com.shufang.flinkapp.avro.bean.User.Builder setName(java.lang.CharSequence value) {
validate(fields()[0], value);
this.name = value;
fieldSetFlags()[0] = true;
return this;
}
/**
* Checks whether the 'name' field has been set.
* @return True if the 'name' field has been set, false otherwise.
*/
public boolean hasName() {
return fieldSetFlags()[0];
}
/**
* Clears the value of the 'name' field.
* @return This builder.
*/
public com.shufang.flinkapp.avro.bean.User.Builder clearName() {
name = null;
fieldSetFlags()[0] = false;
return this;
}
/**
* Gets the value of the 'favorite_number' field.
* @return The value.
*/
public java.lang.Integer getFavoriteNumber() {
return favorite_number;
}
/**
* Sets the value of the 'favorite_number' field.
* @param value The value of 'favorite_number'.
* @return This builder.
*/
public com.shufang.flinkapp.avro.bean.User.Builder setFavoriteNumber(java.lang.Integer value) {
validate(fields()[1], value);
this.favorite_number = value;
fieldSetFlags()[1] = true;
return this;
}
/**
* Checks whether the 'favorite_number' field has been set.
* @return True if the 'favorite_number' field has been set, false otherwise.
*/
public boolean hasFavoriteNumber() {
return fieldSetFlags()[1];
}
/**
* Clears the value of the 'favorite_number' field.
* @return This builder.
*/
public com.shufang.flinkapp.avro.bean.User.Builder clearFavoriteNumber() {
favorite_number = null;
fieldSetFlags()[1] = false;
return this;
}
/**
* Gets the value of the 'favorite_color' field.
* @return The value.
*/
public java.lang.CharSequence getFavoriteColor() {
return favorite_color;
}
/**
* Sets the value of the 'favorite_color' field.
* @param value The value of 'favorite_color'.
* @return This builder.
*/
public com.shufang.flinkapp.avro.bean.User.Builder setFavoriteColor(java.lang.CharSequence value) {
validate(fields()[2], value);
this.favorite_color = value;
fieldSetFlags()[2] = true;
return this;
}
/**
* Checks whether the 'favorite_color' field has been set.
* @return True if the 'favorite_color' field has been set, false otherwise.
*/
public boolean hasFavoriteColor() {
return fieldSetFlags()[2];
}
/**
* Clears the value of the 'favorite_color' field.
* @return This builder.
*/
public com.shufang.flinkapp.avro.bean.User.Builder clearFavoriteColor() {
favorite_color = null;
fieldSetFlags()[2] = false;
return this;
}
@Override
@SuppressWarnings("unchecked")
public User build() {
try {
User record = new User();
record.name = fieldSetFlags()[0] ? this.name : (java.lang.CharSequence) defaultValue(fields()[0]);
record.favorite_number = fieldSetFlags()[1] ? this.favorite_number : (java.lang.Integer) defaultValue(fields()[1]);
record.favorite_color = fieldSetFlags()[2] ? this.favorite_color : (java.lang.CharSequence) defaultValue(fields()[2]);
return record;
} catch (org.apache.avro.AvroMissingFieldException e) {
throw e;
} catch (java.lang.Exception e) {
throw new org.apache.avro.AvroRuntimeException(e);
}
}
}
@SuppressWarnings("unchecked")
private static final org.apache.avro.io.DatumWriter<User>
WRITER$ = (org.apache.avro.io.DatumWriter<User>)MODEL$.createDatumWriter(SCHEMA$);
@Override public void writeExternal(java.io.ObjectOutput out)
throws java.io.IOException {
WRITER$.write(this, SpecificData.getEncoder(out));
}
@SuppressWarnings("unchecked")
private static final org.apache.avro.io.DatumReader<User>
READER$ = (org.apache.avro.io.DatumReader<User>)MODEL$.createDatumReader(SCHEMA$);
@Override public void readExternal(java.io.ObjectInput in)
throws java.io.IOException {
READER$.read(this, SpecificData.getDecoder(in));
}
@Override protected boolean hasCustomCoders() { return true; }
@Override public void customEncode(org.apache.avro.io.Encoder out)
throws java.io.IOException
{
out.writeString(this.name);
if (this.favorite_number == null) {
out.writeIndex(1);
out.writeNull();
} else {
out.writeIndex(0);
out.writeInt(this.favorite_number);
}
if (this.favorite_color == null) {
out.writeIndex(1);
out.writeNull();
} else {
out.writeIndex(0);
out.writeString(this.favorite_color);
}
}
@Override public void customDecode(org.apache.avro.io.ResolvingDecoder in)
throws java.io.IOException
{
org.apache.avro.Schema.Field[] fieldOrder = in.readFieldOrderIfDiff();
if (fieldOrder == null) {
this.name = in.readString(this.name instanceof Utf8 ? (Utf8)this.name : null);
if (in.readIndex() != 0) {
in.readNull();
this.favorite_number = null;
} else {
this.favorite_number = in.readInt();
}
if (in.readIndex() != 0) {
in.readNull();
this.favorite_color = null;
} else {
this.favorite_color = in.readString(this.favorite_color instanceof Utf8 ? (Utf8)this.favorite_color : null);
}
} else {
for (int i = 0; i < 3; i++) {
switch (fieldOrder[i].pos()) {
case 0:
this.name = in.readString(this.name instanceof Utf8 ? (Utf8)this.name : null);
break;
case 1:
if (in.readIndex() != 0) {
in.readNull();
this.favorite_number = null;
} else {
this.favorite_number = in.readInt();
}
break;
case 2:
if (in.readIndex() != 0) {
in.readNull();
this.favorite_color = null;
} else {
this.favorite_color = in.readString(this.favorite_color instanceof Utf8 ? (Utf8)this.favorite_color : null);
}
break;
default:
throw new java.io.IOException("Corrupt ResolvingDecoder.");
}
}
}
}
}
4 定义序列化与反序列化的Kafka-Schema
package com.shufang.flinkapp.avro.schemas;
import com.shufang.flinkapp.avro.bean.User;
import com.shufang.flinkapp.avro.bean.User1;
import org.apache.avro.io.BinaryDecoder;
import org.apache.avro.io.BinaryEncoder;
import org.apache.avro.io.DecoderFactory;
import org.apache.avro.io.EncoderFactory;
import org.apache.avro.specific.SpecificDatumReader;
import org.apache.avro.specific.SpecificDatumWriter;
import org.apache.flink.api.common.serialization.DeserializationSchema;
import org.apache.flink.api.common.serialization.SerializationSchema;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
/**
* 该类主要是实现Kafka数据的
*/
public class MyAvroSchema implements SerializationSchema<User> , DeserializationSchema<User1> {
// User1与User2个类的schema是一样的，所以在反序列化的时候是可以共用的
@Override
public User1 deserialize(byte[] message) throws IOException {
// 用来保存结果数据
User1 user = new User1();
// 创建输入流用来读取二进制文件
ByteArrayInputStream arrayInputStream = new ByteArrayInputStream(message);
// 创建输入序列化执行器
SpecificDatumReader<User1> stockSpecificDatumReader = new SpecificDatumReader<User1>(user.getSchema());
//创建二进制解码器
BinaryDecoder binaryDecoder = DecoderFactory.get().directBinaryDecoder(arrayInputStream, null);
try {
// 数据读取
user=stockSpecificDatumReader.read(null, binaryDecoder);
} catch (IOException e) {
e.printStackTrace();
}
// 结果返回
return user;
}
@Override
public boolean isEndOfStream(User1 nextElement) {
return false;
}
@Override
public byte[] serialize(User user) {
// 创建序列化执行器
SpecificDatumWriter<User> writer = new SpecificDatumWriter<User>(user.getSchema());
// 创建一个流 用存储序列化后的二进制文件
ByteArrayOutputStream out = new ByteArrayOutputStream();
// 创建二进制编码器
BinaryEncoder encoder = EncoderFactory.get().directBinaryEncoder(out, null);
try {
// 数据入都流中
writer.write(user, encoder);
} catch (IOException e) {
e.printStackTrace();
}
return out.toByteArray();
}
@Override
public TypeInformation<User1> getProducedType() {
return TypeInformation.of(User1.class);
}
}
5 创建FlinkKafkaProducer将Avro序列化数据写入到Kafka
package com.shufang.flinkapp.avro.test;
import com.shufang.flinkapp.avro.bean.User;
import com.shufang.flinkapp.avro.schemas.MyAvroSchema;
import org.apache.avro.file.DataFileWriter;
import org.apache.avro.io.DatumWriter;
import org.apache.avro.specific.SpecificDatumWriter;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer;
import org.apache.kafka.clients.producer.ProducerConfig;
import java.util.Properties;
public class TestFlinkKafkaAvroProducer {
private static final String BOOTSTRAP_SERVER = "shufang101:9092,shufang102:9092,shufang103:9092";
public static void main(String[] args) throws Exception {
StreamExecutionEnvironment streamEnv = StreamExecutionEnvironment.getExecutionEnvironment();
/**
* 创建User的对象
*/
User user1 = new User();
user1.setName("Alyssa");
user1.setFavoriteNumber(256);
User user2 = new User("Ben", 7, "red");
User user3 = User.newBuilder()
.setName("Charlie")
.setFavoriteColor("blue")
.setFavoriteNumber(null)
.build();
DataStreamSource<User> sourceDS = streamEnv.fromElements(user1, user2, user3);
String topic = "test_avro";
// kafka producer的相关配置
Properties producerConfig = new Properties();
producerConfig.setProperty(ProducerConfig.ACKS_CONFIG, "all"); // 设置ACK应答为1，可以选择-1，0(all)
producerConfig.setProperty(ProducerConfig.TRANSACTION_TIMEOUT_CONFIG, 15 * 1000 * 60 + ""); //超时时长为15分钟
producerConfig.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVER);
// 创建Kafka的生产者
FlinkKafkaProducer<User> producer = new FlinkKafkaProducer<>(topic, new MyAvroSchema(), producerConfig);
sourceDS.print("avro = ");
sourceDS.addSink(producer);
streamEnv.execute();
}
}
6 创建FlinkKafkaConsumer进行Avro反序列化并转Json写出
package com.shufang.flinkapp.avro.test;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.shufang.flinkapp.avro.bean.User;
import com.shufang.flinkapp.avro.bean.User1;
import com.shufang.flinkapp.avro.schemas.MyAvroSchema;
import com.shufang.flinkapp.util.KafkaUtil;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.common.serialization.StringDeserializer;
import java.util.Properties;
public class TestFlinkKafkaAvroConsumer {
private static final String BOOTSTRAP_SERVER = "shufang101:9092,shufang102:9092,shufang103:9092";
public static void main(String[] args) throws Exception {
StreamExecutionEnvironment streamEnv = StreamExecutionEnvironment.getExecutionEnvironment();
String topic = "test_avro";
String groupId = "avroTestGroup";
String deserClass = "com.shufang.flinkapp.avro.schemas.MyAvroSchema.java";
// 配置Kafka的配置选项，这个可以通过PropertiesConfiguration从配置文件获取，当然还有很多其它的方法
Properties consumerProps = new Properties();
consumerProps.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, BOOTSTRAP_SERVER);
consumerProps.setProperty(ConsumerConfig.GROUP_ID_CONFIG, groupId);
consumerProps.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
consumerProps.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, MyAvroSchema.class.getName());
FlinkKafkaConsumer<User1> consumer = new FlinkKafkaConsumer<>(topic, new MyAvroSchema(), consumerProps);
consumer.setStartFromEarliest();
DataStreamSource<User1> userDS = streamEnv.addSource(consumer);
/*
* User = :3> {"name": "Alyssa", "favorite_number": 256, "favorite_color": null}
* User = :4> {"name": "Ben", "favorite_number": 7, "favorite_color": "red"}
* User = :2> {"name": "Charlie", "favorite_number": null, "favorite_color": "blue"}
*/
//userDS.print("User = ");
SingleOutputStreamOperator<String> user1DS = userDS.map(new MapFunction<User1, String>() {
@Override
public String map(User1 user1) throws Exception {
return user1.toString();
}
});
/*
* 3> {"name": "Alyssa", "favorite_number": 256, "favorite_color": null}
* 4> {"name": "Ben", "favorite_number": 7, "favorite_color": "red"}
* 2> {"name": "Charlie", "favorite_number": null, "favorite_color": "blue"}
*/
user1DS.print();
user1DS.addSink(KafkaUtil.getProducer("test_json"));
streamEnv.execute();
}
}