avro示例

avro是一个序列化框架

特点是

• 丰富的数据结构
• 使用快速的压缩二进制数据格式
• 自包含文件，模式和数据
• RPC
• 简单的动态语言结合功能，Avro 和动态语言结合后，读写数据文件和使用 RPC 协议都不需要生成代码，而代码生成作为一种可选的优化只值得在静态类型语言中实现

pom依赖

    
        org.apache.avro
        avro
        1.9.2
    

1. 不生成code示例

1.1 编写avsc文件User.avsc

{
    "namespace":"com.example.person",
    "type":"record",
    "name":"User",
    "fields":[
        {"name":"name", "type":"string"},
        {"name":"id", "type":"int"},
        {"name":"salary","type":"int"},
        {"name":"age","type":"int"},
        {"name":"address","type":"string"}
    ]
}

1.2 按User.avsc进行序列化，写文件

    public static void serializing() throws IOException {
        Schema schema = new Schema.Parser().parse(new File("D:\\2020\\avro-test\\src\\main\\avro\\User.avsc"));
        File file = new File("D:\\avro-test\\src\\main\\avro\\User.avro");
        
        DatumWriter datumWriter = new GenericDatumWriter(schema);
        DataFileWriter dataFileWriter = new DataFileWriter(datumWriter);
        dataFileWriter.create(schema, file);

        GenericRecord a = new GenericData.Record(schema);
        a.put("name", "zhangsan");
        a.put("id", 1);
        a.put("salary", 1000);
        a.put("age", 27);
        a.put("address", "shanghai tangzheng");
        dataFileWriter.append(a);

        a = new GenericData.Record(schema);
        a.put("name", "lisi");
        a.put("id", 2);
        a.put("salary", 2000);
        a.put("age", 29);
        a.put("address", "shanghai tangzheng");
        dataFileWriter.append(a);

        dataFileWriter.close();
    }

1.3 按User.avsc读取序列化的文件

    public static void deserializing() throws IOException {
        Schema schema = new Schema.Parser().parse(new File("D:\\avro-test\\src\\main\\avro\\User.avsc"));
        File file = new File("D:\\avro-test\\src\\main\\avro\\User.avro");

        DatumReader datumReader = new GenericDatumReader(schema);
        DataFileReader dataFileReader = new DataFileReader(file, datumReader);
        GenericRecord user = null;
        while(dataFileReader.hasNext()) {
            user = dataFileReader.next();
            System.out.println(user);
        }
        dataFileReader.close();
    }

输出如下

{"name": "zhangsan", "id": 1, "salary": 1000, "age": 27, "address": "shanghai tangzheng"}
{"name": "lisi", "id": 2, "salary": 2000, "age": 29, "address": "shanghai tangzheng"}

2. 生成code示例

2.1 编写avsc文件User.avsc

{
    "namespace":"com.example.person",
    "type":"record",
    "name":"User",
    "fields":[
        {"name":"name", "type":"string"},
        {"name":"id", "type":"int"},
        {"name":"salary","type":"int"},
        {"name":"age","type":"int"},
        {"name":"address","type":"string"}
    ]
}

2.2 调用avro-tools生成java代码

java -jar avro-tools-1.9.2.jar compile schema User.avsc ouput

生成的java类 User位于output/com/example/person目录下。

2.3 序列化文件

    public static void testSerial() throws IOException {
        DatumWriter userDatumWriter = new SpecificDatumWriter(User.class);
        DataFileWriter dataFileWriter = new DataFileWriter(userDatumWriter);
        dataFileWriter.create(User.SCHEMA$, new File("D:\\2020\\src\\main\\avro\\User.avro));

        User a = new User();
        a.setName("zhangsan");
        a.setId(1);
        a.setSalary(1000);
        a.setAge(28);
        a.setAddress("shanghai pudong");
        dataFileWriter.append(a);

        a = new User();
        a.setName("lisi");
        a.setId(2);
        a.setSalary(2000);
        a.setAge(32);
        a.setAddress("shanghai pudong");
        dataFileWriter.append(a);

        a = new User();
        a.setName("mawu");
        a.setId(3);
        a.setSalary(3000);
        a.setAge(38);
        a.setAddress("shanghai pudong");
        dataFileWriter.append(a);

        dataFileWriter.close();
        System.out.println("ok...");
    }

2.4 反序列化文件

public static void testDeserial() throws IOException {
        DatumReader reader = new SpecificDatumReader(User.class);
        DataFileReader dataFileReader = new DataFileReader(new File(path), reader);
        User user = null;
        while(dataFileReader.hasNext()) {
            user = dataFileReader.next();
            System.out.println(user);
        }
    }

输出结果如下

{"name": "zhangsan", "id": 1, "salary": 1000, "age": 28, "address": "shanghai pudong"}
{"name": "lisi", "id": 2, "salary": 2000, "age": 32, "address": "shanghai pudong"}
{"name": "mawu", "id": 3, "salary": 3000, "age": 38, "address": "shanghai pudong"}