Flink写Redis Demo

1 pom文件

    4.0.0

    org.learn.LearnFlink
    LearnFlink
    1.0

    
        1.9.0
    

    
        
            local
            
                compile
            
            
                true
            
        
        
            pro
            
                provided
            
        
    

    
        
            org.apache.flink
            flink-java
            ${flink.version}
            ${scope}
        
        
            org.apache.flink
            flink-streaming-java_2.12
            ${flink.version}
            ${scope}
        

        
            org.slf4j
            slf4j-log4j12
            1.7.7
            ${scope}
        
        
            log4j
            log4j
            1.2.17
            ${scope}
        

        
            org.apache.flink
            flink-connector-kafka_2.12
            ${flink.version}
        
        
            org.apache.flink
            flink-connector-redis_2.10
            1.1.5
        

        
            com.google.code.gson
            gson
            2.8.5
        
    

    
        
            
                org.apache.maven.plugins
                maven-shade-plugin
                3.0.0
                
                    
                        package
                        
                            shade
                        
                        
                            
                                
                                    com.google.code.findbugs:jsr305
                                    org.slf4j:*
                                    log4j:*
                                
                            
                            
                                
                                    
                                    *:*
                                    
                                        META-INF/*.SF
                                        META-INF/*.DSA
                                        META-INF/*.RSA
                                    
                                
                            
                            
                                
                                    org.learn.Driver
                                
                            
                        
                    
                
            
            
                org.apache.maven.plugins
                maven-compiler-plugin
                
                    8
                    8
                
            
        
    

2 代码

2.1 入口类

public class Driver implements Serializable {

    public static void main(String[] args) throws Exception {
        String className = args[0];
        Class clazz = Class.forName(className);
        Method method = clazz.getDeclaredMethod("execute");

        method.invoke(clazz.newInstance());
    }
}

2.2 执行类

public class KafkaToRedis implements Serializable {

    private static final Logger logger = LoggerFactory.getLogger(KafkaToRedis.class);

    public static void execute() throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        // Checkpoint
        env.enableCheckpointing(5000, CheckpointingMode.AT_LEAST_ONCE);

        // parallelism
        env.setParallelism(1);
        env.setMaxParallelism(1);

        Properties props = new Properties();
        props.setProperty("bootstrap.servers", "localhost:9092");
        props.setProperty("group.id", "demoGroup");

        FlinkKafkaConsumer kafkaConsumer = new FlinkKafkaConsumer("demoTopic", new SimpleStringSchema(), props);
        kafkaConsumer.setStartFromLatest();

        env.addSource(kafkaConsumer)
            .map(new MapFunction() {
                @Override
                public TeacherInfo map(String value) throws Exception {
                    logger.info("==========> begin map, value: " + value);
                    TeacherInfo info = new TeacherInfo();
                    String[] arr = value.split(",");
                    info.setId(Integer.parseInt(arr[0]));
                    info.setName(arr[1]);
                    info.setGender(arr[2]);
                    info.setAge(arr[3]);
                    return info;
                }
            })
            .addSink(new SinkToRedis());
        env.execute();
    }
}

public class SinkToRedis extends RichSinkFunction {

    private static final Logger logger = LoggerFactory.getLogger(SinkToRedis.class);

    private Jedis jedis = null;

    @Override
    public void open(Configuration parameters) throws Exception {
        super.open(parameters);
        jedis = RedisUtil.getResource();
        logger.info("==========> open jedis: " + jedis.toString());
    }

    @Override
    public void invoke(TeacherInfo value, Context context) throws Exception {
        jedis.set(value.getId() + "", new Gson().toJson(value));
        logger.info("==========> set value to redis: " + value.toString());
    }

    @Override
    public void close() throws Exception {
        super.close();
        if (null != jedis) {
            logger.info("==========> close jedis: " + jedis.toString());
            jedis.close();
        }
    }
}

2.3 工具类

public class RedisUtil {

    private static volatile JedisPool jedisPool = null;

    public static Jedis getResource() {
        if (null == jedisPool) {
            synchronized (RedisUtil.class) {
                if (null == jedisPool) {
                    JedisPoolConfig config = new JedisPoolConfig();
                    config.setMaxTotal(4);
                    config.setMaxIdle(1);
                    config.setMaxWaitMillis(100);
                    config.setTestOnBorrow(true);
                    jedisPool = new JedisPool(config, "localhost", 6379);
                }
            }
        }
        return jedisPool.getResource();
    }
    
}

3 打包

mvn clean package -Ppro

4 启动

1. 启动 Flink；
2. 启动 Redis；
3. 启动 Zookeeper 和 Kafka。

5 运行

5.1 提交 Job

flink.bat run F:\learn\LearnFlink-1.0.jar org.learn.datastream.KafkaToRedis

5.2 向 kafka 发数据

1,aaa,male,10
2,bbb,female,20
3,ccc,male,30
4,ddd,male,40

6 日志

7 结果

8 Cancel Job