Flink 学习三 Flink 流 & process function API

Flink 学习三 Flink 流&process function API

1.Flink 多流操作

1.1.split 分流 (deprecated)

把一个数据流根据数据分成多个数据流 1.2 版本后移除

1.2.分流操作 (使用侧流输出)

public class _02_SplitStream {

    public static void main(String[] args) throws Exception {

        // 获取环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);

        DataStreamSource<Integer> streamSource = env.fromElements(1, 2, 3, 4, 5);
        SingleOutputStreamOperator<Integer> processed = streamSource.process(new ProcessFunction<Integer, Integer>() {
            /**
             *
             * @param value 输出的数据
             * @param ctx A 上下文
             * @param out 主要流输出器
             * @throws Exception
             */
            @Override
            public void processElement(Integer value, ProcessFunction<Integer, Integer>.Context ctx,
                                       Collector<Integer> out) throws Exception {
                if (value % 3 == 0) {
                    //测流数据
                    ctx.output(new OutputTag<Integer>("3%0",TypeInformation.of(Integer.class)) , value);
                }if (value % 3 == 1) {
                    //测流数据
                    ctx.output(new OutputTag<Integer>("3%1",TypeInformation.of(Integer.class)) , value);
                }
                //主流 ,数据
                out.collect(value);
            }
        });

        DataStream<Integer> output0 = processed.getSideOutput(new OutputTag<>("3%0",TypeInformation.of(Integer.class)));
        DataStream<Integer> output1 = processed.getSideOutput(new OutputTag<>("3%1",TypeInformation.of(Integer.class)));
        output1.print();

        env.execute();
    }
}

1.3.connect

connect 连接 DataStream ,DataStream ==> ConnectedStream

两个DataStream 连接成一个新的ConnectedStream ,虽然两个流连接在一起,但是两个流依然是相互独立的,这个方法的最大用处是: 两个流共享State 状态

两个流在内部还是各自处理各自的逻辑 比如 CoMapFunction 内的map1,map2 还是各自处理 streamSource,streamSource2;

数据类型可以不一致

public class _03_ConnectedStream {

    public static void main(String[] args) throws Exception {

        // 获取环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);

        DataStreamSource<Integer> streamSource = env.fromElements(1, 2, 3, 4, 5);
        DataStreamSource<Integer> streamSource2 = env.fromElements(10, 20, 30, 40, 50);

        ConnectedStreams<Integer, Integer> connected = streamSource.connect(streamSource2);

        // 原来的 MapFunction ==>  CoMapFunction  ; flatMap ==> CoMapFunction
        SingleOutputStreamOperator<Object> mapped = connected.map(new CoMapFunction<Integer, Integer, Object>() {
            @Override
            public Object map1(Integer value) throws Exception {
                return value + 1;
            }

            @Override
            public Object map2(Integer value) throws Exception {
                return value * 10;
            }
        });

        mapped.print();

        env.execute();
    }
}

------------------------------------------------------------------
  --------------------         --------------------    
    streamSource         --->         map1  
  --------------------         --------------------

  --------------------         --------------------    
    streamSource2       --->          map2  
  --------------------         -------------------- 
------------------------------------------------------------------    

1.4.union

可以合并多个流,流数据类型必须一致,


public class _04_UnionStream {

    public static void main(String[] args) throws Exception {

        // 获取环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);

        DataStreamSource<Integer> streamSource = env.fromElements(1, 2, 3, 4, 5);
        DataStreamSource<Integer> streamSource2 = env.fromElements(10, 20, 30, 40, 50,80,1110);
        DataStream<Integer> unioned = streamSource.union(streamSource2);
        SingleOutputStreamOperator<String> union = unioned.map(new MapFunction<Integer, String>() {
            @Override
            public String map(Integer value) throws Exception {
                return "union" + value;
            }
        });
        union.print();
        env.execute();
    }
}


------------------------------------------------------------------
  --------------------          
    streamSource               
  --------------------              --------------------
						=====>        map
  --------------------              --------------------    
    streamSource2               
  --------------------         
------------------------------------------------------------------    

1.5.coGroup

coGroup 本质上是join 算子的底层算子

有界流的思想去处理; 比如上说是时间窗口: 5S内数据分组匹配

        <左边流>.coGroup(<右边流>)
                .where(<KeySelector>)
                .equalTo(<KeySelector>)
                .window(<窗口>)
                .apply(<处理逻辑>)

Flink 学习三 Flink 流 & process function API_第1张图片

数据组比如说是时间窗口是5或者是10s 为一批数据, 时间窗口内的数据完成后,根据 where,和 equalTo 选择的key 数据一致 来分组

public class _05_CoGroupStream {

    public static void main(String[] args) throws Exception {

        // 获取环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        DataStream<Person> name_idCard = env.socketTextStream("192.168.141.131", 8888).map(x -> {
            Person person = new Person();
            person.setName(x.split(",")[0]);
            person.setIdCard(x.split(",")[1]);
            return person;
        }).returns(TypeInformation.of(Person.class)).name("==idCard==");
        //name_idCard.print();

        DataStream<Person> name_addr = env.socketTextStream("192.168.141.131", 7777).map(x -> {
            Person person = new Person();
            person.setName(x.split(",")[0]);
            person.setAddr(x.split(",")[1]);
            return person;
        }).returns(TypeInformation.of(Person.class)).name("==addr==");
        //name_addr.print();

        DataStream<Person> dataStream = name_idCard.coGroup(name_addr)
                // 左边流的key
                .where(new KeySelector<Person, Object>() {
                    @Override
                    public Object getKey(Person value) throws Exception {
                        return value.getName();
                    }
                })
                // 右边流的key
                .equalTo(new KeySelector<Person, Object>() {
                    @Override
                    public Object getKey(Person value) throws Exception {
                        return value.getName();
                    }
                })
                //时间窗口
                .window(TumblingProcessingTimeWindows.of(Time.seconds(5)))
                //处理逻辑  左边 Person ,右边  Person ,输出 Person
                .apply(new CoGroupFunction<Person, Person, Person>() {
                    /**
                     * first 协调组第一个流个数据
                     * second 协调组第二个流数据
                     */
                    @Override
                    public void coGroup(Iterable<Person> first, Iterable<Person> second, Collector<Person> out) throws Exception {
                        //左连接实现
                        Iterator<Person> iterator = first.iterator();
                        while (iterator.hasNext()) {
                            Person next1 = iterator.next();
                            Iterator<Person> iterator1 = second.iterator();
                            Boolean noDataFlag = true;
                            while (iterator1.hasNext()) {
                                Person result = new Person(next1);
                                Person next = iterator1.next();
                                result.setAddr(next.getAddr());
                                out.collect(result);
                                noDataFlag = false;
                            }
                            if (noDataFlag) {
                                out.collect(next1);
                            }
                        }
                    }
                });

        dataStream.print();

        env.execute();
    }
}

1.6. join 关联操作

用于关联两个流,需要指定join 条件;需要在窗口中进行关联后的计算逻辑

join 使用coGroup 实现的

public class _06_JoinStream {

    public static void main(String[] args) throws Exception {

        // 获取环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        //Perple 数据打平为Tuple  name,idCard,addr
        DataStream<Tuple3<String, String,String>> name_idCard = env.socketTextStream("192.168.141.131", 8888).map(x -> {
            return Tuple3.of(x.split(",")[0],x.split(",")[1],"");
        }).returns(TypeInformation.of(new TypeHint<Tuple3<String, String,String>>() {
        })) ;

        DataStream<Tuple3<String, String,String>> name_addr = env.socketTextStream("192.168.141.131", 7777).map(x -> {
            return Tuple3.of(x.split(",")[0],"",x.split(",")[1]);
        }) .returns(TypeInformation.of(new TypeHint<Tuple3<String, String,String>>() {
        }));
        //name_addr.print();

        DataStream<Tuple3<String, String,String>> dataStream = name_idCard.join(name_addr)
                // 左边流的f0 字段
                .where(tp3->tp3.f0)
                // 右边流的f0 字段
                .equalTo(tp3->tp3.f0)
                //时间窗口
                .window(TumblingProcessingTimeWindows.of(Time.seconds(20)))
                //处理逻辑  左边 Person ,右边  Person ,输出 Person
                .apply(new JoinFunction<Tuple3<String, String,String>, Tuple3<String, String,String>, Tuple3<String, String,String>>() {
                    /**
                     * @param first 匹配到的数据  first input.
                     * @param second 匹配到的数据 second input.
                     * @return
                     * @throws Exception
                     */
                    @Override
                    public Tuple3 join(Tuple3 first, Tuple3 second) throws Exception {
                        return Tuple3.of(first.f0,first.f1,second.f2);
                    }
                });

        dataStream.print();
        env.execute();
    }
}

1.7.broadcast

   datastream1: 用户id|行为|操作数据                   datastream2: 用户id|用户name|用户phone   
windows time1 ---------------------------------- 	---------------------------------
				12  |click| xxdssd						12  |aa| 131	
				13  |click| dasd             			 13  |cc| 1331					
				14  |click| ad    						14  |dd| 1321	
windows time2 ---------------------------------- 	---------------------------------
				12  |click| sfs          															
				13  |click| sdfs       
				15  |click| ghf     					17  |dd| 1321											
windows time3 ----------------------------------  	---------------------------------
				14  |click| ghf   
				17  |click| ghf 												
       
       注: 左边流数据是基础数据,使用 join不合适 ,适合 broadcast
           broadcast 适用于关联字典表 
  
       主流算子		<<<----------------------------------	广播状态				
       
       

public class _07_BroadcastStream {

	public static void main(String[] args) throws Exception {

		// 获取环境
		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

		// 数据打平为 用户id|行为|操作数据
		DataStream<Tuple3<String, String, String>> operationInfo = env.socketTextStream("192.168.141.131", 8888)
				.map(x -> {
					return Tuple3.of(x.split(",")[0], x.split(",")[1], x.split(",")[2]);
				}).returns(TypeInformation.of(new TypeHint<Tuple3<String, String, String>>() {
				}));

		// 数据打平为 用户id|用户name|用户phone
		DataStream<Tuple3<String, String, String>> baseInfo = env.socketTextStream("192.168.141.131", 7777).map(x -> {
			return Tuple3.of(x.split(",")[0], x.split(",")[1], x.split(",")[2]);
		}).returns(TypeInformation.of(new TypeHint<Tuple3<String, String, String>>() {
		}));

        //状态描述
		MapStateDescriptor<String, Tuple3<String, String, String>> userBaseInfoStateDesc = new MapStateDescriptor<>(
				"user base info", TypeInformation.of(String.class),
				TypeInformation.of(new TypeHint<Tuple3<String, String, String>>() {
				}));
		// 基础信息 变成广播流
		BroadcastStream<Tuple3<String, String, String>> userBaseInfoBroadcast = baseInfo
				.broadcast(userBaseInfoStateDesc);

		// 关联行为流和广播流
		BroadcastConnectedStream<Tuple3<String, String, String>, Tuple3<String, String, String>> connected = operationInfo
				.connect(userBaseInfoBroadcast);

		SingleOutputStreamOperator<Tuple5<String, String, String, String, String>> processed =
				// 连接后,处理的逻辑
				// connected 如果是keyedStream ===> 参数就是 KeyedBroadcastProcessFunction
				// connected 如果不是keyedStream ===> 参数就是 BroadcastProcessFunction
				connected.process(new BroadcastProcessFunction<Tuple3<String, String, String>, // 左流的数据
						Tuple3<String, String, String>, // 广播的类型
						Tuple5<String, String, String, String, String> // 返回数据类型
				>() {

					/**
					 * 此方法是处理主流方法 主流来一条处理一下
					 * 
					 * @throws Exception
					 */
					@Override
					public void processElement(Tuple3<String, String, String> value, // 左流 主流 数据
							BroadcastProcessFunction<Tuple3<String, String, String>, Tuple3<String, String, String>, Tuple5<String, String, String, String, String>>.ReadOnlyContext ctx, // 上下文
							Collector<Tuple5<String, String, String, String, String>> out // 输出器
					) throws Exception {
						// 基础数据还没有 broadcastStateReadOnly
						// 和 processBroadcastElement 里面获取的 broadcastState 数据一致,只是是只读的
						// 数据是一致的
						ReadOnlyBroadcastState<String, Tuple3<String, String, String>> broadcastStateReadOnly = ctx
								.getBroadcastState(userBaseInfoStateDesc);
						if (broadcastStateReadOnly == null) {
							out.collect(Tuple5.of(value.f0, value.f1, value.f2, null, null));
						} else {
							Tuple3<String, String, String> baseInfo = broadcastStateReadOnly.get(value.f0);
							// 基础数据为空
							if (baseInfo == null) {
								out.collect(Tuple5.of(value.f0, value.f1, value.f2, null, null));
							} else {
								out.collect(Tuple5.of(value.f0, value.f1, value.f2, baseInfo.f1, baseInfo.f2));
							}
						}
					}

					/**
					 *
					 * 处理广播流数据:拿到数据后,存到状态里面
					 */
					@Override
					public void processBroadcastElement(Tuple3<String, String, String> value, // 广播流里面的一条数据
							BroadcastProcessFunction<Tuple3<String, String, String>, Tuple3<String, String, String>, Tuple5<String, String, String, String, String>>.Context ctx, // 上下文
							Collector<Tuple5<String, String, String, String, String>> out // 输出器
					) throws Exception {
						// 上下文 里面获取状态
						BroadcastState<String, Tuple3<String, String, String>> broadcastState = ctx
								.getBroadcastState(userBaseInfoStateDesc);
                          //状态里面 以用户id 作为key , 基础信息为value
						broadcastState.put(value.f0, value);
					}
				});

		processed.print();

		env.execute();
	}
}

2.Flink 编程 process function

2.1 process function 简介

process function相对于前面的map , flatmap ,filter 的区别就是,对数据的处理有更大的自由度; 可以获取到数据的上下文,数据处理逻辑 ,如何控制返回等交给编写者;

在事件驱动的应用中,使用最频繁的api 就是process function

注: 在对不同的流的时候, process function 的类型也不一致

数据流的转换

Flink 学习三 Flink 流 & process function API_第2张图片

不同的DataStream 的process 处理方法需要的参数类型有如下几种

2.2 ProcessFunction


public class _01_ProcessFunction {

	public static void main(String[] args) throws Exception {

		// 获取环境
		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

		// 数据打平为 用户id|行为|操作数据
		DataStreamSource<String> streamSource = env.fromElements("1,click,data1", "2,click1,data2", "10,flow,data1",
				"22,doubleclick,data22");
		DataStream<Tuple3<String, String, String>> operationInfo = streamSource.map(x -> {
			return Tuple3.of(x.split(",")[0], x.split(",")[1], x.split(",")[2]);
		}).returns(TypeInformation.of(new TypeHint<Tuple3<String, String, String>>() {
		}));

		// ProcessFunction
		SingleOutputStreamOperator<String> processed = operationInfo
				.process(new ProcessFunction<Tuple3<String, String, String>, String>() {
					// 处理元素
					@Override
					public void processElement(Tuple3<String, String, String> value,
							ProcessFunction<Tuple3<String, String, String>, String>.Context ctx, Collector<String> out)
							throws Exception {
						// 可以做主流输出
						out.collect(value.f0 + value.f1 + value.f2);
						// 可以做侧流输出
						ctx.output(new OutputTag<Tuple3<String, String, String>>("adasd",
								TypeInformation.of(new TypeHint<Tuple3<String, String, String>>() {
								})), value);
					}

					// 其余 声明周期方法 ... 任务状态 ... 都可以获取
					@Override
					public void open(Configuration parameters) throws Exception {
						super.open(parameters);
					}
				});

		processed.print();

		env.execute();
	}

}

2.3 KeyedProcessFunction

public class _02_KeyedProcessFunction {

	public static void main(String[] args) throws Exception {

		// 获取环境
		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

		// 数据打平为 用户id|行为|操作数据
		DataStreamSource<String> streamSource = env.fromElements("1,click,data1", "2,click1,data2", "10,flow,data1",
				"22,doubleclick,data22", "2,doubleclick,data22");
		DataStream<Tuple3<String, String, String>> operationInfo = streamSource.map(x -> {
			return Tuple3.of(x.split(",")[0], x.split(",")[1], x.split(",")[2]);
		}).returns(TypeInformation.of(new TypeHint<Tuple3<String, String, String>>() {
		}));

		// keyedStream
		KeyedStream<Tuple3<String, String, String>, String> keyedStream = operationInfo.keyBy(tp3 -> tp3.f0);

		// ProcessFunction
		SingleOutputStreamOperator<String> processed = keyedStream
				.process(new ProcessFunction<Tuple3<String, String, String>, String>() {
					@Override
					public void processElement(Tuple3<String, String, String> value,
							ProcessFunction<Tuple3<String, String, String>, String>.Context ctx, Collector<String> out)
							throws Exception {
                        out.collect((value.f0 + value.f1 + value.f2).toUpperCase(Locale.ROOT));
					}
				});

        processed.print();

		env.execute();
	}

}

2.4 ProcessWindowFunction

2.5 ProcessAllWindowFunction

2.6 CoProcessFunction

2.7 ProcessJoinFunction

2.8 BroadcastProcessFunction

参考1.7

2.9 KeyedBroadcastProcessFunction

3.测试

package demo.sff.flink.exercise;

import demo.sff.flink.source.Person;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.state.BroadcastState;
import org.apache.flink.api.common.state.MapStateDescriptor;
import org.apache.flink.api.common.state.ReadOnlyBroadcastState;
import org.apache.flink.api.common.typeinfo.TypeHint;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.api.java.tuple.Tuple5;
import org.apache.flink.connector.file.sink.FileSink;
import org.apache.flink.connector.jdbc.JdbcConnectionOptions;
import org.apache.flink.connector.jdbc.JdbcExecutionOptions;
import org.apache.flink.connector.jdbc.JdbcSink;
import org.apache.flink.connector.jdbc.JdbcStatementBuilder;
import org.apache.flink.core.fs.Path;
import org.apache.flink.formats.parquet.ParquetWriterFactory;
import org.apache.flink.formats.parquet.avro.ParquetAvroWriters;
import org.apache.flink.streaming.api.datastream.*;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.co.BroadcastProcessFunction;
import org.apache.flink.streaming.api.functions.sink.SinkFunction;
import org.apache.flink.streaming.api.functions.sink.filesystem.OutputFileConfig;
import org.apache.flink.streaming.api.functions.sink.filesystem.bucketassigners.DateTimeBucketAssigner;
import org.apache.flink.util.Collector;
import org.apache.flink.util.OutputTag;

import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.Random;

/**
 * 创建流 Stream 1: id | event | count 1,event1,3 2,event1,5 3,event1,4
 *
 * Stream 2: id | gender | city 1 , male ,beijin 2 ,female,shanghai
 *
 * 需求 : 1.Stream 1 按照 count字段展开为对应的个数 比如id=1 展开为3条 1,event1,随机1 1,event1,随机2
 * 1,event1,随机3 ,id=2 展开为5 条
 *
 * 2.Stream 1 关联上 Stream 2 数据
 *
 * 3.关联不上 测流 其余主流
 *
 * 4.主流,性别分组,取出最大随机数
 *
 * 5.主流写入mysql
 *
 * 6.测流写入parquet
 */
public class Test1 {

	public static void main(String[] args) throws Exception {
		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
		env.setParallelism(1);

		// 创建流 后面可以使用socket 替换 stream2 先写入广播 不然关联不上
		DataStreamSource<String> stream1 = env.fromElements("1,event1,3", "2,event1,5", "3,event3,4");
		DataStreamSource<String> stream2 = env.fromElements("1,male,beijin", " 2,female,shanghai");

		DataStream<Tuple3<String, String, String>> streamOperator1 = stream1
				.map(x -> Tuple3.of(x.split(",")[0], x.split(",")[1], x.split(",")[2]))
				.returns(TypeInformation.of(new TypeHint<Tuple3<String, String, String>>() {
				}));
		DataStream<Tuple3<String, String, String>> streamOperator2 = stream2
				.map(x -> Tuple3.of(x.split(",")[0], x.split(",")[1], x.split(",")[2]))
				.returns(TypeInformation.of(new TypeHint<Tuple3<String, String, String>>() {
				}));

		// 需求1
		DataStream<Tuple3<String, String, String>> mapDataStream = streamOperator1
				.flatMap(new FlatMapFunction<Tuple3<String, String, String>, Tuple3<String, String, String>>() {
					@Override
					public void flatMap(Tuple3<String, String, String> value,
							Collector<Tuple3<String, String, String>> out) throws Exception {
						Integer integer = Integer.valueOf(value.f2);
						for (Integer i = 0; i < integer; i++) {
							int r = new Random().nextInt(100);
							out.collect(Tuple3.of(value.f0, value.f1, r + ""));
						}
					}
				}).returns(TypeInformation.of(new TypeHint<Tuple3<String, String, String>>() {
				}));

		// mapDataStream.print();

		// 需求2 stream2 数据广播
		MapStateDescriptor<String, Tuple3<String, String, String>> descriptor = new MapStateDescriptor<String, Tuple3<String, String, String>>(
				"userinfo", TypeInformation.of(String.class),
				TypeInformation.of(new TypeHint<Tuple3<String, String, String>>() {
				}));
		BroadcastStream<Tuple3<String, String, String>> tuple3BroadcastStream = streamOperator2.broadcast(descriptor);

		BroadcastConnectedStream<Tuple3<String, String, String>, Tuple3<String, String, String>> tuple3BroadcastConnectedStream = mapDataStream
				.connect(tuple3BroadcastStream);

		SingleOutputStreamOperator<Tuple5<String, String, String, String, String>> processed = tuple3BroadcastConnectedStream
				.process(
						new BroadcastProcessFunction<Tuple3<String, String, String>, Tuple3<String, String, String>, Tuple5<String, String, String, String, String>>() {

							@Override
							public void processElement(Tuple3<String, String, String> value,
									BroadcastProcessFunction<Tuple3<String, String, String>, Tuple3<String, String, String>, Tuple5<String, String, String, String, String>>.ReadOnlyContext ctx,
									Collector<Tuple5<String, String, String, String, String>> out) throws Exception {
								ReadOnlyBroadcastState<String, Tuple3<String, String, String>> broadcastState = ctx
										.getBroadcastState(descriptor);
								// 需求3.关联不上 测流 其余主流
								if (broadcastState == null) {
									// out.collect(Tuple5.of(value.f0, value.f1, value.f2, null, null));
									ctx.output(new OutputTag<String>("nojoin", TypeInformation.of(String.class)),
											value.f0 + value.f1 + value.f2);
								} else {
									Tuple3<String, String, String> stringTuple3 = broadcastState.get(value.f0);
									if (stringTuple3 == null) {
										// out.collect(Tuple5.of(value.f0, value.f1, value.f2, null, null));
										ctx.output(new OutputTag<String>("nojoin", TypeInformation.of(String.class)),
												value.f0 + value.f1 + value.f2);
									} else {
										out.collect(Tuple5.of(value.f0, value.f1, value.f2, stringTuple3.f1,
												stringTuple3.f2));
									}
								}
							}

							@Override
							public void processBroadcastElement(Tuple3<String, String, String> value,
									BroadcastProcessFunction<Tuple3<String, String, String>, Tuple3<String, String, String>, Tuple5<String, String, String, String, String>>.Context ctx,
									Collector<Tuple5<String, String, String, String, String>> out) throws Exception {
								BroadcastState<String, Tuple3<String, String, String>> broadcastState = ctx
										.getBroadcastState(descriptor);
								broadcastState.put(value.f0, value);

							}
						})
				.returns(TypeInformation.of(new TypeHint<Tuple5<String, String, String, String, String>>() {
				}));
		// 主流
		processed.print();
		// 测流
		DataStream<String> sideOutput = processed
				.getSideOutput(new OutputTag<String>("nojoin", TypeInformation.of(String.class)));

		// sideOutput.print();

		// 需求4 主流,性别分组,取出最大随机数
		SingleOutputStreamOperator<Tuple5<String, String, Integer, String, String>> streamOperator = processed
				.keyBy(x -> x.f3)
				.map(new MapFunction<Tuple5<String, String, String, String, String>, Tuple5<String, String, Integer, String, String>>() {
					@Override
					public Tuple5<String, String, Integer, String, String> map(
							Tuple5<String, String, String, String, String> value) throws Exception {
						return Tuple5.of(value.f0, value.f1, Integer.valueOf(value.f2), value.f3, value.f4);
					}
				}).returns(TypeInformation.of(new TypeHint<Tuple5<String, String, Integer, String, String>>() {
				}));
		SingleOutputStreamOperator<Tuple5<String, String, Integer, String, String>> maxBy = streamOperator
				.keyBy(tp5 -> tp5.f3).maxBy(2);
		maxBy.print();

		// 5.主流写入mysql  未验证 待测试
		String sql = " insert into testa values (?,?,?,?,?) on duplicate key a=?,b=?,c=?,d=?,e=?  ";
		SinkFunction<Tuple5<String, String, Integer, String, String>> jdbcSink = JdbcSink.sink(sql,
				new JdbcStatementBuilder<Tuple5<String, String, Integer, String, String>>() {
					@Override
					public void accept(PreparedStatement preparedStatement,
							Tuple5<String, String, Integer, String, String> tuple5) throws SQLException {
						preparedStatement.setString(0, tuple5.f0);
						preparedStatement.setString(1, tuple5.f1);
						preparedStatement.setInt(2, tuple5.f2);
						preparedStatement.setString(3, tuple5.f3);
						preparedStatement.setString(4, tuple5.f4);
						preparedStatement.setString(5, tuple5.f0);
						preparedStatement.setString(6, tuple5.f1);
						preparedStatement.setInt(7, tuple5.f2);
						preparedStatement.setString(8, tuple5.f3);
						preparedStatement.setString(9, tuple5.f4);
					}
				}, JdbcExecutionOptions.builder().withBatchSize(2) // 两条数据一批插入
						.withMaxRetries(3) // 失败插入重试次数
						.build(),
				new JdbcConnectionOptions.JdbcConnectionOptionsBuilder().withPassword("root") // jdbc 连接信息
						.withUsername("root")// jdbc 连接信息
						.withUrl("jdbc:mysql://192.168.141.131:3306/flinkdemo").build());
		streamOperator.addSink(jdbcSink);

		// 6.测流写入parquet  未验证 待测试
		ParquetWriterFactory<String> writerFactory = ParquetAvroWriters.forReflectRecord(String.class);
		FileSink<String> build = FileSink.forBulkFormat(new Path("d:/sink"), writerFactory)
				.withBucketAssigner(new DateTimeBucketAssigner<String>()) // 文件分桶策略
				.withBucketCheckInterval(5)// 文件夹异步线程创建和检测周期
				.withOutputFileConfig(OutputFileConfig.builder().withPartPrefix("flinkdemo") // 文件前缀
						.withPartSuffix(".txt") // 文件后缀
						.build())// 文件的输出格式对象
				.build();

		sideOutput.sinkTo(build);

		env.execute();

	}

}

你可能感兴趣的:(大数据之路,flink,学习,java)