Flink 自定义Trigger
需求，滑动窗口统计，keyby下过来一条就触发窗口统计，如果没消息过来，按60s触发一次窗口。
只能自定义Trigger
直接上代码
package com.tc.flink.demo.stream;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.tc.flink.conf.KafkaConfig;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.windowing.time.Time;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011;
import java.util.Properties;
public class WindowCountOrTimeTrigger {
public static void main(String[] args) throws Exception {
final StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);
Properties propsConsumer = new Properties();
propsConsumer.setProperty("bootstrap.servers", KafkaConfig.KAFKA_BROKER_LIST);
propsConsumer.setProperty("group.id", "trafficwisdom-streaming");
propsConsumer.put("enable.auto.commit", false);
propsConsumer.put("max.poll.records", 1000);
FlinkKafkaConsumer011 consumer = new FlinkKafkaConsumer011("topic-test", new SimpleStringSchema(), propsConsumer);
consumer.setStartFromLatest();
DataStream stream = env.addSource(consumer);
stream.print();
DataStream> exposure = stream.map(new MapFunction>() {
@Override
public Tuple2 map(String value) throws Exception {
try {
JSONObject jsonObject = JSON.parseObject(value);
String itemId = jsonObject.getString("itemId");
return new Tuple2(itemId, 1);
} catch (Exception e) {
return Tuple2.of(null, null);
}
}
}).filter(tuple2 -> tuple2.f0 != null);
DataStream> result = exposure.keyBy(0).timeWindow(Time.minutes(5)).trigger(TimeCountTrigger.of(1, Time.minutes(1))).sum(1);
result.print();
env.execute();
}
}
自定义的Trigger
package com.tc.flink.demo.stream;
import org.apache.flink.annotation.PublicEvolving;
import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.common.state.ReducingState;
import org.apache.flink.api.common.state.ReducingStateDescriptor;
import org.apache.flink.api.common.typeutils.base.LongSerializer;
import org.apache.flink.streaming.api.windowing.time.Time;
import org.apache.flink.streaming.api.windowing.triggers.Trigger;
import org.apache.flink.streaming.api.windowing.triggers.TriggerResult;
import org.apache.flink.streaming.api.windowing.windows.Window;
/**
* A {@link Trigger} that fires once the count of elements in a pane reaches the given count.
*
* @param The type of {@link Window Windows} on which this trigger can operate.
*/
@PublicEvolving
public class TimeCountTrigger extends Trigger {
private static final long serialVersionUID = 1L;
private final long maxCount;
private final long interval;
private final ReducingStateDescriptor stateDesc =
new ReducingStateDescriptor<>("count", new Sum(), LongSerializer.INSTANCE);
private final ReducingStateDescriptor timeStateDesc =
new ReducingStateDescriptor<>("fire-time", new Min(), LongSerializer.INSTANCE);
private TimeCountTrigger(long maxCount, long interval) {
this.maxCount = maxCount;
this.interval = interval;
}
@Override
public TriggerResult onElement(Object element, long timestamp, W window, TriggerContext ctx) throws Exception {
System.out.println("onMaxCount ....");
ReducingState fireTimestamp = ctx.getPartitionedState(timeStateDesc);
timestamp = ctx.getCurrentProcessingTime();
if (fireTimestamp.get() == null) {
long start = timestamp - (timestamp % interval);
long nextFireTimestamp = start + interval;
ctx.registerProcessingTimeTimer(nextFireTimestamp);
fireTimestamp.add(nextFireTimestamp);
}
ReducingState count = ctx.getPartitionedState(stateDesc);
count.add(1L);
if (count.get() >= maxCount) {
count.clear();
return TriggerResult.FIRE;
}
return TriggerResult.CONTINUE;
}
@Override
public TriggerResult onEventTime(long time, W window, TriggerContext ctx) {
return TriggerResult.CONTINUE;
}
@Override
public TriggerResult onProcessingTime(long time, W window, TriggerContext ctx) throws Exception {
System.out.println("onProcessingTime ....");
ReducingState fireTimestamp = ctx.getPartitionedState(timeStateDesc);
if (fireTimestamp.get().equals(time)) {
fireTimestamp.clear();
fireTimestamp.add(time + interval);
ctx.registerProcessingTimeTimer(time + interval);
return TriggerResult.FIRE;
}
return TriggerResult.CONTINUE;
}
@Override
public void clear(W window, TriggerContext ctx) throws Exception {
System.out.println("clear ....");
ctx.getPartitionedState(stateDesc).clear();
ReducingState fireTimestamp = ctx.getPartitionedState(timeStateDesc);
long timestamp = fireTimestamp.get();
ctx.deleteProcessingTimeTimer(timestamp);
fireTimestamp.clear();
}
@Override
public boolean canMerge() {
return true;
}
@Override
public void onMerge(W window, OnMergeContext ctx) throws Exception {
ctx.mergePartitionedState(stateDesc);
ctx.mergePartitionedState(timeStateDesc);
}
@Override
public String toString() {
return "TimeCountTrigger(" + maxCount + ")";
}
/**
* Creates a trigger that fires once the number of elements in a pane reaches the given count.
*
* @param maxCount The count of elements at which to fire.
* @param The type of {@link Window Windows} on which this trigger can operate.
*/
public static TimeCountTrigger of(long maxCount, Time interval) {
return new TimeCountTrigger<>(maxCount, interval.toMilliseconds());
}
private static class Sum implements ReduceFunction {
private static final long serialVersionUID = 1L;
@Override
public Long reduce(Long value1, Long value2) throws Exception {
return value1 + value2;
}
}
private static class Min implements ReduceFunction {
private static final long serialVersionUID = 1L;
@Override
public Long reduce(Long value1, Long value2) throws Exception {
return Math.min(value1, value2);
}
}
}
模拟发消息
{"action":"exposure","itemId":"1cdlTUJUCidYgcUQALhCpg==","time":"2019-11-06 16:01:00"}
{"action":"exposure","itemId":"LXFMRmnKqM7JiV75KQt+GQ==","time":"2019-11-06 16:02:00"}
{"action":"exposure","itemId":"LXFMRmnKqM7JiV75KQt+GQ==","time":"2019-11-06 16:03:00"}
统计结果这个样子的
3> {"action":"exposure","itemId":"1cdlTUJUCidYgcUQALhCpg==","time":"2019-11-06 16:01:00"}
onMaxCount ....
1> (1cdlTUJUCidYgcUQALhCpg==,1)
onProcessingTime ....
1> (1cdlTUJUCidYgcUQALhCpg==,1)
3> {"action":"exposure","itemId":"LXFMRmnKqM7JiV75KQt+GQ==","time":"2019-11-06 16:02:00"}
onMaxCount ....
4> (LXFMRmnKqM7JiV75KQt+GQ==,1)
onProcessingTime ....
4> (LXFMRmnKqM7JiV75KQt+GQ==,1)
onProcessingTime ....
1> (1cdlTUJUCidYgcUQALhCpg==,1)
3> {"action":"exposure","itemId":"LXFMRmnKqM7JiV75KQt+GQ==","time":"2019-11-06 16:03:00"}
onMaxCount ....
4> (LXFMRmnKqM7JiV75KQt+GQ==,2)
onProcessingTime ....
onProcessingTime ....
1> (1cdlTUJUCidYgcUQALhCpg==,1)
4> (LXFMRmnKqM7JiV75KQt+GQ==,2)
onProcessingTime ....
onProcessingTime ....
clear ....
clear ....
说明：TimeCountTrigger其实根据org.apache.flink.streaming.api.windowing.triggers.ContinuousProcessingTimeTrigger和org.apache.flink.streaming.api.windowing.triggers.CountTrigger合并而来。