flink1.17 eventWindow不要配置processTrigger
理论上可以eventtime processtime混用,但是下面代码测试发现bug,输入一条数据会一直输出.
flink github无法提bug/问题. apache jira账户新建后竟然flink又需要一个账户,放弃
bug复现操作
idea运行代码后 往source kafka发送一条数据
a,1,1690304400000可以看到无限输出:
理论上时间语义不建议混用,但是在rich函数中的确可以做到混用且正常使用
问题复现代码
package com.yy.flinkWindowAndTrigger
import com.yy.flinkWindow.M1
import org.apache.flink.api.common.eventtime.WatermarkStrategy
import org.apache.flink.configuration.{Configuration, RestOptions}
import org.apache.flink.connector.kafka.source.KafkaSource
import org.apache.flink.connector.kafka.source.enumerator.initializer.OffsetsInitializer
import org.apache.flink.streaming.api.functions.timestamps.BoundedOutOfOrdernessTimestampExtractor
import org.apache.flink.streaming.api.scala._
import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows
import org.apache.flink.streaming.api.windowing.time.Time
import org.apache.flink.streaming.api.windowing.time.Time.seconds
import org.apache.flink.streaming.api.windowing.triggers.{ContinuousProcessingTimeTrigger, CountTrigger, ProcessingTimeTrigger}
import org.apache.flink.streaming.api.windowing.windows.TimeWindow
import org.joda.time.Seconds
object flinkEventWindowAndProcessTriggerBUGLearn {
def main(args: Array[String]): Unit = {
// flink 启动本地webui
val conf = new Configuration
conf.setInteger(RestOptions.PORT, 28080)
// val env = StreamExecutionEnvironment.getExecutionEnvironment
val env = StreamExecutionEnvironment.createLocalEnvironmentWithWebUI(conf)
// val env = StreamExecutionEnvironment.getExecutionEnvironment
env.setParallelism(1)
env.configure(conf)
/*
kafka输入:
a,1,1690304400000 //对应 2023-07-26 01:00:00 (无限输出) //如果传入 a,1,1693037756000 对应:2023-08-26 16:15:56 (1条/s)
a,1,7200000 // 1970-01-1 10:00:00
*/
val brokers = "172.18.105.147:9092"
val source = KafkaSource.builder[String].setBootstrapServers(brokers)
.setTopics("t1")
.setGroupId("my-group-23asdf46")
.setStartingOffsets(OffsetsInitializer.latest())
// .setDeserializer() // 参数: KafkaRecordDeserializationSchema
.setDeserializer(new M1())
.build()
val ds1 = env.fromSource(source, WatermarkStrategy.noWatermarks(), "Kafka Source")
val s1 = ds1
.map(_.split(","))
.map(x => C1(x(0), x(1).toInt, x(2).toLong)) // key number 时间戳
.assignTimestampsAndWatermarks(new OTAWatermarks(Time.seconds(0)))
.keyBy(_.f1)
.window(TumblingEventTimeWindows.of(seconds(10)))
.trigger(ContinuousProcessingTimeTrigger.of[TimeWindow](seconds(10L)))
.reduce((x, y) => C1(x.f1, x.f2 + y.f2, 100L))
s1.print()
env.execute("KafkaNewSourceAPi")
}
// 乱序流
class OTAWatermarks(time: Time) extends BoundedOutOfOrdernessTimestampExtractor[C1](time) {
override def extractTimestamp(element: C1): Long = {
element.f3
}
}
// key num timestamp
case class C1(f1: String, f2: Int, f3: Long)
}
-
-
maven pom
4.0.0
org.example
FlinkLocalDemo
1.0-SNAPSHOT
jar
FlinkLocalDemo
http://maven.apache.org
UTF-8
1.17.1
2.12
2.12.8
joda-time
joda-time
2.12.5
org.apache.flink
flink-avro
${flink.version}
org.apache.flink
flink-runtime-web
${flink.version}
com.alibaba.fastjson2
fastjson2
2.0.33
com.alibaba
fastjson
1.2.83
junit
junit
3.8.1
test
org.apache.flink
flink-table-common
${flink.version}
org.apache.flink
flink-connector-kafka
${flink.version}
provided
org.apache.flink
flink-json
${flink.version}
org.apache.flink
flink-scala_${scala.binary.version}
${flink.version}
provided
org.apache.flink
flink-streaming-scala_${scala.binary.version}
${flink.version}
org.apache.flink
flink-csv
${flink.version}
org.apache.flink
flink-table-api-java-bridge
${flink.version}
org.apache.flink
flink-table-api-scala-bridge_${scala.binary.version}
${flink.version}
org.apache.flink
flink-table-planner-loader
${flink.version}
org.apache.flink
flink-table-runtime
${flink.version}
provided
org.apache.flink
flink-connector-files
${flink.version}
org.apache.flink
flink-clients
${flink.version}
org.apache.flink
flink-connector-jdbc
3.1.0-1.17
provided
mysql
mysql-connector-java
8.0.11
org.apache.maven.plugins
maven-shade-plugin
2.4.3
package
shade
*:*
META-INF/*.SF
META-INF/*.DSA
META-INF/*.RSA
org.scala-tools
maven-scala-plugin
2.15.2
compile
testCompile
net.alchim31.maven
scala-maven-plugin
3.2.2
scala-compile-first
process-resources
add-source
compile
${scala.version}
org.apache.maven.plugins
maven-assembly-plugin
2.5.5
jar-with-dependencies
org.apache.maven.plugins
maven-compiler-plugin
3.1
11
11