一直以来都是使用spark structstreaming 读取kafka数据，最近一直在尝试用flink读取kafka数据。
开发环境配置:
kafka + flink + idea + gradle
- 1：flink 在gradle中的配置如下：
plugins {
id 'java'
id 'scala'
}
group 'com.demo.flink'
version '1.0-SNAPSHOT'
sourceCompatibility = 1.8
repositories {
maven{ url 'http://maven.aliyun.com/nexus/content/groups/public'}
mavenCentral()
}
jar{
manifest{
attributes 'Main-Class' : 'flink_demo'
}
}
dependencies {
// testCompile group: 'junit', name: 'junit', version: '4.12'
compile group: 'org.apache.flink', name: 'flink-scala_2.11', version: '1.10.2'
compile group: 'org.apache.flink', name: 'flink-core', version: '1.10.2'
compile group: 'org.apache.flink', name: 'flink-streaming-scala_2.11', version: '1.10.2'
compile group: 'org.apache.flink', name: 'flink-clients_2.11', version: '1.10.2'
compile group: 'org.apache.flink', name: 'flink-connector-kafka_2.11', version: '1.10.0'
compile group: 'org.apache.flink', name : 'flink-avro', version: '1.10.0'
compile group: 'org.apache.kafka', name: 'kafka-clients', version: '1.1.0'
}
Flink 代码
- 2：flink读取kafka消息
import java.util.Properties
import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer
import org.apache.flink.streaming.api.scala._
import org.apache.flink.streaming.util.serialization.SimpleStringSchema
object flink_kafka {
def main(args: Array[String]): Unit = {
// 配置flink的上下文环境
val env = StreamExecutionEnvironment.getExecutionEnvironment
env.enableCheckpointing(5000)
import org.apache.flink.streaming.api.TimeCharacteristic
env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
// 配置kafka的属性
val prop = new Properties()
prop.setProperty("bootstrap.servers","ip:9092")
prop.setProperty("group.id","cn1")
prop.setProperty("key.deserializer","org.apache.kafka.common.serialization.StringDeserializer")
prop.setProperty("value.deserializer","org.apache.kafka.common.serialization.StringDeserializer")
// 创建一个消费者，读取topuc为test的消息
val consumer = new FlinkKafkaConsumer[String]("test", new SimpleStringSchema(), prop).setStartFromEarliest()
// 将消费者添加到输入源
val stream = env.addSource(consumer)
//打印并执行信息
stream.print()
env.execute("flink kafka demo")
}
}
- 3 : 输出结果
D:\jdk\bin\java.exe "-javaagent:D:\idea\IntelliJ IDEA 2018.2.8\lib\idea_rt.jar=62570:D:\idea\IntelliJ IDEA 2018.2.8\bin" -Dfile.encoding=UTF-8 -classpath D:\jdk\jre\lib\charsets.jar;D:\jdk\jre\lib\deploy.jar;D:\jdk\jre\lib\ext\access-bridge-64.jar;D:\jdk\jre\lib\ext\cldrdata.jar;D:\jdk\jre\lib\ext\dnsns.jar;D:\jdk\jre\lib\ext\jaccess.jar;D:\jdk\jre\lib\ext\jfxrt.jar;D:\jdk\jre\lib\ext\localedata.jar;D:\jdk\jre\lib\ext\nashorn.jar;D:\jdk\jre\lib\ext\sunec.jar;D:\jdk\jre\lib\ext\sunjce_provider.jar;D:\jdk\jre\lib\ext\sunmscapi.jar;D:\jdk\jre\lib\ext\sunpkcs11.jar;D:\jdk\jre\lib\ext\zipfs.jar;D:\jdk\jre\lib\javaws.jar;D:\jdk\jre\lib\jce.jar;D:\jdk\jre\lib\jfr.jar;D:\jdk\jre\lib\jfxswt.jar;D:\jdk\jre\lib\jsse.jar;D:\jdk\jre\lib\management-agent.jar;D:\jdk\jre\lib\plugin.jar;D:\jdk\jre\lib\resources.jar;D:\jdk\jre\lib\rt.jar;D:\idea\flink_demo\demo\out\production\classes;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.apache.flink\flink-streaming-scala_2.11\1.10.2\c5317ac6651c7583a592c5adf57ca48761193f45\flink-streaming-scala_2.11-1.10.2.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.apache.flink\flink-scala_2.11\1.10.2\d80e0ab2c3b97742598cdba7c2ed3047bb24262a\flink-scala_2.11-1.10.2.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.apache.flink\flink-streaming-java_2.11\1.10.2\65c41cbadda97b986bdaaaf8ddad23337bb38452\flink-streaming-java_2.11-1.10.2.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.apache.flink\flink-clients_2.11\1.10.2\d3bdc977356d92acec5fc126ac3321ebc430a69c\flink-clients_2.11-1.10.2.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.apache.flink\flink-optimizer_2.11\1.10.2\2464d5e86658d6d931b836bf27f3173c682f8d52\flink-optimizer_2.11-1.10.2.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.apache.flink\flink-runtime_2.11\1.10.2\98a3162ae84337cfb436b793fee008f95060e3f4\flink-runtime_2.11-1.10.2.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.apache.flink\flink-java\1.10.2\57d24de3f2c1271f0fdf85affd386f0ba22040cb\flink-java-1.10.2.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.apache.flink\flink-hadoop-fs\1.10.2\eab43b996bf4b0bf2f4bb7811b21e81ea055bc7c\flink-hadoop-fs-1.10.2.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.apache.flink\flink-core\1.10.2\b5feacc4764894f0fbc9abbd92e9d5296351b70\flink-core-1.10.2.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.apache.flink\flink-connector-kafka_2.11\1.10.0\b90f39434aab5b7eafb0b83f8f5746881394db99\flink-connector-kafka_2.11-1.10.0.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.apache.flink\flink-avro\1.10.0\28acfea9042a497d44d5d1d832bacc543153cc66\flink-avro-1.10.0.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.apache.kafka\kafka-clients\2.2.0\f887978d8863dc33da32a07ee4877183590ed367\kafka-clients-2.2.0.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.apache.flink\flink-shaded-asm-7\7.1-9.0\cf78ac08c839f2e0f7764fd0040c385e9c0459f1\flink-shaded-asm-7-7.1-9.0.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.scala-lang\scala-compiler\2.11.12\a1b5e58fd80cb1edc1413e904a346bfdb3a88333\scala-compiler-2.11.12.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.scala-lang\scala-reflect\2.11.12\2bb23c13c527566d9828107ca4108be2a2c06f01\scala-reflect-2.11.12.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.scala-lang.modules\scala-xml_2.11\1.0.5\77ac9be4033768cf03cc04fbd1fc5e5711de2459\scala-xml_2.11-1.0.5.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\com.typesafe.akka\akka-stream_2.11\2.5.21\b38d270d0158d38f103bce546d708155469b89c6\akka-stream_2.11-2.5.21.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\com.typesafe\ssl-config-core_2.11\0.3.7\6e82461eef336716387f6c233555cf916165c70f\ssl-config-core_2.11-0.3.7.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.scala-lang.modules\scala-parser-combinators_2.11\1.1.1\5dbfa7b37ae609f2629f957f399731b58b00a799\scala-parser-combinators_2.11-1.1.1.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.scala-lang\scala-library\2.11.12\bf5534e6fec3d665bd6419c952a929a8bdd4b591\scala-library-2.11.12.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.apache.flink\flink-annotations\1.10.2\c1d18570f65682dbb80909e79dd788ae84e60e49\flink-annotations-1.10.2.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.apache.avro\avro\1.8.2\91e3146dfff4bd510181032c8276a3a0130c0697\avro-1.8.2.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\com.typesafe.akka\akka-slf4j_2.11\2.5.21\432b7c8f844df7afb6359cf2212dd9058b9a9545\akka-slf4j_2.11-2.5.21.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.clapper\grizzled-slf4j_2.11\1.3.2\8e83c864c7a16b391bd6e4099a9c40a437781aea\grizzled-slf4j_2.11-1.3.2.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.slf4j\slf4j-api\1.7.25\da76ca59f6a57ee3102f8f9bd9cee742973efa8a\slf4j-api-1.7.25.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\com.google.code.findbugs\jsr305\1.3.9\40719ea6961c0cb6afaeb6a921eaa1f6afd4cfdf\jsr305-1.3.9.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.apache.flink\flink-metrics-core\1.10.2\6182d8a0b9510d9a77ee15b14855635a3a8ecaca\flink-metrics-core-1.10.2.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.apache.flink\flink-connector-kafka-base_2.11\1.10.0\cd6400739fd6a902540290659d1636333c968b9e\flink-connector-kafka-base_2.11-1.10.0.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.apache.flink\flink-queryable-state-client-java\1.10.2\a8fe1a43e207143c6f9c79e8559ed202901f79d\flink-queryable-state-client-java-1.10.2.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.apache.flink\force-shading\1.10.2\1eb05b7965a62d6d1b0695ec61e35cfbcf4b42f6\force-shading-1.10.2.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.apache.commons\commons-lang3\3.3.2\90a3822c38ec8c996e84c16a3477ef632cbc87a3\commons-lang3-3.3.2.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\com.esotericsoftware.kryo\kryo\2.24.0\c6b206e80cfd97e66a1364003724491c757b92f\kryo-2.24.0.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\commons-collections\commons-collections\3.2.2\8ad72fe39fa8c91eaaf12aadb21e0c3661fe26d5\commons-collections-3.2.2.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.apache.commons\commons-compress\1.18\1191f9f2bc0c47a8cce69193feb1ff0a8bcb37d5\commons-compress-1.18.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.apache.flink\flink-shaded-guava\18.0-9.0\950b436f48022f9020f1f5dc7bbe321784c42451\flink-shaded-guava-18.0-9.0.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\commons-cli\commons-cli\1.3.1\1303efbc4b181e5a58bf2e967dc156a3132b97c0\commons-cli-1.3.1.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.apache.commons\commons-math3\3.5\13af68e9a11576953f5c4b09436f8207be9d3a27\commons-math3-3.5.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\com.esotericsoftware.minlog\minlog\1.2\59bfcd171d82f9981a5e242b9e840191f650e209\minlog-1.2.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.objenesis\objenesis\2.1\87c0ea803b69252868d09308b4618f766f135a96\objenesis-2.1.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\commons-io\commons-io\2.4\b1b6ea3b7e4aa4f492509a4952029cd8e48019ad\commons-io-2.4.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.apache.flink\flink-shaded-netty\4.1.39.Final-9.0\27ddcd1b7d0df819236b489fa624bedf3c0865a9\flink-shaded-netty-4.1.39.Final-9.0.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.apache.flink\flink-shaded-jackson\2.10.1-9.0\74d32223d733b934f541a69bc7bc4aad8fb221a2\flink-shaded-jackson-2.10.1-9.0.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.javassist\javassist\3.24.0-GA\d7466fc2e3af7c023e95c510f06448ad29b225b3\javassist-3.24.0-GA.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\com.typesafe.akka\akka-actor_2.11\2.5.21\22d15d47c9e33393f09f688b2c58dbc81f960a1e\akka-actor_2.11-2.5.21.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\com.typesafe.akka\akka-protobuf_2.11\2.5.21\b3e264d1314272f78f2718e77aaa0110fa9af831\akka-protobuf_2.11-2.5.21.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\com.github.scopt\scopt_2.11\3.5.0\4adcea609ae83ae73be401872f0a36c018b34fea\scopt_2.11-3.5.0.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.xerial.snappy\snappy-java\1.1.7.2\307b286efd119ad2c6d4291128bf110bddc68088\snappy-java-1.1.7.2.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\com.twitter\chill_2.11\0.7.6\8154b0442f0c3d5954fc132e2f44f7df13d2bfcc\chill_2.11-0.7.6.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.lz4\lz4-java\1.5.0\d36fb639f06aaa4f17307625f80e2e32f815672a\lz4-java-1.5.0.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\com.github.luben\zstd-jni\1.3.8-1\455bd7c5a8a68b1aebcace632b6ebcb4fd92d32a\zstd-jni-1.3.8-1.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.codehaus.jackson\jackson-mapper-asl\1.9.13\1ee2f2bed0e5dd29d1cb155a166e6f8d50bbddb7\jackson-mapper-asl-1.9.13.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.codehaus.jackson\jackson-core-asl\1.9.13\3c304d70f42f832e0a86d45bd437f692129299a4\jackson-core-asl-1.9.13.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\com.thoughtworks.paranamer\paranamer\2.7\3ed64c69e882a324a75e890024c32a28aff0ade8\paranamer-2.7.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.tukaani\xz\1.5\9c64274b7dbb65288237216e3fae7877fd3f2bee\xz-1.5.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\com.typesafe\config\1.3.3\4b68c2d5d0403bb4015520fcfabc88d0cbc4d117\config-1.3.3.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.scala-lang.modules\scala-java8-compat_2.11\0.7.0\a31b1b36bcf0d53657733b5d40c78d5f090a5dea\scala-java8-compat_2.11-0.7.0.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\org.reactivestreams\reactive-streams\1.0.2\323964c36556eb0e6209f65c1cef72b53b461ab8\reactive-streams-1.0.2.jar;C:\Users\13720\.gradle\caches\modules-2\files-2.1\com.twitter\chill-java\0.7.6\a74b8d019d6941d82ff86d1612e382f4a0ae8259\chill-java-0.7.6.jar flink_kafka
SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
SLF4J: Defaulting to no-operation (NOP) logger implementation
SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
11> hello world
11> h 1
11> h 1
11> h 2
11> h 3
11> h
11> h4
11> h 6
11> hello
11> hello world