由于最近毕设需要设计一个小功能,一个日志分析并转换合并放到kafka上的一个需求,今天来总结积记录一下思路与代码实现。
首先先明确业务流程,我们需要:
OK,明确业务需求后我们开始上编码部分,大家也可以根据自己的业务来参考。
pom文件内容:
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0modelVersion>
<groupId>com.huyue.flinkgroupId>
<artifactId>FlinkConsumerAOIartifactId>
<version>0.0.1-SNAPSHOTversion>
<packaging>jarpackaging>
<name>FlinkConsumerAOIname>
<url>http://maven.apache.orgurl>
<properties>
<project.build.sourceEncoding>UTF-8project.build.sourceEncoding>
properties>
<dependencies>
<dependency>
<groupId>com.alibabagroupId>
<artifactId>fastjsonartifactId>
<version>1.2.56version>
dependency>
<dependency>
<groupId>org.redissongroupId>
<artifactId>redissonartifactId>
<version>3.11.6version>
dependency>
<dependency>
<groupId>org.apache.poigroupId>
<artifactId>poiartifactId>
<version>4.0.1version>
dependency>
<dependency>
<groupId>org.apache.poigroupId>
<artifactId>poi-ooxmlartifactId>
<version>4.0.1version>
dependency>
<dependency>
<groupId>org.apache.poigroupId>
<artifactId>poi-ooxml-schemasartifactId>
<version>4.0.1version>
dependency>
<dependency>
<groupId>c3p0groupId>
<artifactId>c3p0artifactId>
<version>0.9.0.4version>
dependency>
<dependency>
<groupId>com.zaxxergroupId>
<artifactId>HikariCPartifactId>
<version>3.1.0version>
dependency>
<dependency>
<groupId>net.sourceforge.javacsvgroupId>
<artifactId>javacsvartifactId>
<version>2.0version>
dependency>
<dependency>
<groupId>org.postgresqlgroupId>
<artifactId>postgresqlartifactId>
<version>42.2.5version>
dependency>
<dependency>
<groupId>org.apache.kafkagroupId>
<artifactId>kafka-clientsartifactId>
<version>2.2.0version>
dependency>
<dependency>
<groupId>org.apache.flinkgroupId>
<artifactId>flink-clients_2.12artifactId>
<version>1.11.1version>
dependency>
<dependency>
<groupId>org.apache.flinkgroupId>
<artifactId>flink-streaming-java_2.12artifactId>
<version>1.11.1version>
dependency>
<dependency>
<groupId>org.apache.flinkgroupId>
<artifactId>flink-javaartifactId>
<version>1.11.1version>
dependency>
<dependency>
<groupId>org.apache.flinkgroupId>
<artifactId>flink-connector-kafka_2.12artifactId>
<version>1.11.1version>
dependency>
<dependency>
<groupId>commons-logginggroupId>
<artifactId>commons-loggingartifactId>
<version>1.1.1version>
dependency>
<dependency>
<groupId>commons-cligroupId>
<artifactId>commons-cliartifactId>
<version>1.4version>
dependency>
<dependency>
<groupId>org.slf4jgroupId>
<artifactId>slf4j-apiartifactId>
<version>1.8.0-beta0version>
dependency>
<dependency>
<groupId>org.slf4jgroupId>
<artifactId>slf4j-log4j12artifactId>
<version>1.8.0-beta0version>
dependency>
<dependency>
<groupId>log4jgroupId>
<artifactId>log4jartifactId>
<version>1.2.17version>
dependency>
<dependency>
<groupId>junitgroupId>
<artifactId>junitartifactId>
<version>3.8.1version>
<scope>testscope>
dependency>
dependencies>
project>
这里为了方便看到Flink的分析过程日志,我们引入log4j的jar包,并且需要将log4j.properties
文件放到项目的src目录下,
# log4j.properties文件
# Root logger option
log4j.rootLogger=info, stdout
# Direct log messages to stdout
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.Target=System.out
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
主函数部分,主要是kafka配置、定义数据流、调用分析算子以及合流sink到kafka;
/**
* @throws Exception
* @Author: Hu.Yue
* @Title: main
* @Description: TODO
* @param @param args
* @return void
* @throws
*/
public static void main(String[] args) throws Exception {
//分析数据流1
String topic = "inTopic1";
//分析数据流2
String topic2 = "inTopic2";
//合并数据流
String outTopic = "outTopic";
//创建环境
StreamExecutionEnvironment environment = StreamExecutionEnvironment.getExecutionEnvironment();
//配置恢复点
environment.enableCheckpointing(10000);
environment.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);
Properties props = new Properties();
//kafka集群地址
props.setProperty("bootstrap.servers", "10.144.3.155:9092,10.144.5.233:9092,10.144.4.54:9092");
//认证(如果没有此配置可以忽略)
props.put("sasl.jaas.config", "org.apache.kafka.common.security.scram.ScramLoginModule required username='xxx' password='xxx';");
props.put("security.protocol", "SASL_PLAINTEXT");
props.put("sasl.mechanism", "PLAIN");
//分组定义
props.setProperty("group.id", "java_group1");
props.setProperty("kafka.consumer.auto.offset.reset", "enableAutoCommit");
//消费序列化
props.setProperty("key.deserializer", StringDeserializer.class.getName());
props.setProperty("value.deserializer", StringDeserializer.class.getName());
//生产序列化
props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
//参数配置
props.put("acks", "all");
props.put("retries", 3);
props.put("batch.size", 65536);
props.put("linger.ms", 1);
props.put("buffer.memory", 33554432);
props.put("max.request.size", 10485760);
//定义String类型的两个消费者
FlinkKafkaConsumer<String> consumerBoard = new FlinkKafkaConsumer<String>(topic, new SimpleStringSchema(), props);
FlinkKafkaConsumer<String> consumerFovComp = new FlinkKafkaConsumer<String>(topic2, new SimpleStringSchema(), props);
//设置消费原则
consumerBoard.setStartFromEarliest();
consumerFovComp.setStartFromEarliest();
//定义数据流,调用方法在下面介绍
DataStream<String> dataStream_board = ConversionBoard(environment,consumerBoard);
DataStream<String> dataStream_dovComp = ConversionFovComp(environment,consumerFovComp);
//合流操作,并RichSinkFunction自定义上传的key与value
DataStream<String> unionStream = dataStream_board.union(dataStream_dovComp);
unionStream.addSink(new RichSinkFunction<String>() {
private static final long serialVersionUID = 1L;
KafkaProducer<String, String> producer = null;
@Override
public void open(Configuration parameters) {
producer = new KafkaProducer<String, String>(props);
}
@Override
public void invoke(String value) throws Exception {
if(null != value) {
String key = String.format("%d", (int) (Math.random() * 6));
ProducerRecord<String,String> producerRecord = new ProducerRecord<String,String>(outTopic, key, value);
producer.send(producerRecord);
producer.flush();
}
}
});
//控制台打印数据流信息
unionStream.print();
//提交操作
environment.execute("union DataStream");
}
俩条数据流的格式的不同的,算子需要同通过某种逻辑规则将数据转化成相同的后进行合流
/**
* @Author: Hu.Yue
* @Title: ConversionBoard
* @Description: board-pass
* @param @param environment
* @param @param consumer
* @param @return
* @param @throws Exception
* @return DataStream>
* @throws
*/
public static DataStream<String> ConversionBoard(StreamExecutionEnvironment environment, FlinkKafkaConsumer<String> consumer) throws Exception{
DataStream<String> dataStream = environment.addSource(consumer)
//对数据进行包装,使一条数据流包含可以分组的key值和便于统计的一个count值
.flatMap(new FlatMapFunction<String, Tuple3<String,String,Integer>>() {
private static final long serialVersionUID = 1L;
public void flatMap(String value, Collector<Tuple3<String, String, Integer>> out) throws Exception {
ObjectMapper oMapper = new ObjectMapper();
JsonNode node = oMapper.readTree(value);
//这里我们使用其中的4个字段作为key,每一个数据后面带一个1,
out.collect(new Tuple3<String, String, Integer>(
node.get("xxx").asText()+ node.get("yyy").asText() + node.get("zzz").asText() + node.get("bbb").asText(),
value,
1));
}
})
//通过刚才拼接的key来进行分组
.keyBy(0)
//设置会话时间来等这一组数据全部到齐,这里的三秒是测试时使用的,真实的生产环境可以拉长时间来多等一会
.window(ProcessingTimeSessionWindows.withGap(Time.seconds(3)))
//聚合函数,对相同的key值数据流进行count操作并合并成一条数据输出
.reduce(new ReduceFunction<Tuple3<String,String,Integer>>() {
private static final long serialVersionUID = 1L;
public Tuple3<String, String, Integer> reduce(Tuple3<String, String, Integer> value1,
Tuple3<String, String, Integer> value2) throws Exception {
return new Tuple3<String, String, Integer>(value2.f0,value2.f1,value2.f2+value1.f2);
}
})
//过滤不符合我们预期的数据
.filter(new FilterFunction<Tuple3<String,String,Integer>>() {
private static final long serialVersionUID = 1L;
public boolean filter(Tuple3<String, String, Integer> value) throws Exception {
ObjectMapper oMapper = new ObjectMapper();
JsonNode node = oMapper.readTree(value.f1);
if((node.get("boardSN").asText().length() == 10) &&
(!node.get("idStation").asText().startsWith("L")) &&
("\"P \"".equals(node.get("status").asText().trim()) || "\"Y \"".equals(node.get("status").asText().trim())) &&
(node.get("modifierDate").asText() != null) &&
(node.get("bBoardsn").asText() == null || "".equals(node.get("bBoardsn").asText()) || node.get("bBoardsn").asText() != node.get("boardSN").asText())) {
return true;
}
return false;
}
})
//对数据进行符合结构转化
.flatMap(new FlatMapFunction<Tuple3<String,String,Integer>, String>() {
private static final long serialVersionUID = 1L;
@Override
public void flatMap(Tuple3<String, String, Integer> value, Collector<String> out) throws Exception {
ObjectMapper mapper = new ObjectMapper();
JsonNode node = mapper.readTree(value.f1);
String mcbSno = node.get("boardSN").asText().length() > 32 ? node.get("boardSN").asText().toUpperCase().substring(0,32) : node.get("boardSN").asText();
String fdate = node.get("fdate").asText();
String fixNo = node.get("idStation").asText().toUpperCase().substring(0,4);
String cModel = node.get("cModel").asText();
String badgeNo = node.get("modifier").asText().length() > 7 ? node.get("modifier").asText().substring(0,7) : node.get("modifier").asText();
String cycleTime = node.get("cycleTime").asText();
Integer unionQty = 1;
String wc= "";
String pdLine = fixNo.substring(0,3);
String pcbSide = node.get("cModel").asText().toUpperCase().trim().substring(
node.get("cModel").asText().length()-1);
//这里拿到按需(key--value.f1)统计的count值
unionQty = value.f2;
switch (pcbSide) {
case "1":
wc = "05";
break;
case "2":
wc = "0B";
break;
case "3":
wc = "0C";
break;
case "4":
wc = "0D";
break;
case "5":
wc = "PE";
break;
default:
break;
}
if(wc != null && !"".equals(wc)) {
//转化成符合FIS的数据结构
FisResult fis = new FisResult(
mcbSno + "_" + wc + "_" +fdate,
mcbSno,
wc,
1,
pdLine,
"",
"",
badgeNo,
unionQty,
"57",
fdate,
fixNo,
cycleTime,
cModel,
"f3");
//转化成JSON串后被out收录
String result = JSON.toJSONString(fis);
out.collect(result);
}
}
});
//一定记得返回流
return dataStream;
}
下一个数据流分析方法和上面的雷同,只是分析逻辑不同而已,这里考虑篇幅就不介绍了,重要的部分我已经打好备注,大家按需食用即可,有问题欢迎评论区交流。