# -*- coding: utf-8 -*-
from pyflink.datastream import StreamExecutionEnvironment
from pyflink.datastream.functions import MapFunction, RuntimeContext, KeyedProcessFunction
from abc import ABC, abstractmethod
from pyflink.datastream import StreamExecutionEnvironment
from pyflink.datastream.functions import MapFunction, RuntimeContext, KeyedProcessFunction
from pyflink.datastream.state import MapStateDescriptor
from pyflink.datastream.connectors.kafka import FlinkKafkaConsumer
from pyflink.common.typeinfo import Types, TypeInformation
from pyflink.datastream.connectors.elasticsearch import Elasticsearch7SinkBuilder, ElasticsearchEmitter, FlushBackoffType
from pyflink.datastream.connectors import DeliveryGuarantee
from pyflink.common.serialization import SimpleStringSchema
import json
import re
from datetime import datetime
from elasticsearch import Elasticsearch
import re
import redis
# 创建 StreamExecutionEnvironment 对象
env = StreamExecutionEnvironment.get_execution_environment()
env.set_parallelism(1)
# 读取文件,创建 DataStream 对象
data_stream = env.read_text_file('/root/pyflink/elink_test.txt')
# 对每行数据添加字符串 'aaaa'
def to_dict(self):
return {
"bus_seq": self.bus_seq,
"message": self.message,
}
class MyMapFunction(MapFunction):
def open(self, runtime_context: RuntimeContext):
pool = redis.ConnectionPool(host='127.0.0.1',port=6379,max_connections=50)
self.r = redis.Redis(connection_pool=pool)
def close(self):
self.r.close()
def map(self,line):
process_id='';
bus_seq=''
if not line.startswith("ES"):
return
if '
pat=re.compile(r"
bus_seq=pat.search(line).group(1)
#bus_seq=pat.findall(line)
process_id=line.split()[1]
self.r.set(process_id,bus_seq)
process_id=line.split()[1]
if not len(process_id)==6 :
process_id=line.split()[2]
bus_seq=self.r.get(process_id).decode('UTF-8')
if not bus_seq:
return
#self.r.delete(process_id)
#log_event = LogEvent(bus_seq.decode('UTF-8'),line)
LogEvent={}
datetime.now().strftime("%Y-%m-%d")
index_name='flink-test'+date_str
if len(process_id)>0:
#LogEvent['bus_seq']=bus_seq.decode('UTF-8')
LogEvent['bus_seq']=bus_seq
LogEvent['message']=line
LogEvent['index_name']=index_name
else:
return
#print(type(LogEvent))
return LogEvent
class EsSink(MapFunction):
def open(self, runtime_context: RuntimeContext):
self.es = Elasticsearch("http://127.0.0.1:9200")
def close(self):
pass
def map(self,LogEvent):
try:
data = {
"@timestamp": datetime.now().strftime( "%Y-%m-%dT%H:%M:%S.000+0800" ),
"content" : LogEvent.message,
"bus_seq" : LogEvent.bus_seq
}
except:
return
self.es.index( index="flink_test", document=data )
env.add_jars("file:///root/lib/flink-sql-connector-elasticsearch7-3.0.1-1.16.jar")
date_str = datetime.now().strftime("%Y-%m-%d")
es7_sink = Elasticsearch7SinkBuilder() \
.set_bulk_flush_max_actions(1) \
.set_emitter(ElasticsearchEmitter.static_index('flink-test2023-06-07')) \
.set_hosts(['127.0.0.1:9200']) \
.build()
#new_stream = data_stream.map(MyMapFunction()).sink_to(es7_sink)
new_stream = data_stream.map(MyMapFunction(),output_type=Types.MAP(Types.STRING(),Types.STRING())).sink_to(es7_sink)
#new_stream = data_stream.map(MyMapFunction(),output_type=Types.MAP(Types.STRING(),Types.STRING()))
#new_stream = data_stream.map(MyMapFunction(),output_type=Types.MAP(Types.STRING(),Types.STRING()))
# 输出到控制台
#new_stream.print()
# 执行任务
env.execute('Add "bus_seq" to each line')