pyflink 流处理 自定义函数

[root@master pyflink]# cat elink_test.txt
ES047 625978 uuuuuuuuuuuuuuuuuuuuuuuuuuuuuuu
ES047 625978 094323 __________sys_app.c:78 [1573]10402020230515094322018855B0520230515094323ES047SJGY9999999992570100xxxx13000.00TL010015xxx3000.000SXF0020.00IBANK.PWD-SM4.zpk15601402391000068test00.00 ES047 625978 11111111111111111
ES047 625978 22222222222222222
ES047 625978 33333333333333333
ES047 625978 44444444444444444
ES047 625978 55555555555555555
6666666666666 66666666666666666
ES047 625978 77777777777777777
ES047 625978 88888888888888888
ES047 625978 99999999999999999


[root@master pyflink]# cat test.py 
# -*- coding: utf-8 -*-
from pyflink.datastream import StreamExecutionEnvironment
from pyflink.datastream.functions import  MapFunction, RuntimeContext, KeyedProcessFunction
from abc import ABC, abstractmethod
from pyflink.datastream import StreamExecutionEnvironment
from pyflink.datastream.functions import  MapFunction, RuntimeContext, KeyedProcessFunction
from pyflink.datastream.state import MapStateDescriptor
from pyflink.datastream.connectors.kafka import FlinkKafkaConsumer
from pyflink.common.typeinfo import Types, TypeInformation
from pyflink.datastream.connectors.elasticsearch import Elasticsearch7SinkBuilder, ElasticsearchEmitter, FlushBackoffType
from pyflink.datastream.connectors import DeliveryGuarantee
from pyflink.common.serialization import SimpleStringSchema
import json
import re
from datetime import datetime
from elasticsearch import Elasticsearch
from pyflink.datastream.functions import RuntimeContext, FlatMapFunction

import re
import redis


# 创建 StreamExecutionEnvironment 对象
env = StreamExecutionEnvironment.get_execution_environment()
env.set_parallelism(1)

# 读取文件,创建 DataStream 对象
data_stream = env.read_text_file('/root/pyflink/elink_test.txt')
def my_map_func(value):
    return  value.split(' ')[1]
new_stream = data_stream.map(my_map_func)
#new_stream = data_stream.map(MyMapFunction(),output_type=Types.MAP(Types.STRING(),Types.STRING()))
#new_stream = data_stream.map(MyMapFunction(),output_type=Types.MAP(Types.STRING(),Types.STRING()))
# 输出到控制台
new_stream.print()

# 执行任务
env.execute('Add "bus_seq" to each line')

[root@master pyflink]# python test.py 
625978
625978
625978
625978
625978
625978
625978
66666666666666666
625978
625978
625978
[root@master pyflink]# 

你可能感兴趣的:(Flink实时计算,数学建模,java,spring)