2019-11-10

FLINK DATASTREAM API学习： https://www.bilibili.com/video/av47970985?from=search&seid=7970250823204628517\

dataStream.global()    上游所有实例发送给下游的第一个实例

dataStream.broadcast()  上游每个实例发送给下游的所有实例

dataStream.forward()   上下游实例个数一样（即并行度一样）时，一对一发送

dataStream.recale()     同一节点本地轮询，上游实例发送给下游的实例

dataStream.rebalance()  Round-Robin，轮询分配，跨节点轮询

dataStream.shuffle()   随机均匀分配

dataStream.partitioncustom()  自定义单播

TUPLE和case class不支持NULL，ROW支持NULL

---

运行中报错：找不到KryoException

解决办法：添加依赖  groupId com.esotericsoftware.kryo  artifactId com.esotericsoftware.kryo 至于version用IDE提示的

---

package myflink;

import org.apache.flink.api.common.functions.FoldFunction;

import org.apache.flink.api.java.functions.KeySelector;

import org.apache.flink.api.java.tuple.Tuple;

import org.apache.flink.api.java.tuple.Tuple2;

import org.apache.flink.streaming.api.datastream.DataStream;

import org.apache.flink.streaming.api.datastream.KeyedStream;

import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;

import org.apache.flink.streaming.api.functions.sink.SinkFunction;

import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;

import java.util.HashMap;

import java.util.Map;

import java.util.Random;

public class GroupedProcessingTimeWindowExample {

private static class DataSourceextends RichParallelSourceFunction>{

private volatile boolean running =true;

        public void run(SourceContext> sourceContext)throws Exception {

Random random =new Random(System.currentTimeMillis());

            while (running){

Thread.sleep((getRuntimeContext().getIndexOfThisSubtask()+1)*5000+500);

                String key ="类别"+(char)('A'+random.nextInt(3));

                int value = random.nextInt(10)+1;

                System.out.println(String.format("Emit:\t%s , %d",key,value));

                sourceContext.collect(new Tuple2(key,value));

            }

}

public void cancel() {

running =false;

        }

public static void main(String[] args)  {

System.out.println(1);

            StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

            env.setParallelism(2);

            DataStream> dataStream=env.addSource(new DataSource());

            KeyedStream, Tuple> keyedStream = dataStream.keyBy(0);

            keyedStream.sum(1).keyBy(new KeySelector, Object>() {

public ObjectgetKey(Tuple2 stringIntegerTuple2)throws Exception {

return "";

                }

}).fold(new HashMap(), new FoldFunction, Map>() {

public Mapfold(Map stringIntegerMap, Tuple2 o)throws Exception {

stringIntegerMap.put(o.f0,o.f1);

                    return stringIntegerMap;

                }

}).addSink(new SinkFunction>() {

public void invoke(Map value, Context context)throws Exception {

System.out.println(value.values().stream().mapToInt(v->v).sum());

                }

});

            try {

env.execute("GroupedProcessingTimeWindowExample");

            }catch (Exception e) {

e.printStackTrace();

            }

}

}

}