Flink实时当日净销售额、毛利额、消费会员数

1. 需求

我们想统计：
1.当日净销售额
2.当日毛利额
3.当日消费会员数（去重）
4.后期map操作关联redis 进行拉宽，例如毛利率，前台毛利率，同比，环比等

2. 逻辑

当我拿到需求的时候，这个是需要keyby 和 state的，当初想的是采用redis存储当日用户，后来感觉不好，flink内部自带的就有mapstate，故采用mapstate，来进行存储。窗口大小，我们设定为event_time的一天，
因为时区问题，我们的offset 控制到中国的时区，因为我们的开的窗口很大，一天的滚动窗口，那么我们不可能等待一天来拿到数据，这样还谈什么实时，flink内部提供了trigger机制，这个机制的作用就是可以把窗口中的中间数据拿出来，但是我们每来一条数据，那么就需要全量窗口中的数据进行计算，这样数据量可能比较大，那么我们有没有可能，只计算我们没计算的数据呢，那么此时就用到了我们的驱逐器，驱逐器设置为，只要处理过的就不处理，然后在我们的计算函数中，我们应该统计我们的数据，其实我们完全可以将我们的state当做我们的缓存来进行操作，来一些数据我怎么做，无非就是增删改查，其实软件开发，就是这样的，
和flink的设计思想很类似，数据-> 转换->应用

      StreamExecutionEnvironment bsEnv = StreamExecutionEnvironment.getExecutionEnvironment();

        BasicKafkaConfig kakfaConfig = KafkaUtil.getKakfaConfig("ods_stock_sales_realtime");

        logger.info("[OfflineStockItemLaunch] The KafkaConfig obtained is {}" , kakfaConfig);

        FlinkKafkaConsumer010<StockSaleMdDetail> consumer = new FlinkKafkaConsumer010<StockSaleMdDetail>(kakfaConfig.getTopic()
                , new StockSaleMdDetailSchema(), kakfaConfig.getKafkaProps());
        consumer.setStartFromEarliest();
        bsEnv.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        CheckpointConfig checkpointConfig = bsEnv.getCheckpointConfig();
        bsEnv.enableCheckpointing(5000L);
        checkpointConfig.setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE);
        checkpointConfig.setMaxConcurrentCheckpoints(1);
        checkpointConfig.setCheckpointTimeout(100000L);
        checkpointConfig.setFailOnCheckpointingErrors(true);
        checkpointConfig.enableExternalizedCheckpoints(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);
        FsStateBackend fsStateBackend = new FsStateBackend("hdfs://cdh1:8020/flink/checkpoints/");
        bsEnv.setStateBackend(fsStateBackend);
        logger.info("[OnlineSalesAnalysisLaunch] Start consuming data in kafka topic is {}" ,"ods_stock_sales_realtime");

        DataStreamSource<StockSaleMdDetail> datas = bsEnv.addSource(consumer);
        SingleOutputStreamOperator<StockSaleMdDetail> eventTimeDatasStream = datas.assignTimestampsAndWatermarks(new BoundedOutOfOrdernessTimestampExtractor<StockSaleMdDetail>(Time.seconds(0)) {
            @Override
            public long extractTimestamp(StockSaleMdDetail element) {
                return element.getSoldTime();
            }
        });

        SingleOutputStreamOperator<StockSaleMdDetail> filterSoldModeStream = eventTimeDatasStream.filter(new FilterFunction<StockSaleMdDetail>() {
            @Override
            public boolean filter(StockSaleMdDetail stockSaleMdDetail) throws Exception {
                HashSet<String> sold_modes = new HashSet<>();
                sold_modes.add("CXS");
                sold_modes.add("MDZT");
                sold_modes.add("WD");
                sold_modes.add("SS");
                sold_modes.add("MDFH");
                stockSaleMdDetail.getSoldmMode();
                return sold_modes.contains(stockSaleMdDetail.getSoldmMode()) && (stockSaleMdDetail.getOnlineCity().equals("xxx") || stockSaleMdDetail.getOnlineCity().equals("xxxxx"));
            }
        }).map(new MapFunction<StockSaleMdDetail, StockSaleMdDetail>() {
            @Override
            public StockSaleMdDetail map(StockSaleMdDetail stockSaleMdDetail) throws Exception {
                SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");
                String date = simpleDateFormat.format(new Date(stockSaleMdDetail.getSoldTime()));
                stockSaleMdDetail.setSold_day(date);
                return stockSaleMdDetail;
            }
        });
        KeyedStream<StockSaleMdDetail, Tuple> soldmModeKeyedStream = filterSoldModeStream.keyBy("onlineCity", "sold_day", "soldmMode");

        WindowedStream<StockSaleMdDetail, Tuple, TimeWindow> trigger =
                soldmModeKeyedStream.window(TumblingEventTimeWindows.of(Time.days(1), Time.hours(-8)))
                        .trigger(ContinuousEventTimeTrigger.of(Time.seconds(1)));

        trigger.evictor(TimeEvictor.of(Time.seconds(0), true))

                .process(new ProcessWindowFunction<StockSaleMdDetail, OnlineSoldModeAnalysisData, Tuple, TimeWindow>() {
                    MapState<Integer, String> uv;
                    ValueState<Double> netSale;
                    ValueState<Double> grossProfit;
                    ValueState<Double> promotionGrossProfit;

                    @Override
                    public void open(Configuration parameters) throws Exception {

                        MapStateDescriptor<Integer, String> mapStateDescriptor = new MapStateDescriptor<Integer, String>("OnlineSalesAnalysisLaunch-soldmode-uv-mapstate", Integer.class, String.class);
                        uv = getRuntimeContext().getMapState(mapStateDescriptor);

                        ValueStateDescriptor<Double> netSaleDescriptor = new ValueStateDescriptor<Double>("OnlineSalesAnalysisLaunch-soldmode-netsale-valuestate", Double.class);
                        netSale = getRuntimeContext().getState(netSaleDescriptor);

                        ValueStateDescriptor<Double> grossProfitDescriptor = new ValueStateDescriptor<Double>("OnlineSalesAnalysisLaunch-soldmode-grossProfit-valuestate", Double.class);
                        grossProfit = getRuntimeContext().getState(grossProfitDescriptor);

                        ValueStateDescriptor<Double> promotionGrossProfitDescriptor = new ValueStateDescriptor<Double>("OnlineSalesAnalysisLaunch-soldmode-promotionGrossProfit-valuestate", Double.class);
                        promotionGrossProfit = getRuntimeContext().getState(promotionGrossProfitDescriptor);
                    }

                    @Override
                    public void process(Tuple tuple, Context context, Iterable<StockSaleMdDetail> elements, Collector<OnlineSoldModeAnalysisData> out) throws Exception {

                        Iterator<StockSaleMdDetail> iterator = elements.iterator();
                        String day = "";
                        Double netsal = 0.0;
                        Double gross = 0.0;
                        Double promotion = 0.0;
                        String onlineMode = null;
                        String ctgy = null;
                        while (iterator.hasNext()) {
                            StockSaleMdDetail next = iterator.next();
                            netsal += next.getNetSale().doubleValue();
                            gross += next.getGrossProfit().doubleValue();
                            promotion += next.getVndr_money().doubleValue();
                            uv.put(next.getUserId(), null);
                            day = next.getSold_day();
                            onlineMode = next.getSoldmMode();
                            ctgy = next.getOnlineCity();
                        }

                        Double historynetSale = netSale.value();
                        Double historynetGross = grossProfit.value();
                        Double historynetPromotion = promotionGrossProfit.value();


                        if (historynetSale == null || historynetGross == null || historynetPromotion == null) {
                            historynetSale = 0.0;
                            historynetGross = 0.0;
                            historynetPromotion = 0.0;
                        }
                        netSale.update(historynetSale + netsal);
                        grossProfit.update(historynetGross + gross);
                        promotionGrossProfit.update(historynetPromotion + promotion);
                        Integer users = 0;

                        Iterator<Integer> iterator1 = uv.keys().iterator();

                        while (iterator1.hasNext()) {
                            users += 1;
                            iterator1.next();
                        }
                        OnlineSoldModeAnalysisData onlineSoldModeAnalysisData = new OnlineSoldModeAnalysisData();
                        onlineSoldModeAnalysisData.date = day;
                        onlineSoldModeAnalysisData.netSale = historynetSale + netsal;
                        onlineSoldModeAnalysisData.grossProfit = historynetGross + gross;
                        onlineSoldModeAnalysisData.promotionGrossProfit = historynetPromotion + promotion;
                        onlineSoldModeAnalysisData.onlineMode = ctgy;
                        onlineSoldModeAnalysisData.soldMode = onlineMode;
                        onlineSoldModeAnalysisData.uv = users;
                        out.collect(onlineSoldModeAnalysisData);
                    }

                }).print();