kafka stream（kafka流处理）

kafka stream（kafka流处理）

问题：

1）列转化问题(列转化行)

1. user_attendees friend_id

//打开zookeeper
[root@bigdata bin]# zkServer.sh start
//启动kafka
[root@bigdata bin]# ./kafka-server-start.sh ../config/server.properties 
//启动Spark
[root@bigdata ~]# cd /opt/bigdata/spark243/sbin
[root@bigdata sbin]# ./start-all.sh
//查看kafuka下面的文件名
[root@bigdata bin]# ./kafka-topics.sh --zookeeper 127.0.0.1:2181 --list
//还原kafka里面的消息队列数据
[root@bigdata bin]# ./kafka-streams-application-reset.sh --zookeeper 127.0.0.1:2181 --application-id user_friends --input-topics user
_friends

//导入maven中的kafka


    org.apache.kafka
    kafka_2.12
    1.0.0

//配置maven最低为1.8以后才可以用
    
        
            
                org.apache.maven.plugins
                maven-compiler-plugin
                3.8.0
                
                    1.8
                    1.8
                
            
        
    

主程序

package com.mykstream.services;

import org.apache.kafka.common.serialization.Serdes;
import org.apache.kafka.streams.KafkaStreams;
import org.apache.kafka.streams.StreamsBuilder;
import org.apache.kafka.streams.StreamsConfig;
import org.apache.kafka.streams.kstream.ForeachAction;
import org.apache.kafka.streams.kstream.KStream;

import java.util.Properties;

public class MyDemo {
    public static void main(String[] args) {
        Properties props = new Properties();
        props.put(StreamsConfig.APPLICATION_ID_CONFIG,"user_friends");
        props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG,"192.168.56.101:9092");
        props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
        props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG,Serdes.String().getClass());

        StreamsBuilder build = new StreamsBuilder();

        KStream<Object,String> user_friends = build.stream("user_friends");

        ForeachAction<Object, String> faction = new ForeachAction<Object, String>() {
            public void apply(Object o, String s) {
                System.out.println("key"+o+"========>value:"+s);
            }
        };
        user_friends.foreach(faction);
        KafkaStreams ks = new KafkaStreams(build.build(), props);
        ks.start();
    }
}

打胖包

打开虚拟机

//启动spark
[root@bigdata bin]# cd /opt/bigdata/spark243/bin/
//启动  spark-submit   胖包路径  运行主类路径
[root@bigdata bin]# ./spark-submit /opt/mykafkasan.jar --class com.mykstream.services.MyDemo