Maxwell,Kafka, Spark Streaming and Hive

为什么80%的码农都做不了架构师？>>>   

 run maxwell

./maxwell --user='maxwell' --password='maxwell' --host=hdp1 --producer=kafka --kafka.bootstrap.servers=hdp3:6667 --kafka_topic=maxwell

 run kafka 

/usr/hdp/2.5.0.0-1245/kafka/bin/kafka-console-consumer.sh --zookeeper hdp1:2181,hdp2:2181,hdp3:2181 --topic maxwell --from-beginning

run pyspark

kafkaStream = KafkaUtils.createStream(streamingContext, [ZK quorum], [consumer group id], [per-topic number of Kafka partitions to consume])

from pyspark import SparkContext
from pyspark.streaming import StreamingContext
from pyspark.streaming.kafka import KafkaUtils
import json
import time
from pyspark.sql import Row,HiveContext
from pyspark import SparkConf
def process(rdd):
    try:
            rowRdd = rdd.map(lambda w: Row(word=w))
            wordsDataFrame = sqlContext.createDataFrame(rowRdd)
            # Creates a temporary view using the DataFrame.
            wordsDataFrame.registerTempTable("tmp")
            partition=time.strftime('%Y-%m-%d',time.localtime(time.time()))
            sqlscript="insert overwrite table  kafka_flume_hdfs_tt partition(ods_date='%s') select * from tmp"%partition
            wordCountsDataFrame = sqlContext.sql(sqlscript)
    except:
            pass
def start():
    sconf=SparkConf()  
    sconf.set('spark.cores.max',3)  
    sc=SparkContext(appName='streaming',conf=sconf)
    global sqlContext
    sqlContext=HiveContext(sc)
    ssc = StreamingContext(sc, 10)
    kafkaStream = KafkaUtils.createStream(ssc, 'hdp1:2181', 'spark-streaming', {'maxwell':1})
    parsed = kafkaStream.map(lambda v: json.loads(v[1]))
    value=parsed.map(lambda x:x['data']['id'])
    value.pprint()
    value.foreachRDD(process)
    ssc.start()
    ssc.awaitTermination()

if __name__ == '__main__':
    start()

Download the JAR of the artifact from Maven Central http://search.maven.org/,Group Id = org.apache.spark, Artifact Id = spark-streaming-kafka-assembly, Version = 1.6.2.Then, include the jar in the pyspark command as

pyspark --jars spark-streaming-kafka-assembly_2.10-1.6.1.jar --python_file.py

 

转载于:https://my.oschina.net/aubao/blog/2239923