参考书籍：《spark最佳实践》
实验步骤：
1.Python程序生成访问日志
2.通过脚本将日志自动上传至HDFS
3.spark streaming程序监控HDFS目录，自动处理新的文件。
log.py文件代码：
import random
import time
class WebLogGeneration(object):
def __init__(self):
self.user_agent_dist={
0.0:"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0)",
0.1:"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0)",
0.2:"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0;.NET CLR 2.0.50727)",
0.3:"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.1.4322)",
0.4:"Mozilla/5.0 (Windows NT 6.1; Trident/7.0; rv:11.0) like Gecko",
0.5:"Mozilla/5.0 (Windows NT 6.1; WOW64;rv:41.0) Gecko/20100101 Firefox/41.0",
0.6:"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.1.4322)",
0.7:"Mozilla/5.0 (iPhone; CPU iPhone OS 7_0_3 like Mac OS X) AppleWebKit/537.51.1 (KHTML,like Gecko) Version/7.0 Mobile/11B511 Safari/9537.53",
0.8:"Mozilla/5.0 (Linux; Android 4.2.1; Galaxy Nexus Build/JOP40D) AppleWebKit/535.19 (KHTML,like Gecko) Chrome/18.0.1025.166 Mobile Safari/535.19",
0.9:"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML,like Gecko) Chrome/45.0.2454.85 Safari/537.36",
1:" "
}
self.ip_slice_list = [10,29,30,46,55,63,72,87,98,132,156,124,1677,143,187,168,190,201,202,214,215,222]
self.url_path_list = ["login.php","view.php","list.php","upload.php","admin/login.php","edit.php","index.html"]
self.http_refer = [
"http://www.baidu.com/s?wd={query}",
"http://www.google.cn/search?q={query}",
"http://www.sogou.com/web?query={query}",
"http://www.yahoo.com/s?p={query}",
"http://cn.bing.com/search?q={query}"
]
self.search_keyword = ["spark","hadoop","hive","spark mlib","spark sql"]
def sample_ip(self):
slice = random.sample(self.ip_slice_list, 4)
return ".".join(str(item) for item in slice)
def sample_url(self):
return random.sample(self.url_path_list,1)[0]
def sample_user_agent(self):
dist_uppon = random.uniform(0, 1)
return self.user_agent_dist[float('%0.1f' % dist_uppon)]
def sample_refer(self):
if random.uniform(0, 1) > 0.2:
return "-"
refer_str = random.sample(self.http_refer, 1)
query_str = random.sample(self.search_keyword, 1)
return refer_str[0].format(query=query_str)
def sample_one_log(self,count = 3):
time_str = time.strftime("%Y-%m-%d %H:%M:%S",time.localtime())
while count > 1:
query_log = "{ip} - - [{local_time}] \"GET /{url} HTTP/1.1\" 200 0 \"{refer}\" \"{user_agent}\" \"-\"".format(ip=self.sample_ip(),local_time=time_str,url=self.sample_url(),refer=self.sample_refer(),user_agent=self.sample_user_agent())
print query_log
count = count - 1
if __name__ == "__main__":
web_log_gene = WebLogGeneration()
web_log_gene.sample_one_log(random.uniform(1000,2000))
run.sh脚本代码：
#!/bin/bash
streaming_dir="/spark/streaming"
while [ 1 ];do
python log.py >> test.log
tmplog="access.`date +'%s'`.log"
hadoop fs -put test.log ${streaming_dir}/tmp/$tmplog
hadoop fs -mv ${streaming_dir}/tmp/$tmplog ${streaming_dir}/
echo "`date +"%F %T"` put $tmplog to HDFS succeed"
sleep 1
done
SparkStreamingTest.scala文件代码：
package spark.example
import org.apache.spark.SparkConf
import org.apache.spark.Streaming.{Seconds,streamingContext}
object SparkStreamingTest{
def main(args:Array[String]){
val batch = 10
val conf = new SparkConf().setAppName("NginxAnay")
val ssc = new streamingContext(conf,Seconds(batch))
val lines = ssc.textFileStream("hdfs://7master:9000/spark/streaming")
// 1.总PV
lines.count().print()
// 2.各IP的PV，按PV倒叙
// 空格分隔的第一个字段就是IP
lines.map(line =>{(line.split(" ")(0), 1)}).reduceByKey(_ + _).transform(rdd =>{
rdd.map(ip_pv => (ip_pv._2, ip_pv._1)).sortByKey(false).map(ip_pv => (ip_pv._2,ip_pv._1))
}).print()
// 3.搜索引擎PV
// 先输出搜索引擎和查询关键词，避免统计搜索关键词时重复计算
// 输出(host, query_keys)
val refer = lines.map(_.split("\"")(3))
val searchEnginInfo = refer.map(r =>{
val f = r.split('/')
val searchEngines = Map(
"www.google.cn" -> "q",
"www.yahoo.com" -> "p",
"cn.bing.com" -> "q",
"www.baidu.com" -> "wd",
"www.sogou.com" -> "query"
)
if(f.length > 2){
val host = f(2)
if(searchEngines.contains(host)){
val query = r.split('?')(1)
if(query.length > 0){
val arr_search_q = query.split('&').filter(_.indexOf(searchEngines(host)+"=") == 0)
if(arr_search_q.length > 0){
(host,arr_search_q(0).split('=')(1))
}else{
(host, "")
}
}else{
(host, "")
}
}else{
("", "")
}
}else{
("", "")
}
})
// 输出搜索引擎PV
searchEnginInfo.filter(_._1.length > 0).map(p => {(p._1, 1)}).reduceByKey(_ + _).print()
// 4.关键词PV
searchEnginInfo.filter(_._2.length > 0).map(p => {(p._2, 1)}).reduceByKey(_ + _).print()
// 5.终端类型PV
lines.map(_.split("\"")(5)).map(agent =>{
val types = Seq("iPhone", "Android")
var r = "Default"
for (t <- types){
if(agent.indexOf(t) != -1)
r = t
}
(r, 1)
}).reduceByKey(_ + _).print()
// 6.各页面PV
lines.map(line => {
(line.split("\"")(1).split(" ")(1), 1)
}).reduceByKey(_ + _).print()
// 启动计算，等待执行结束
ssc.start()
ssc.awaitTermination()
}
}
run_SparkStreamingTest.sh文件代码：
/usr/local/src/spark-1.6.0-bin-hadoop2.6/bin/spark-submit --master local[2] \
--class spark.example.SparkStreamingTest target/scala-2.11/wordcount_2.11-1.6.0.jar