日志的分析
package hadoop
import java.security.MessageDigest
import java.text.SimpleDateFormat
import IPInfo.IP
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{Dataset, SQLContext, SparkSession}
import org.apache.spark.{SparkConf, SparkContext}
import java.util.Properties
import com.sun.deploy.net.URLEncoder
/**
* Created by sicong on 2017/1/13.
*/
object YiTianYiKeAnalyser {
val prop = new Properties()
prop.put("user", "root")
prop.put("password", "")
case class Record(user: String, ip: String, country: String, province: String, city: String,
restime: Int, time: Long, code: Int, size: Long,
firm: String, device: String, rom: String, ke: String)
case class Devices(city: String,num:Int,device: String)
case class Ipmap(ip: String, provinceCode: Int, cityCode: Int, province: String, city: String)
case class CityFlow(city:String,flow:Long)
case class StatusCode(code:Int,num:Int)
case class ThreadCache(dateParser: SimpleDateFormat, sha1Digester: MessageDigest)
val threadSafeCache = new ThreadLocal[ThreadCache]();
val Iphelpk = new IP()
def getIpInfohga(Str: String): String = {
Iphelpk.mains(Str)
}
def logbegin(): Unit = {
val spark = SparkSession
.builder()
.appName("Spark SQL Example").master("local[4]")
.config("spark.some.config.option", "some-value")
.getOrCreate()
// readsp(spark)
parseLog("/Users/sicong/Downloads/yitianyike.txt", spark)
}
def getNeedParseLog(): Array[String] = {
// TODO
// 1. 当前时间前推 7 小时;2 当前时间前推 12 小时;
// 以 1、2 为时间范围,查询日志列表
// 日志列表与近期的处理记录比对,若获得的日志为处理,则解析日志,成功后标记为已处理
Array("/Users/Simon/Downloads/7xna64.com2.z0.glb.qiniucdn.com_2017-01-03-23_0602")
}
def logdevicesMysql(kk: Dataset[Devices], s: String):Unit={
val prop = new Properties()
prop.put("user", "root")
prop.put("password", "")
kk.write.mode("append").jdbc("jdbc:mysql://localhost:3306/test1?useUnicode=true&characterEncoding=utf8", s,prop)
}
def CizeFlowStatus(logrdd: RDD[Record],sparkSession: SparkSession) ={
import sparkSession.implicits._
val dataOfFlow=logrdd.map(x=>(x.city,x.size)).groupByKey().map(x=>(x._1,(x._2.sum.toDouble/(1024)).round))
val logMysqldata=dataOfFlow.flatMap(x=>
Seq(CityFlow(x._1,x._2))).toDS()
logMysqldata.write.mode("append").jdbc("jdbc:mysql://localhost:3306/test1?useUnicode=true&characterEncoding=utf8", "test1.CityFlow",prop)
}
def HttpStatusCode(logrdd:RDD[Record],sparkSession: SparkSession): Unit ={
import sparkSession.implicits._
val logMysqldata= logrdd.map(x=>(x.code,x)).groupByKey().flatMap(x=>{
Seq(StatusCode(x._1,x._2.size))
}).toDS()
logMysqldata.write.mode("append").jdbc("jdbc:mysql://localhost:3306/test1?useUnicode=true&characterEncoding=utf8", "test1.StatusCode",prop)
}
def cityTopURL(logrdd:RDD[Record],sparkSession: SparkSession): Unit ={
import sparkSession.implicits._
logrdd.map(x=>(x.ke+x.city,x)).groupByKey().map(x=>(x._2.size,x._1)).sortBy(x=>x,false,1).foreach(x=>println(x))
}
def parseLog(url: String, spark: SparkSession): Unit = {
import spark.implicits._
val peopleDF = spark.sparkContext
.textFile(url)
val logrdd = peopleDF.flatMap(line => {
val record = parses(line)
if (record != null) {
Seq(record)
} else {
Seq()
}
})
//这里对logrdd进行缓存到内存cache 因为接下来的每个算子action如果没有缓存到内存是会每次重新从头开始计算
logrdd.cache()
//统计个个省份对应的流量的接口
// CizeFlowStatus(logrdd,spark)
//统计个个状态码的占有率
// HttpStatusCode(logrdd,spark)
//cityTopURL(logrdd,spark)
//
// val logrdds=logrdd.map(x=>(x.ip+x.device,x))
// .groupByKey().map(x=>(x._2.head))
// val datacount=logrdds.count()
// val deviceOfNum=logrdds.map(x=>(x.device,x)).groupByKey().map(x=>(x._2.head.device,x._2.head.city,x._2.size))
// val kk= deviceOfNum.flatMap(x=>{
// Seq(Devices(x._2,x._3,x._1))
// }).toDS()
// logdevicesMysql(kk,"test1.Devices")
//filter(x=>x._2.size > 1)
// foreach(x=>println(x))
//.take(1000).foreach(x=>{println(x.device)})
/// .toDS()
// logrdd.foreach(x=>println(x))
// spark.sql("set names utf8")
// val jdbcDF = spark.read
// .format("jdbc")
// .option("url", "jdbc:mysql://localhost:3306/test1?useUnicode=true&characterEncoding=utf8")
// .option("dbtable", "test1.Devices")
// .option("user", "root")
// .option("password", "")
// .load()
//
//怎么样写入数据库
// val prop = new Properties()
// prop.put("user", "root")
// prop.put("password", "")
// kk.write.mode("append").jdbc("jdbc:mysql://localhost:3306/test1?useUnicode=true&characterEncoding=utf8", "test1.Devices",prop)
//val prop = new Properties()
// // prop.put("user", "root")
// // prop.put("password", "")
// // peopleDf.write.mode("append").jdbc("jdbc:mysql://localhost:3306/mytest", "mytest.student3",prop)
// .map(x => (x.ip+x.device, x))
//.groupByKey().filter(x=>x._2.size > 1).map(x=>x._2.head).take(1000).foreach(x=>{println(x.device)})
// .foreach {
// x =>
// println(s"ip:${x._1}")
// x._2.foreach(y => println(y.device))
// println()
// }
// .map(x => (x.ip, x)).groupByKey().map(_._2.head).toDS()
// logrdd.createOrReplaceTempView("fusion")
// spark.sql("select * from fusion").foreach(x => println(x))
// spark.sql("select count(*) from fusion").foreach(x => print(x))
}
def logprovincecity(str: String): Array[String] = {
val Iphelp = new IP();
val data = Iphelp.mains(str)
data.substring(1, data.length - 1).split(",")
}
def parses(line: String): Record = {
setThreadCache()
val as = line.split(" ")
val ip = as(0)
val restime = as(2).toInt
val time = "1403931367000".toLong
val code = as(8).toInt
val size = as(9).toLong
val ua = line.substring(line.lastIndexOf(" \"") + 2, line.lastIndexOf("\""))
val region = logprovincecity(ip)
val province = changeEncodeing(region(0))
val city = changeEncodeing(region(1))
val country = changeEncodeing(region(2))
//val province =region(0)
// val city = region(1)
// val country =region(2)
val driver = parseUa(ua)
val firm = driver._1
val device = driver._2
val rom = driver._3
val user = mixtureUser(ip, ua)
val ke = parseToKey(as(6))
val obj = Record(user, ip, country, province, city,
restime, time, code, size,
firm, device, rom, ke)
obj
}
def changeEncodeing(string: String): String={
string
}
// 106.18.21.156 - 282 [03/Jan/2017:23:30:14 +0800] "GET http://7xna64.com2.z0.glb.qiniucdn.com/Fjm_mLtcPN3DbTtLpywOmX5gq9cl.jpg?imageView2/2/w/1080/h/1920&e=1483545599&token=Q-hCY0VbL4F6NTX3TgRvE_T3vcpNEo2Gr3S9RA-b:ffDUURujc65VJLj1mKdGDMOrhIg= HTTP/1.1" 200 478114 "-" "AndroidDownloadManager/5.1+(Linux;+U;+Android+5.1;+OPPO+R9m+Build/LMY47I)"
// 139.148.121.96 - 248 [03/Jan/2017:23:30:11 +0800] "GET http://7xna64.com2.z0.glb.qiniucdn.com/FiU3bxGjI6PutwVphDQQihBgP0uw.jpg?imageView2/2/w/1080/h/1920&e=1483545599&token=Q-hCY0VbL4F6NTX3TgRvE_T3vcpNEo2Gr3S9RA-b:1wKdyBO_iYMQh7_MBqGcifYQX50= HTTP/1.1" 200 552867 "-" "AndroidDownloadManager/5.1.1+(Linux;+U;+Android+5.1.1;+OPPO+R9+Plusm+A+Build/LMY47V)"
// 220.178.4.219 - 1 [03/Jan/2017:23:30:35 +0800] "GET http://7xna64.com2.z0.glb.qiniucdn.com/FiwmSuSIuu981zLWENSCOJvIoj2P.jpg?imageView2/2/w/1080/h/1920&e=1483592399&token=Q-hCY0VbL4F6NTX3TgRvE_T3vcpNEo2Gr3S9RA-b:vsvEgQcb8-cU3BDLNp6sLCG72DI= HTTP/1.1" 200 456693 "-" "Dalvik/2.1.0+(Linux;+U;+Android+5.1.1;+NX529J+Build/LMY47V)"
// def parse(line: String): Record = {
// println(line)
// try {
// setThreadCache()
//
// val as = line.split(" ")
//
// val ip = as(0)
// val restime = as(2).toInt
// val time = parseToDate(line)
//
// val code = as(8).toInt
// val size = as(9).toLong
// val ua = line.substring(line.lastIndexOf(" \"") + 2, line.lastIndexOf("\""))
// val region = parseRegion(ip)
// val province = region._1
// val city = region._2
//
// val driver = parseUa(ua)
// val firm = driver._1
// val device = driver._2
// val rom = driver._3
//
// val user = mixtureUser(ip, ua)
// val country = ""
// val key = parseToKey(as(6))
//
// val obj = Record(user, ip, country, province, city,
// restime, time, code, size,
// firm, device, rom, key)
// obj
// } catch {
// case e: Exception => {
// println(s"wrong line: ${line}")
// return null
// }
// }
// }
def setThreadCache(): Unit = {
val cache = threadSafeCache.get()
if (cache == null) {
val dateParser = new SimpleDateFormat("dd/MMM/yyyy:HH:mm:ss ZZZZ")
val sha1 = MessageDigest.getInstance("SHA1")
threadSafeCache.set(ThreadCache(dateParser, sha1))
}
}
def parseToDate(line: String): Long = {
threadSafeCache.get().dateParser.parse(line.substring(line.indexOf("[") + 1, line.indexOf("]"))).getTime() / 1000
}
private val Iphelp = new IP();
def getIpInfo(Str: String): String = {
Iphelp.mains(Str)
}
def parseRegion(ip: String) = {
// TODO
// IP.load()
("", "")
}
// AndroidDownloadManager/5.1.1+(Linux;+U;+Android+5.1.1;+OPPO+R9+Plusm+A+Build/LMY47V)
// Dalvik/2.1.0+(Linux;+U;+Android+5.1.1;+NX529J+Build/LMY47V)
// Dalvik/2.1.0+(Linux;+U;+Android+5.1.1;+NX523J_V1+Build/LMY47V)
// Dalvik/2.1.0+(Linux;+U;+Android+6.0.1;+vivo+Y55A+Build/MMB29M)
// AndroidDownloadManager/5.1+(Linux;+U;+Android+5.1;+OPPO+R9m+Build/LMY47I)
// ua 也包含其它字符
// -
// Java/1.7.0_09
// Go-http-client/1.1
// VAYXXLWZIKRFDGFHPOXDNHJTDLTNBTV
// ("Android 6.0.1", "vivo Y55A", "Build/MMB29M")
def parseUa(ua: String): (String, String, String) = {
try {
val t1 = ua.split(";").reverse
val t2 = t1(0).split("\\+")
return (t1(1).replaceAll("\\+", " ").trim, t2.slice(0, t2.length - 1).mkString(" ").trim, t2(t2.length - 1))
} catch {
case e: Exception => {
return ("Error", "Error", "Error")
}
}
}
def mixtureUser(ip: String, ua: String) = {
hash(ip + ":" + ua)
}
def hash(s: String): String = {
threadSafeCache.get().sha1Digester.digest(s.getBytes).map("%02x".format(_)).mkString
}
def parseToKey(url: String) = {
// https://a 至少有 9 个字符
val l = url.indexOf("?", 9);
val end = if (l > 0) l else url.length()
url.substring(url.indexOf("/", 9) + 1, end)
}
def getIpInfos(Str: String): Array[String] = {
// val hell = new hello();
// hell.getipdata(Str).split(";")
return Array()
}
def readsp(spark: SparkSession): Unit ={
import spark.implicits._
val prop = new Properties()
prop.put("user", "root")
prop.put("password", "")
val jdbcDF2 = spark.read
.jdbc("jdbc:mysql://localhost:3306", "test1.tutorials_tbl",prop)
jdbcDF2.foreach(x=>println(x))
}
def main(args: Array[String]): Unit = {
IP.load("/Users/sicong/Downloads/17monipdb/17monipdb.dat");
logbegin()
}
}