Spark-core综合练习-IP匹配
ip.txt 部分数据:
220.177.248.0|220.177.255.255|3702650880|3702652927|亚洲|中国|江西|南昌||电信|360100|China|CN|115.892151|28.676493
220.178.0.0|220.178.56.113|3702652928|3702667377|亚洲|中国|安徽|合肥||电信|340100|China|CN|117.283042|31.86119
220.178.56.114|220.178.57.33|3702667378|3702667553|亚洲|中国|安徽|合肥|巢湖|电信|340181|China|CN|117.874155|31.600518
220.178.57.34|220.178.57.34|3702667554|3702667554|亚洲|中国|安徽|合肥|肥东|电信|340122|China|CN|117.47128|31.88525
220.178.57.35|220.178.57.53|3702667555|3702667573|亚洲|中国|安徽|合肥|巢湖|电信|340181|China|CN|117.874155|31.600518
220.178.57.54|220.178.57.54|3702667574|3702667574|亚洲|中国|安徽|合肥|肥东|电信|340122|China|CN|117.47128|31.88525
220.178.57.55|220.178.57.145|3702667575|3702667665|亚洲|中国|安徽|合肥|巢湖|电信|340181|China|CN|117.874155|31.600518
220.178.57.146|220.178.57.146|3702667666|3702667666|亚洲|中国|安徽|合肥|肥东|电信|340122|China|CN|117.47128|31.88525
220.178.57.147|220.178.57.181|3702667667|3702667701|亚洲|中国|安徽|合肥|巢湖|电信|340181|China|CN|117.874155|31.600518
220.178.57.182|220.178.57.184|3702667702|3702667704|亚洲|中国|安徽|合肥|肥东|电信|340122|China|CN|117.47128|31.88525
220.178.57.185|220.178.57.189|3702667705|3702667709|亚洲|中国|安徽|合肥|巢湖|电信|340181|China|CN|117.874155|31.600518
220.178.57.190|220.178.57.190|3702667710|3702667710|亚洲|中国|安徽|合肥|肥东|电信|340122|China|CN|117.47128|31.88525
220.178.57.191|220.178.62.41|3702667711|3702668841|亚洲|中国|安徽|合肥|巢湖|电信|340181|China|CN|117.874155|31.600518
220.178.62.42|220.178.62.42|3702668842|3702668842|亚洲|中国|安徽|合肥|肥西|电信|340123|China|CN|117.16845|31.72143
220.178.62.43|220.178.62.65|3702668843|3702668865|亚洲|中国|安徽|合肥||电信|340100|China|CN|117.283042|31.86119
220.178.62.66|220.178.62.66|3702668866|3702668866|亚洲|中国|安徽|合肥|肥西|电信|340123|China|CN|117.16845|31.72143
220.178.62.67|220.178.62.193|3702668867|3702668993|亚洲|中国|安徽|合肥||电信|340100|China|CN|117.283042|31.86119
220.178.62.194|220.178.62.194|3702668994|3702668994|亚洲|中国|安徽|合肥|肥西|电信|340123|China|CN|117.16845|31.72143
220.178.62.195|220.178.76.41|3702668995|3702672425|亚洲|中国|安徽|合肥||电信|340100|China|CN|117.283042|31.86119
220.178.76.42|220.178.76.42|3702672426|3702672426|亚洲|中国|安徽|合肥|长丰|电信|340121|China|CN|117.16549|32.47959
220.178.76.43|220.178.76.133|3702672427|3702672517|亚洲|中国|安徽|合肥||电信|340100|China|CN|117.283042|31.86119
220.178.76.134|220.178.76.134|3702672518|3702672518|亚洲|中国|安徽|合肥|长丰|电信|340121|China|CN|117.16549|32.47959
220.178.76.135|220.178.77.49|3702672519|3702672689|亚洲|中国|安徽|合肥||电信|340100|China|CN|117.283042|31.86119
目标文件部分数据:20090121000132.394251.http.format
20090121000132124542000|117.101.215.133|www.jiayuan.com|/19245971|Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; TencentTraveler 4.0)|http://photo.jiayuan.com/index.php?uidhash=d1c3b69e9b8355a5204474c749fb76ef|__tkist=0; myloc=50%7C5008; myage=2009; PROFILE=14469674%3A%E8%8B%A6%E6%B6%A9%E5%92%96%E5%95%A1%3Am%3Aphotos2.love21cn.com%2F45%2F1b%2F388111afac8195cc5d91ea286cdd%3A1%3A%3Ahttp%3A%2F%2Fimages.love21cn.com%2Fw4%2Fglobal%2Fi%2Fhykj_m.jpg; last_login_time=1232454068; SESSION_HASH=8176b100a84c9a095315f916d7fcbcf10021e3af; RAW_HASH=008a1bc48ff9ebafa3d5b4815edd04e9e7978050; COMMON_HASH=45388111afac8195cc5d91ea286cdd1b; pop_1232093956=1232468896968; pop_time=1232466715734; pop_1232245908=1232469069390; pop_1219903726=1232477601937; LOVESESSID=98b54794575bf547ea4b55e07efa2e9e; main_search:14469674=%7C%7C%7C00; registeruid=14469674; REG_URL_COOKIE=http%3A%2F%2Fphoto.jiayuan.com%2Fshowphoto.php%3Fuid_hash%3D0319bc5e33ba35755c30a9d88aaf46dc%26total%3D6%26p%3D5; click_count=0%2C3363619
20090121000132406516000|117.101.222.68|gg.xiaonei.com|/view.jsp?p=389|Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; CIBA)|http://home.xiaonei.com/Home.do?id=229670724|_r01_=1; __utma=204579609.31669176.1231940225.1232462740.1232467011.145; __utmz=204579609.1231940225.1.1.utmccn=(direct)
20090121000132581311000|115.120.36.118|tj.tt98.com|/tj.htm|Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; TheWorld)|http://www.tt98.com/|
20090121000132864647000|123.197.64.247|cul.sohu.com|/20071227/n254338813_22.shtml|Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; TheWorld)|http://cul.sohu.com/20071227/n254338813_22.shtml|ArticleTab=visit:1; IPLOC=unknown; SUV=0901080709152121; vjuids=832dd37a1.11ebbc5d590.0.b20f858f14e918; club_chat_ircnick=JaabvxC4aaacQ; spanel=%7B%22u%22%3A%22%22%7D; vjlast=1232467312,1232467312,30
20090121000133296729000|222.55.57.176|down.chinaz.com|/|Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; iCafeMedia; TencentTraveler 4.0)||cnzz_a33219=0; vw33219=%3A18167791%3A; sin33219=http%3A//www.itxls.com/wz/wyfx/it.html; rtime=0; ltime=1232464387281; cnzz_eid=6264952-1232464379-http%3A//www.itxls.com/wz/wyfx/it.html
20090121000133331104000|123.197.66.93|www.pkwutai.cn|/down/downLoad-id-45383.html|Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 1.7)|http://www.baidu.com/s?tn=b1ank_pg&ie=gb2312&bs=%C3%C0%C6%BC%B7%FE%D7%B0%B9%DC%C0%ED%C8%ED%BC%FE&sr=&z=&cl=3&f=8&wd=%C6%C6%BD%E2%C3%C0%C6%BC%B7%FE%D7%B0%B9%DC%C0%ED%C8%ED%BC%FE&ct=0|
20090121000133446262000|115.120.12.157|v.ifeng.com|/live/|Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1) ; .NET CLR 2.0.50727; CIBA)|http://www.ifeng.com/|userid=1232466610953_4339; location=186; sclocationid=10002; vjuids=22644b162.11ef4bc1624.0.63ad06717b426; vjlast=1232466614,1232467297,13
20090121000133456256000|115.120.7.240|cqbbs.soufun.com|/3110502342~-1~2118/23004348_23004348.htm|Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1) ; .NET CLR 2.0.50727; CIBA)||new_historysignlist=%u534E%u6DA6%u4E8C%u5341%u56DB%u57CE%7Chttp%3A//cqbbs.soufun.com/board/3110502342/%7C%7C%u9A8F%u9038%u7B2C%u4E00%u6C5F%u5CB8%7Chttp%3A//cqbbs.soufun.com/board/3110169184/%7C%7C%u793E%u533A%u4E4B%u661F%7Chttp%3A//cqbbs.soufun.com/board/sqzx/%7C%7C; SoufunSessionID=2y5xyr45kslc0zbdooqnoo55; viewUser=1; vjuids=-870e9088.11ee89aba57.0.be9c3d988def8; vjlast=1232263101,1232380806,11; new_viewtype=1; articlecolor=#000000; usersms_pop_type=1; articlecount=186; __utma=101868291.755195653.1232450942.1232450942.1232450942.1; __utmz=101868291.1232450942.1.1.utmccn=(referral)
20090121000133586141000|117.101.219.241|12.zgwow.com|/launcher/index.htm|Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)||
需求:
- 根据ip地址查询进行统计
- 展示结果样式: ((经度,纬度,所在城市),次数)
参考答案:
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
import scala.collection.mutable
object IPDemoWithScala {
def main(args: Array[String]): Unit = {
val sparkConf: SparkConf = new SparkConf().setAppName("IPDemoWithScala").setMaster("local[4]")
val sc = new SparkContext(sparkConf)
sc.setLogLevel("WARN")
val lines1: RDD[String] = sc.textFile("D:\\workplace\\testFile\\ip\\20090121000132.394251.http.format")
val lines2: RDD[String] = sc.textFile("D:\\workplace\\testFile\\ip\\ip.txt")
//125.213.100.123
val ips1Rdd: RDD[String] = lines1.map(x => x.split("\\|")).map(x => x(1))
//1.0.1.0|1.0.3.255|16777472|16778239|亚洲|中国|福建|福州||电信|350100|China|CN|119.306239|26.075302
val ips2Rdd: RDD[Array[String]] = lines2.map(x => x.split("\\|"))
//ipArray={125,213,100,123}
val ipTransfer: RDD[Long] = ips1Rdd.map(x => {
val ipArray: Array[String] = x.split("[.]")
var ipNum = 0L
for (i <- ipArray) {
ipNum = i.toLong | ipNum << 8L
}
ipNum
})
val ipTransfer2: Array[(Long, Int)] = ipTransfer.map(x => (x, 1)).reduceByKey(_ + _).collect
//ips2Transfer=((16777472,16778239),((119.306239,26.075302,重庆),1))
val ips2Transfer: RDD[((Long, Long), ((String, String,String), Int))] = ips2Rdd.map(x => {
val start = x(2).toLong
val end = x(3).toLong
val city = x(7)
val longitude = x(13) //经度
val latitude = x(14) //纬度
((start, end), ((longitude, latitude,city), 1))
})
var hashMap = new mutable.HashMap[(String, String,String), Int]()
val ipList: List[((Long, Long), ((String, String,String), Int))] = ips2Transfer.toLocalIterator.toList
for (elem <- ipList) {
for (one <- ipTransfer2) {
if (one._1 >= elem._1._1 && one._1 <= elem._1._2) {
if (hashMap.contains(elem._2._1)) {
hashMap(elem._2._1) += one._2
} else {
hashMap.put(elem._2._1, one._2)
}
}
}
}
val result: List[((String, String,String), Int)] = hashMap.toList.sortBy(_._2).reverse
result.foreach(x => println(x))
sc.stop()
}
}
运行结果:
方法二:(通过Spark SQL)
import org.apache.spark.SparkContext
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{DataFrame, SparkSession}
object IPLocationWithSparkSQL {
def main(args: Array[String]): Unit = {
val spark = SparkSession.builder()
.appName("HiveSupportWithScala")
.master("local")
.enableHiveSupport()
.getOrCreate()
val sc: SparkContext = spark.sparkContext
sc.setLogLevel("WARN")
//3:读取本地文件,加载数据
val lines: RDD[String] = sc.textFile("D:/workplace/testFile/ip/ip.txt")
val lines2: RDD[String] = sc.textFile("D:/workplace/testFile/ip/20090121000132.394251.http.format")
//4:将每一行数据进行切分,并转换成People对象
val rulesRDD: RDD[IpLocation] = lines.map(line => line.split("[|]")).map(attr => IpLocation(attr(2).toLong, attr(3).toLong, attr(13), attr(14), attr(7)))
val ipTransfer: RDD[Long] = lines2.map(line => line.split("[|]")).map(x => {
val ipArray: Array[String] = x(1).split("[.]")
var ipNum = 0L
for (i <- ipArray) {
ipNum = i.toLong | ipNum << 8L
}
ipNum
})
import spark.implicits._
val rulesDF: DataFrame = rulesRDD.toDF()
val ipTransferDF: DataFrame = ipTransfer.map(x => TempIp(x)).toDF()
rulesDF.createOrReplaceTempView("iplocation")
ipTransferDF.createOrReplaceTempView("tempIp")
//查询
val sql: String = "select longitude, latitude,city, count(1) as number from tempIp left join iplocation on ip_transfer >= start_ip and ip_transfer <= end_ip group by longitude, latitude,city order by number desc"
spark.sql(sql).show()
sc.stop()
}
}
case class IpLocation(start_ip: Long, end_ip: Long, longitude: String, latitude: String, city: String)
case class TempIp(ip_transfer: Long)
运行结果:
方法三(利用二分法):
object IPDemoWithBinary {
def main(args: Array[String]): Unit = {
val sparkConf: SparkConf = new SparkConf().setAppName("IPDemoWithBinary").setMaster("local[4]")
val sc = new SparkContext(sparkConf)
sc.setLogLevel("WARN")
val lines1: RDD[String] = sc.textFile("D:\\workplace\\testFile\\ip\\20090121000132.394251.http.format")
val lines2: RDD[String] = sc.textFile("D:\\workplace\\testFile\\ip\\ip.txt")
//125.213.100.123
val ips1Rdd: RDD[String] = lines1.map(x => x.split("\\|")).map(x => x(1))
//1.0.1.0|1.0.3.255|16777472|16778239|亚洲|中国|福建|福州||电信|350100|China|CN|119.306239|26.075302
val ips2Rdd: RDD[Array[String]] = lines2.map(x => x.split("\\|"))
//ipArray={125,213,100,123}
val ipTransfer: RDD[Long] = ips1Rdd.map(x => {
val ipArray: Array[String] = x.split("[.]")
var ipNum = 0L
for (i <- ipArray) {
ipNum = i.toLong | ipNum << 8L
}
ipNum
})
val ipTransfer2: Array[(Long, Int)] = ipTransfer.map(x => (x, 1)).reduceByKey(_ + _).collect
//ips2Transfer=((16777472,16778239),((119.306239,26.075302,重庆),1))
val rulesRDD: RDD[((Long, Long), ((String, String, String), Int))] = ips2Rdd.map(x => {
val start = x(2).toLong
val end = x(3).toLong
val city = x(7)
val longitude = x(13) //经度
val latitude = x(14) //纬度
((start, end), ((longitude, latitude, city), 1))
})
var hashMap = new mutable.HashMap[(String, String, String), Int]()
val rules: List[((Long, Long), ((String, String, String), Int))] = rulesRDD.toLocalIterator.toList
//二分法查找
for (elem <- ipTransfer2) {
var low = 0
var high = rules.length - 1
while (low <= high) {
val middle = (low + high) / 2
if ((elem._1 >= rules(middle)._1._1) && (elem._1 <= rules(middle)._1._2))
if (hashMap.contains(rules(middle)._2._1)) {
hashMap(rules(middle)._2._1) += elem._2
} else {
hashMap.put(rules(middle)._2._1, elem._2)
}
if (elem._1 < rules(middle)._1._1)
high = middle - 1
else {
low = middle + 1
}
}
}
val result: List[((String, String, String), Int)] = hashMap.toList.sortBy(_._2).reverse
result.foreach(x => println(x))
println("size --> " + ipTransfer.collect.length)
// sc.parallelize(result).saveAsTextFile("C:\\Users\\ST\\Desktop\\aaa")
sc.stop()
}
}