Spark笔记(4)：计算dataframe中两列的相关系数

spark

---

皮尔森、斯皮尔曼(pearson spearman)计算相关系数

import org.apache.spark.mllib.stat.Statistics

val df1 = sql("select new_rank_level,old_rank_level from ad_tmp.xxx")
val df_real = df1.select("old_rank_level","new_rank_level")
val rdd_real = df_real.rdd.map(x=>(x(0).toString.toDouble ,x(1).toString.toDouble ))
val label = rdd_real.map(x=>x._1.toDouble )
val feature = rdd_real.map(x=>x._2.toDouble )
 
val cor_pearson:Double = Statistics.corr(label, feature, "pearson")
println(cor_pearson)
0.23997483383749665 
 
val cor_spearman:Double = Statistics.corr(label, feature, "spearman")
println(cor_spearman)
cor_spearman: Double = 0.23997567905723607