01.聚类分析-Kmeans的简单实现

Kmeans实例实现代码

软件环境：scala2.10.4+spark1.6.3

import org.apache.log4j.{Level, Logger}
import org.apache.spark.mllib.clustering.{KMeans, KMeansModel}
import org.apache.spark.mllib.linalg.Vectors
import org.apache.spark.{SparkConf, SparkContext}

object MyTest {

  def main(args:Array[String]): Unit ={

    // 屏蔽不必要的日志显示在终端上

    Logger.getLogger("org.apache.spark").setLevel(Level.WARN)
    Logger.getLogger("org.eclipse.jetty.server").setLevel(Level.OFF)

    val conf = new SparkConf().setMaster("local[2]").setAppName("Kmeans test...")
    val sc = new SparkContext(conf)

    //装载数据集
    val mydata = sc.textFile("kmeans_data.txt",1)
    val testdata = sc.textFile("kmeans_data_test.txt",1)
    val parsedData = mydata.map(s => Vectors.dense(s.split(' ').map(_.toDouble)))

    //将数据集聚类，2个类，20次迭代，进行模型训练形成数据模型
    val numClusters = 2
    val numIterations = 20
    val model = KMeans.train(parsedData, numClusters, numIterations)


    // 打印数据模型的中心点
    println("Cluster centers:")
    for (c <- model.clusterCenters) {
      println("  " + c.toString)
    }

    // 使用误差平方之和来评估数据模型
    val cost = model.computeCost(parsedData)
    println("Within Set Sum of Squared Errors = " + cost)

    //使用模型测试单点数据
    println("Vectors 0.2 0.2 0.2 is belongs to clusters:" + model.predict(Vectors.dense("0.2 0.2 0.2".split(' ').map(_.toDouble))))
    println("Vectors 0.25 0.25 0.25 is belongs to clusters:" + model.predict(Vectors.dense("0.25 0.25 0.25".split(' ').map(_.toDouble))))
    println("Vectors 8 8 8 is belongs to clusters:" + model.predict(Vectors.dense("8 8 8".split(' ').map(_.toDouble))))


    // 交叉评估1，只返回结果
    val mytestdata = mydata.map(s => Vectors.dense(s.split(' ').map(_.toDouble)))
    val result1 = model.predict(mytestdata)
    result1.saveAsTextFile("result_kmeans1")

    //模型保存
    model.save(sc,"mymodle.model")

    //调度保存好的模型
    val modeluse = KMeansModel.load(sc,"mymodle.model")

    // 交叉评估2，返回数据集和结果  调用保存的模型
    val result2 = testdata.map {
      line =>
        val linevectore = Vectors.dense(line.split(' ').map(_.toDouble))
        val prediction = modeluse.predict(linevectore)
        line + " " + prediction
    }.saveAsTextFile("result_kmeans2")
    sc.stop()
  }
}

小结：可以利用Kmeans对会员进行分类管理

          组织会员的N个维度数据，进行模型训练，后续新增会员可以按此训练模型进行分类

代码来源：https://www.cnblogs.com/shishanyuan/p/4747778.html