零售户logistics回归分类

package tobacco

import data.copy._
import org.apache.spark.mllib.classification.{LogisticRegressionWithLBFGS, LogisticRegressionWithSGD}
import org.apache.spark.mllib.clustering.KMeans
import org.apache.spark.mllib.evaluation.MulticlassMetrics
import org.apache.spark.mllib.feature.StandardScaler
import org.apache.spark.mllib.linalg.Vectors
import org.apache.spark.mllib.regression.LabeledPoint
import org.apache.spark.sql.{Row, SQLContext}
import org.apache.spark.{SparkContext, SparkConf}
import tobacco.KM_classic._

/** * Created by xcheng on 4/14/16. */
object RetailerLogistic {
  def main(args: Array[String]) {
    val conf: SparkConf = new SparkConf()
    conf.setAppName("店铺分类").setMaster("local[4]")
    val sparkContext: SparkContext = new SparkContext(conf)
    sparkContext.setLogLevel("OFF")
    val sqlContext: SQLContext = new SQLContext(sparkContext)

    val SalePath="/Users/xcheng/Desktop/实验室/销售库存/saledata/*"
    val StorePath ="/Users/xcheng/Desktop/实验室/销售库存/storedata/*"
    val RetailerPath="/Users/xcheng/360云盘/销售库存/零售户2013.csv"

    val rawRetailer =sparkContext.textFile(RetailerPath)
    val parsedRetailer= rawRetailer.filter(!isColumnNameLine(_,"卷烟编码")).map(line=>Retailerparser(line))
    sqlContext.createDataFrame(parsedRetailer).toDF("DAY","ORG_CODE","ORG_NAME","TOBACCO_CODE","TOBACCO_NAME" , "wholesalePrice", "CUS_CODE", "CUS_NAME",
      "ADDRESS", "STATUS", "RetailStatu", "PAYKIND" , "CUSFLOW"
      , "MARKETTYPE", "NEED", "SHENHE", "DINGGOU", "PRICETYPE", "TAR_CONT", "AMOUNT", "MINUTE").registerTempTable("Retailertable")
    sqlContext.sql("select CUS_NAME,SUM(DINGGOU)NUM,SUM(AMOUNT)TOTALAMOUNT,STATUS,RetailStatu,PAYKIND,CUSFLOW,MARKETTYPE FROM Retailertable group by CUS_NAME,STATUS,RetailStatu,PAYKIND,CUSFLOW,MARKETTYPE").registerTempTable("Retaildata")
    val RETAILDATA = sqlContext.sql("select CUS_NAME,NUM,TOTALAMOUNT,TOTALAMOUNT/NUM AVGPRICE,STATUS,RetailStatu,PAYKIND,CUSFLOW,MARKETTYPE FROM Retaildata")
    RETAILDATA.registerTempTable("RETAILTABLE")

    val NumAmount = sqlContext.sql("select CUS_NAME,NUM,TOTALAMOUNT FROM RETAILTABLE")
    NumAmount.cache()
    val trainningData =  NumAmount.map{line =>Vectors.dense(line(1).toString.toDouble,line(2).toString.toDouble)}
    val scaler = new StandardScaler(
      withMean = true,withStd = true
    ).fit(trainningData)
    val scaledVectors =trainningData.map(v => scaler.transform(v))
    scaledVectors.cache()
    val numClusters = 3
    val model = KMeans.train(scaledVectors,numClusters,maxIterations = 100,runs=5)



    val KMresult = NumAmount.map{
      case Row(cus_name,num,totalamount)=>
        val features =Array[Double](num.toString.toDouble,totalamount.toString.toDouble)
        val line= Vectors.dense(features)
        val scaledline=scaler.transform(line)
        val prediction = model.predict(scaledline)
        (cus_name.toString,prediction.toInt)
    }
    sqlContext.createDataFrame(KMresult).toDF("CUS_NAME","CLASS").registerTempTable("KMRESULT")
    val labeledData =  sqlContext.sql("select a.CUS_NAME,AVGPRICE,STATUS,RetailStatu,PAYKIND,CUSFLOW,MARKETTYPE,CLASS FROM RETAILTABLE a JOIN KMRESULT b on a.CUS_NAME=b.CUS_NAME where AVGPRICE is not null and STATUS is not null and RetailStatu is not null and PAYKIND is not null and CUSFLOW is not null and MARKETTYPE is not null and CLASS is not null")
    labeledData.registerTempTable("labledData")
    labeledData.cache()



    val categories = labeledData.map(r => r(2)).distinct.collect.zipWithIndex.toMap
    // categories: scala.collection.immutable.Map[String,Int] = Map("weather" -> 0, "sports" -> 6,
    //  "unknown" -> 4, "computer_internet" -> 12, "?" -> 11, "culture_politics" -> 3, "religion" -> 8,
    // "recreation" -> 2, "arts_entertainment" -> 9, "health" -> 5, "law_crime" -> 10, "gaming" -> 13,
    // "business" -> 1, "science_technology" -> 7)
    val PAYKIND = labeledData.map(r =>r(4)).distinct.collect.zipWithIndex.toMap
    val CUS_FLOW = labeledData.map(r =>r(5)).distinct.collect.zipWithIndex.toMap
    val MARKETTYPE = labeledData.map(r =>r(6)).distinct.collect.zipWithIndex.toMap



    val numCategories = categories.size
    val numPAYKIND = PAYKIND.size
    val numCUS_FLOW = CUS_FLOW.size
    val numMARKETTYPE = MARKETTYPE.size
    // numCategories: Int = 14
    val dataCategories = labeledData.map { r =>
      val label = r(r.length-1)
      val categoryIdx = categories(r(2))
      val PAYKINDIdx = PAYKIND(r(4))
      val CUS_FLOWIdx = CUS_FLOW(r(5))
      val MARKETTYPEIdx = MARKETTYPE(r(6))
      val categoryFeatures = Array.ofDim[Double](numCategories)
      val PAYKINDFeatures = Array.ofDim[Double](numPAYKIND)
      val CUS_FLOWFeatures = Array.ofDim[Double](numCUS_FLOW)
      val MARKETTYPEFeatures = Array.ofDim[Double](numMARKETTYPE)
      categoryFeatures(categoryIdx) = 1.0
      PAYKINDFeatures(PAYKINDIdx) = 1.0
      CUS_FLOWFeatures(CUS_FLOWIdx) = 1.0
      MARKETTYPEFeatures(MARKETTYPEIdx) = 1.0
      //val otherFeatures = trimmed.slice(4, r.size - 1).map(d => if (d == "?") 0.0 else d.toDouble)
      val features = categoryFeatures ++ Array(r(1).toString.toDouble)++PAYKINDFeatures++CUS_FLOWFeatures++MARKETTYPEFeatures
      LabeledPoint(label.toString.toDouble, Vectors.dense(features))
    }
      //val otherFeatures = trimmed.slice(4, r.size - 1).map(d => if (d == "?") 0.0 else d.toDouble)
      val dataSplit = dataCategories.randomSplit(Array(0.9, 0.1))
      val trainingSet = dataSplit(0)
      val testSet = dataSplit(1)
      //特征标准化
      val lrscaler = new StandardScaler(withMean = true, withStd = true).fit(trainingSet.map(point => point.features))
      val scaledTrainingSet = trainingSet.map(point => new LabeledPoint(point.label, lrscaler.transform(point.features))).cache()
      val scaledTestSet = testSet.map(point => new LabeledPoint(point.label, lrscaler.transform(point.features))).cache()




    //(2)逻辑回归
//    val lrModel = LogisticRegressionWithSGD.train(scaledTrainingSet, 10)
//    val predictrueData = scaledTrainingSet.map { point =>
//      if (lrModel.predict(point.features) == point.label) 1 else 0
//    }.sum()
//    val accuracy = predictrueData / dataCategories.count()
//    println(accuracy)
//    println("lrModel.weights=" + lrModel.weights)



    print(scaledTrainingSet.first)
    //(2.0,[-0.0022527894130305775,-0.21447132919853112,-0.17092073688601872,-0.0022527894130305775,-0.013328831916059465,-0.21146360970498967,-0.0022527894130305775,-0.09756625736643412,-0.165659008653667,0.42255229570556735,0.0,-0.0022527894130305775,-0.9275331049294797,-0.0039019655249826813,-0.14009792135887922,-0.05470879170042826,0.2187088097790089,-0.0022527894130305775,-0.04627271644974014,-0.014249301507933554,-0.1466602367013409,-0.15009001696405935,0.27377366715506074,-0.22374441625440286,-0.00450561312575215,-0.00450561312575215,-1.2109792016515482,1.2110301933430998])Exception in thread "main" org.apache.spark.SparkException: Input validation failed.
    val lrLBFGS = new LogisticRegressionWithLBFGS().setNumClasses(3).run(scaledTrainingSet)
    // Compute raw scores on the test set.
    val predictionAndLabels = scaledTestSet.map { case LabeledPoint(label, features) => val prediction = lrLBFGS.predict(features)
      (prediction, label)
    }
    // Get evaluation metrics.
    val metrics = new MulticlassMetrics(predictionAndLabels)
    val precision = metrics.precision
    println("Precision = " + precision)
    println("lrLBFGS.weigths = " + lrLBFGS.weights)
    //




  }

}

结果

(1.0,[-0.0022533670036951068,-0.21476182093642426,-0.17123307560662127,-0.0022533670036951068,-0.013332249873372223,-0.212918211319947,-0.0022533670036951068,-0.09825217519207344,-0.16528163372426338,0.423864243096115,-0.0022533670036951068,-0.0022533670036951068,-0.7776173920866379,-0.004506768333470466,-0.1399272134234487,-0.05500219036734359,0.21939122402955208,-0.0022533670036951068,-0.04633966075970369,-0.014252955597143787,-0.147634994035387,-0.1499524526798466,0.27398837289555317,-0.2240797107674812,-0.005038732969011382,-0.005038732969011382,0.827348486528708,-0.8273049501091364])Precision = 0.867596784625335 lrLBFGS.weigths = [0.001406115894925403,0.1674730975043896,0.36768700000021876,9.334615787612015E-4,-30.27360788305117,0.35288363292480884,-0.004226727652558982,0.060740401547047955,0.1660487031417234,0.5663075527045792,-0.003694370758759241,0.011427044853290975,-0.37909370439716494,0.022135898055399793,0.41804673039931906,0.1698053220819357,0.7443937793940905,9.334615787612015E-4,0.15527777792823452,-21.47537041160153,0.5277017404822946,-0.06765901433310849,0.022991755731273374,0.01898029456571178,0.0026142243164660782,0.0026142243164660782,-0.03813352118181489,0.03810707808664586,0.0013905371335426124,-0.0746228251217224,0.07493671269365446,-0.0012642356441561577,-0.004469981102787315,0.02869308101622886,-0.006526377963565716,-0.03773932497062627,-0.024135131455214218,0.012867799064257703,-0.005495989292389678,0.012200985558700082,-0.2096471790936368,0.005647509027088287,-0.031737168394927955,-0.007405318869251818,0.013365966173929225,-0.0012642356441561577,-0.0032041804134094465,-0.010527382865951061,0.015500169787882545,-0.03577460025651247,0.012019586174357347,0.01022918565969636,1.36365661444462E-4,1.36365661444462E-4,-0.03301430575873032,0.033013232226988376]