Spark机器学习的两种调包方式

1.MLlib

 

#数据类型为 LabeledPoint
#rdd -> LabeledPoint
#LabeledPoint(y值,特征值)
#y值为Dobule型
#特征值为Vectors 为spark数据类型

#导入 LabeledPoint包，Vectors包
labeledpoint = RDD.map(lambda x:(x[0],Vectors.dense(x[1:]))

#导入算法包
#训练和预测
model = 算法.train(labeledpoint)
model.predict(test_labeledpoint_features)

 

2.ML

#数据类型 dataframe dataset

#dataframe -> dataset

from pyspark.sql import SparkSession
from pyspark.ml.feature import StringIndexer
from pyspark.ml import Pipeline
from pyspark.ml.feature import VectorAssembler

#lst_col是一个list 为特征名称
vecAssembler = VectorAssembler(inputCols=lst_col, outputCol="features")
stringIndexer = StringIndexer(inputCol="y", outputCol="label")

pipeline = Pipeline(stages=[vecAssembler, stringIndexer])
pipelineFit = pipeline.fit(data)
dataset = pipelineFit.transform(data)

#训练和预测
model = 算法.fit(trainingData)
lrModel.transform(testData)