Spark with Hadoop InputFormat

基于Yarn的，使用新的API，SBT需要添加，默认是用的1.0.4的client

libraryDependencies += "org.apache.hadoop" % "hadoop-client" % "2.2.0"

代码如下

package myclass

import org.apache.spark.SparkContext
import org.apache.hadoop.io.{Text, LongWritable, IntWritable}
import org.apache.hadoop.mapreduce.lib.input.{TextInputFormat, KeyValueTextInputFormat}

/**
 * Created by jack on 2/27/14.
 */
object MyHadoopFile {
	def main(args: Array[String]) {
		val spark = new SparkContext("local","my hadoop file",System.getenv("SPARK_HOME"),SparkContext.jarOfClass(this.getClass))
		val file = spark.newAPIHadoopFile[LongWritable, Text, TextInputFormat]("hdfs://127.0.0.1:9000/user/jack/data/pagerank_data.txt")
		file.foreach(println)
		System.exit(0)
	}
}

结果

(0,1 2)
(4,1 3)
(8,1 4)
(12,2 1)
(16,3 1)
(20,4 1)