Spark基础知识系列一（从hdfs获取文件）

Spark获取文件

spark可以从本地，hdfs文件系统，s3，及hadoop支持的文件系统获取文件。从本地：file:///usr/local/xx.txt.

import org.apache.spark.{SparkConf, SparkContext}

/**
  * Created by Damon on 3/29/17.
  */
object HelloScala {
  def main(args: Array[String]): Unit = {
    val logFile ="hdfs://localhost:9000/user/spark/word.txt" // Should be some file on your system
    val conf = new SparkConf().setAppName("Simple Application")
    val sc = new SparkContext(conf)
    val logData = sc.textFile(logFile, 2).cache()
    val numAs = logData.filter(line => line.contains("a")).count()
    val numBs = logData.filter(line => line.contains("b")).count()
    println("Lines with a: %s, Lines with b: %s".format(numAs, numBs))
  }
}