Java开发Spark程序

Java开发Spark程序_第1张图片

Java开发Spark程序_第2张图片
Java开发Spark程序_第3张图片

pom.xml

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.dt.spark</groupId>
    <artifactId>SparkApps</artifactId>
    <version>0.0.1-SNAPSHOT</version>
    <packaging>jar</packaging>

    <name>SparkApps</name>
    <url>http://maven.apache.org</url>

    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    </properties>

    <dependencies>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>3.8.1</version>
            <scope>test</scope>
        </dependency>

        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_2.10</artifactId>
            <version>1.6.0</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-sql_2.10</artifactId>
            <version>1.6.0</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-hive_2.10</artifactId>
            <version>1.6.0</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-streaming_2.10</artifactId>
            <version>1.6.0</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>2.6.0</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-streaming-kafka_2.10</artifactId>
            <version>1.6.0</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-graphx_2.10</artifactId>
            <version>1.6.0</version>
        </dependency>
    </dependencies>
    <build>
        <sourceDirectory>src/main/java</sourceDirectory>
        <testSourceDirectory>src/main/test</testSourceDirectory>

        <plugins>
            <plugin>
                <artifactId>maven-assembly-plugin</artifactId>
                <configuration>
                    <descriptorRefs>
                        <descriptorRef>jar-with-dependencies</descriptorRef>
                    </descriptorRefs>
                    <archive>
                        <manifest>
                            <mainClass></mainClass>
                        </manifest>
                    </archive>
                </configuration>
                <executions>
                    <execution>
                        <id>make-assembly</id>
                        <phase>package</phase>
                        <goals>
                            <goal>single</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>

            <plugin>
                <groupId>org.codehaus.mojo</groupId>
                <artifactId>exec-maven-plugin</artifactId>
                <version>1.2.1</version>
                <executions>
                    <execution>
                        <goals>
                            <goal>exec</goal>
                        </goals>
                    </execution>
                </executions>
                <configuration>
                    <executable>java</executable>
                    <includeProjectDependencies>true</includeProjectDependencies>
                    <includePluginDependencies>false</includePluginDependencies>
                    <classpathScope>compile</classpathScope>
                    <mainClass>com.dt.spark.App</mainClass>
                </configuration>
            </plugin>

            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <configuration>
                    <source>1.6</source>
                    <target>1.6</target>
                </configuration>
            </plugin>

        </plugins>
    </build>
</project>

本地模式

/** * 文件名:WordCount.java * * 创建人:Sundujing - [email protected] * * 创建时间:2016年5月9日 上午9:53:28 * * 版权所有:Sundujing */
package com.dt.spark.SparkApps.cores;

import java.util.Arrays;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.api.java.function.VoidFunction;
import scala.Tuple2;

/** * [描述信息:说明类的基本功能] * * @author Sundujing - [email protected] * @version 1.0 Created on 2016年5月9日 上午9:53:28 */
public class WordCount {
    public static void main(String[] args)
    {
        /** * 第一步,创建Spark的配置对象SparkConf,设置Spark程序的运行时的配置信息, * 例如说通过setMaster来设置程序要链接的Spark集群的Master的URL, * 如果设置为local,则代表Spark程序在本地运行,特别适合于机器配置较差的情况 */
        SparkConf conf=new SparkConf().setAppName("Spark workcount written by Java").setMaster("local");

        /** * 第二步,创建SparkContext对象 * SparkContext是Spark程序所有功能的唯一入口,无论是采用Scala,java,python,R等都 * 必须有一个SparkContext(不同语言具体类名称不同,如果是Java的话,则为JavaSparkContext) * 同时还会负责Spark程序在Master注册程序等 * SparkContext是整个Spark应用程序至关重要的一个对象 */
        JavaSparkContext sc =new JavaSparkContext(conf);//其底层实际上是Scala的SparkContext

        /** * 第三步,根据具体的数据来源(HDFS,HBase,Local,FS,DB,S3等),通过JavaSparkContext来创建JavaRDD * JavaRDD的创建方式有三种:根据外部数据来源(例如HDFS), * 根据Scala集合,由其他的RDD操作数据会将RDD划分成一系列Partition, * 分配到每个Partition的数据属于一个Task处理范畴 */
        JavaRDD<String> lines = sc.textFile("D://spark-1.6.1-bin-hadoop2.6//README.md");

        JavaRDD words = lines.flatMap(new FlatMapFunction<String,String>() {//如果是Scala,由于SAM转化,所以可以写成val words=lines.flatMap{line =>line.split(" ")}
            public Iterable call(String line) throws Exception {
            return Arrays.asList(line.split(" "));
            }
            });

        /** * 第4步:对初始的JavaRDD进行Transformation级别的处理,例如map,filter等高阶函数等的编程,来进行具体的数据计算 * 第4.1步:在单词拆分的基础上对每个单词实例进行计数为1,也就是word =>(word,1) */
        JavaPairRDD<String,Integer> pairs=words.mapToPair(new PairFunction<String,String,Integer>()
                {
            public Tuple2<String,Integer> call(String word) throws Exception{
                return new Tuple2<String,Integer>(word,1);
            }
                });
        /** * 统计总次数 */
        JavaPairRDD<String,Integer> wordCount=pairs.reduceByKey(new Function2<Integer,Integer,Integer>()
                {
            public Integer call(Integer v1,Integer v2)throws Exception
            {
                return v1+v2;

                }
                });

        wordCount.foreach(new VoidFunction<Tuple2<String,Integer>>(){
            public void call(Tuple2<String,Integer> pairs) throws Exception {
                System.out.println(pairs._1()+":"+pairs._2());
                }
        });

        sc.close();
    }

}

运行结果:
Java开发Spark程序_第4张图片

集群模式
代码如下:

/** * 文件名:WordCount_Cluster.java * * 创建人:Sundujing - [email protected] * * 创建时间:2016年5月9日 下午12:40:24 * * 版权所有:Sundujing */
package com.dt.spark.SparkApps.cores;


/** * [描述信息:说明类的基本功能] * * @author Sundujing - [email protected] * @version 1.0 Created on 2016年5月9日 下午12:40:24 */

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.api.java.function.VoidFunction;

import scala.Tuple2;

import java.util.Arrays;
import java.util.List;


public class WordCount_Cluster {
public static void main(String[] args)
{
    SparkConf conf = new SparkConf();
//SparkConf conf = new SparkConf().setMaster("spark://yarn-client:7077").setAppName("WordCount by java");
//SparkConf conf = new SparkConf().setMaster("spark://172.171.51.131:7077").setAppName("WordCount by java");
//SparkConf conf = new SparkConf().set("spark.driver.host", "node1") .set("spark.driver.port", "60959").setAppName("WordCount by java");

JavaSparkContext sc = new JavaSparkContext(conf);

JavaRDD lines = sc.textFile("hdfs://node1:8020/tmp/harryport.txt");
//JavaRDD<String> lines = sc.textFile("D://spark-1.6.1-bin-hadoop2.6//README.md");

JavaRDD words = lines.flatMap(new FlatMapFunction<String,String>() {
public Iterable call(String line) throws Exception {
return Arrays.asList(line.split(" "));
}
});
JavaPairRDD<String,Integer> paris = words.mapToPair(new PairFunction<String,String, Integer>() {
public Tuple2<String,Integer> call(String word) throws Exception {
return new Tuple2<String,Integer>(word, 1);
}
});

JavaPairRDD<String,Integer> wordCount = paris.reduceByKey(new Function2<Integer, Integer, Integer>() {
public Integer call(Integer v1, Integer v2) throws Exception {
return v1 + v2;
}
});

List<Tuple2<String,Integer>> list = wordCount.collect();
for(Tuple2<String,Integer> pari :list)
{
System.out.println(pari._1() + ":" + pari._2());
}
sc.close();
}
}

读取hdfs://node1:8020/tmp/harryport.txt中的内容,并分词统计

1.打包
Export-》jar file

jar名javacount.jar
2.写运行脚本javacount.sh

./spark-submit --class com.dt.spark.SparkApps.cores.WordCount_Cluster /root/javacount.jar

3.传入集群
Java开发Spark程序_第5张图片
4.进入spark/bin目录下
这里写图片描述
5.执行脚本

 sh /root/javacount.sh

6.运行结果
Java开发Spark程序_第6张图片

你可能感兴趣的:(java,spark)