2023_Spark_实验十六：编写LoggerLevel方法及getLocalSparkSession方法

一、搭建Spark项目结构

在SparkProject模块的pom.xml文件中增加一下依赖，并等待依赖包下载完毕，如上图。

​





2.11

2.1.1













com.thoughtworks.paranamer

paranamer

2.8





org.apache.spark

spark-core_${scala.version}

${spark.version}





org.apache.spark

spark-sql_${scala.version}

${spark.version}





org.apache.spark

spark-streaming_2.11

${spark.version}





org.apache.spark

spark-mllib_2.11

2.1.1





org.apache.spark

spark-streaming-kafka-0-10_2.11

2.3.0





org.apache.spark

spark-streaming-kafka-0-8_${scala.version}

2.3.0





net.jpountz.lz4

lz4

1.3.0





mysql

mysql-connector-java

8.0.18





org.apache.flume.flume-ng-clients

flume-ng-log4jappender

1.7.0















org.apache.spark

spark-hive_2.12

2.4.8













org.apache.maven.plugins

maven-compiler-plugin

3.8.1



1.8

1.8







org.apache.maven.plugins

maven-assembly-plugin





jar-with-dependencies











​

二、解决无法创建scala文件问题

三、编写LoggerLevel特质

在特质  下增加如下代码

Logger.getLogger("org").setLevel(Level.ERROR)

这个时候需要导包

完整代码如下：

 

import org.apache.log4j.{Level, Logger}



trait LoggerLevel {

Logger.getLogger("org").setLevel(Level.ERROR)

}

四、编写getLocalSparkSession方法

以下是完整代码：

import org.apache.spark.sql.SparkSession



object SparkUnit {

/**

* 一个class参数

**/

def getLocalSparkSession(appName: String): SparkSession = {

SparkSession.builder().appName(appName).master("local[2]").getOrCreate()

}



def getLocalSparkSession(appName: String, support: Boolean): SparkSession = {

if (support) SparkSession.builder().master("local[2]").appName(appName).enableHiveSupport().getOrCreate()

else getLocalSparkSession(appName)

}



def getLocalSparkSession(appName: String, master: String): SparkSession = {

SparkSession.builder().appName(appName).master(master).getOrCreate()

}



def getLocalSparkSession(appName: String, master: String, support: Boolean): SparkSession = {

if (support) SparkSession.builder().appName(appName).master(master).enableHiveSupport().getOrCreate()

else getLocalSparkSession(appName, master)

}



def stopSpark(ss: SparkSession) = {

if (ss != null) {

ss.stop()

}

}



}