【Spark-HDFS小文件合并】使用 Spark 实现 HDFS 小文件合并
【Spark-HDFS小文件合并】使用 Spark 实现 HDFS 小文件合并
- 1）导入依赖
- 2）代码实现
-
- 2.1.HDFSUtils
- 2.2.MergeFilesApplication
需求描述：
1、使用 Spark 做小文件合并压缩处理。
2、实际生产中相关配置、日志、明细可以记录在 Mysql 中。
3、core-site.xml、hdfs-site.xml、hive-site.xml、yarn-site.xmlx 等文件放在项目的 resources 目录下进行认证。
4、下面的案例抽取出了主体部分的代码，具体实现时需要结合 HDFS 工具类，利用好 Mysql 做好配置、日志、以及相关明细，结合各自业务进行文件合并。
1）导入依赖
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0modelVersion>
<groupId>test.cn.suitcasegroupId>
<artifactId>mergefilesartifactId>
<version>4.0.0version>
<properties>
<project.build.sourceEncoding>UTF-8project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8project.reporting.outputEncoding>
<java.version>1.8java.version>
<maven.compiler.source>1.8maven.compiler.source>
<maven.compiler.target>1.8maven.compiler.target>
<encoding>UTF-8encoding>
<spark.version>2.4.8spark.version>
<scala.version>2.11.12scala.version>
properties>
<dependencies>
<dependency>
<groupId>org.apache.logging.log4jgroupId>
<artifactId>log4j-coreartifactId>
<version>2.20.0version>
dependency>
<dependency>
<groupId>org.apache.hadoopgroupId>
<artifactId>hadoop-commonartifactId>
<version>3.3.2version>
dependency>
<dependency>
<groupId>org.apache.hadoopgroupId>
<artifactId>hadoop-clientartifactId>
<version>3.3.2version>
dependency>
<dependency>
<groupId>org.apache.hadoopgroupId>
<artifactId>hadoop-hdfsartifactId>
<version>3.3.2version>
dependency>
<dependency>
<groupId>org.apache.logging.log4jgroupId>
<artifactId>log4j-apiartifactId>
<version>2.20.0version>
dependency>
<dependency>
<groupId>org.scala-langgroupId>
<artifactId>scala-libraryartifactId>
<version>${scala.version}version>
dependency>
<dependency>
<groupId>org.scala-langgroupId>
<artifactId>scala-compilerartifactId>
<version>${scala.version}version>
dependency>
<dependency>
<groupId>org.scala-langgroupId>
<artifactId>scala-reflectartifactId>
<version>${scala.version}version>
dependency>
<dependency>
<groupId>org.apache.sparkgroupId>
<artifactId>spark-core_2.11artifactId>
<version>${spark.version}version>
<scope>providedscope>
dependency>
<dependency>
<groupId>org.apache.sparkgroupId>
<artifactId>spark-launcher_2.11artifactId>
<version>${spark.version}version>
<scope>providedscope>
dependency>
<dependency>
<groupId>org.apache.sparkgroupId>
<artifactId>spark-sql_2.11artifactId>
<version>${spark.version}version>
<scope>providedscope>
dependency>
<dependency>
<groupId>org.apache.sparkgroupId>
<artifactId>spark-hive_2.11artifactId>
<version>${spark.version}version>
<scope>providedscope>
dependency>
<dependency>
<groupId>com.alibabagroupId>
<artifactId>fastjsonartifactId>
<version>2.0.32version>
dependency>
<dependency>
<groupId>mysqlgroupId>
<artifactId>mysql-connector-javaartifactId>
<version>8.0.33version>
dependency>
<dependency>
<groupId>com.fasterxml.jackson.coregroupId>
<artifactId>jackson-coreartifactId>
<version>2.14.2version>
dependency>
dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.pluginsgroupId>
<artifactId>maven-compiler-pluginartifactId>
<version>3.1version>
<configuration>
<source>${java.version}source>
<target>${java.version}target>
configuration>
plugin>
<plugin>
<groupId>org.apache.maven.pluginsgroupId>
<artifactId>maven-shade-pluginartifactId>
<version>3.0.0version>
<executions>
<execution>
<phase>packagephase>
<goals>
<goal>shadegoal>
goals>
<configuration>
<artifactSet>
<excludes>
<exclude>org.apache.flink:force-shadingexclude>
<exclude>com.google.code.findbugs:jsr305exclude>
<exclude>org.slf4j:*exclude>
<exclude>org.apache.logging.log4j:*exclude>
excludes>
artifactSet>
<filters>
<filter>
<artifact>*:*artifact>
<excludes>
<exclude>META-INF/*.SFexclude>
<exclude>META-INF/*.DSAexclude>
<exclude>META-INF/*.RSAexclude>
excludes>
filter>
filters>
configuration>
execution>
executions>
plugin>
<plugin>
<groupId>org.apache.maven.pluginsgroupId>
<artifactId>maven-surefire-pluginartifactId>
<version>2.22.1version>
<configuration>
<groups>IntegrationTestgroups>
configuration>
plugin>
<plugin>
<groupId>org.apache.maven.pluginsgroupId>
<artifactId>maven-compiler-pluginartifactId>
<version>3.1version>
<configuration>
<source>${java.version}source>
<target>${java.version}target>
configuration>
plugin>
<plugin>
<groupId>org.apache.maven.pluginsgroupId>
<artifactId>maven-surefire-pluginartifactId>
<version>2.22.1version>
plugin>
plugins>
build>
project>
2）代码实现
2.1.HDFSUtils
public class HDFSUtils {
private static Logger logger = LoggerFactory.getLogger(HDFSUtils.class);
private static final Configuration hdfsConfig = new Configuration();
private static FileSystem fs;
public static void init() {
System.out.println(Thread.currentThread().getContextClassLoader());
try {
hdfsConfig.addResource(Thread.currentThread().getContextClassLoader().getResource("./core-site.xml"));
hdfsConfig.addResource(Thread.currentThread().getContextClassLoader().getResource("./hdfs-site.xml"));
fs = FileSystem.get(hdfsConfig);
} catch (FileNotFoundException fnfe) {
fnfe.printStackTrace();
logger.error("Load properties failed.");
} catch (IOException ioe) {
ioe.printStackTrace();
logger.error(String.format("IOException: " + ioe.getMessage()));
}
}
public static long getDirectorySize(String directoryPath) {
final Path path = new Path(directoryPath);
long size = 0;
try {
size = fs.getContentSummary(path).getLength();
} catch (IOException ex) {
}
return size;
}
public static long getFileCount(String directoryPath) {
final Path path = new Path(directoryPath);
long count = 0;
try {
count = fs.getContentSummary(path).getFileCount();
} catch (IOException ex) {
}
return count;
}
public static long getBlockSize() {
return fs.getDefaultBlockSize(fs.getHomeDirectory());
}
public static String getFile(String filePath) {
final Path path = new Path(filePath);
FSDataInputStream dis = null;
String fileName = null;
try {
if (fs.exists(path) && fs.isFile(path)) {
dis = fs.open(path);
StringWriter stringWriter = new StringWriter();
IOUtils.copy(dis, stringWriter, "UTF-8");
fileName = stringWriter.toString();
return fileName;
} else {
throw new FileNotFoundException();
}
} catch (IOException ioException) {
logger.error("Get file from hdfs failed: " + ioException.getMessage());
} finally {
if (dis != null) {
try {
dis.close();
} catch (IOException ex) {
logger.error("close FSDataInputStream failed: " + ex.getMessage());
}
}
}
return fileName;
}
public static Boolean exists(String filePath) {
Path path = new Path(filePath);
Boolean ifExists = false;
try {
ifExists = fs.exists(path);
return ifExists;
} catch (IOException ex) {
logger.error(String.format("hdfs file %s not exists", filePath));
}
return ifExists;
}
public static boolean renameDir(String existingName, String newName) {
final Path existingPath = new Path(existingName);
final Path finalName = new Path(newName);
try {
if (exists(newName)) {
logger.error(String.format("Path %s already exists when try to rename %s to %s.", newName, existingName, newName));
return false;
}
return fs.rename(existingPath, finalName);
} catch (IOException ex) {
logger.error("Rename hdfs directory failed: " + ex.getMessage());
}
return false;
}
public static boolean removeDirSkipTrash(String dir) {
Path path = new Path(dir);
boolean rv = false;
try {
if (exists(dir)) {
if (fs.delete(path, true)) {
logger.info(String.format("文件夹 %s 删除成功.", path));
rv = true;
}
} else {
logger.error(String.format("要删除的文件夹 %s 不存在", dir));
return false;
}
} catch (IOException ex) {
logger.error("文件夹 %s 存在但是删除失败");
}
return rv;
}
public static List<String> listDirs(String baseDir) {
Path path = new Path(baseDir);
List<String> dirs = new ArrayList<>();
try {
FileStatus[] fileStatuses = fs.globStatus(path);
for (int i = 0; i < fileStatuses.length; i++) {
dirs.add(fileStatuses[i].getPath().toUri().getRawPath());
}
}
} catch (Exception ex) {
logger.error(String.format("List directories under %s failed.", baseDir));
}
return dirs;
}
public static void close() {
try {
fs.close();
} catch (IOException ex) {
logger.error("hdfs file system close failed: " + ex.getMessage());
}
}
}
2.2.MergeFilesApplication
下面的案例抽取出了主体部分的代码，具体实现时需要结合 HDFS 工具类，利用好 Mysql 做好配置、日志、以及相关明细，结合各自业务进行文件合并。
public class MergeFilesApplication {
public static void main(String[] args) {
System.out.println(Arrays.asList(args));
//指定hadoop用户
System.setProperty("HADOOP_USER_NAME", "hdfs");
System.setProperty("user.name", "hdfs");
//获取 SparkSession 对象
SparkSession sparkSession = SparkSession.builder()
.config("spark.scheduler.mode", "FAIR")//配置调度模式
.config("spark.sql.warehouse.dir", "/warehouse/tablespace/external/hive")//配置warehouse目录
.appName("MergeFilesApplication")
.getOrCreate();
//合并文件
sparkSession.read()//spark读取
.parquet(sourceDir)//读取数据源目录
.coalesce(partitions)//配置spark分区数
.sortWithinPartitions("col1", "col2")//每个分区内按照指定需要的列进行排序
.write()//spark写入
.mode(SaveMode.Append)//写入模式为追加
.option("compression", "gzip")//压缩方式以为gzip
.parquet(targetMergedDir);//写入目标目录
}
}