Hadoop系列——SpringBoot集成Hadoop、实现hdfs上传下载文件（1）

• pom.xml

  
  
      org.apache.hadoop
      hadoop-client
      2.7.6
  
  
      org.apache.hadoop
      hadoop-hdfs
      2.7.6
  

  
   
• HadoopConfig.java

  package com.hahashujia.hadoop.config;
  
  import lombok.extern.slf4j.Slf4j;
  import org.apache.hadoop.fs.FileSystem;
  import org.springframework.beans.factory.annotation.Value;
  import org.springframework.context.annotation.Bean;
  import org.springframework.context.annotation.Configuration;
  
  /**
   * @author yanan.wang
   * @describe
   * @createTime 2019-12-19 15:13
   */
  @Configuration
  @Slf4j
  public class HadoopConfig {
  
      @Value("${hadoop.user}")
      private String user;
      @Value("${hadoop.password}")
      private String password;
      @Value("${hdfs.hdfs-site}")
      private String hdfsSite;
      @Value("${hdfs.core-site}")
      private String coreSite;
  
      @Bean("fileSystem")
      public FileSystem createFs() throws Exception {
  
          System.setProperty("HADOOP_USER_NAME", user);
  //        System.setProperty("HADOOP_USER_PASSWORD", password);
          //读取配置文件
          org.apache.hadoop.conf.Configuration conf = new org.apache.hadoop.conf.Configuration();
          conf.addResource(coreSite);
          conf.addResource(hdfsSite);
          conf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem");
  
          log.info("===============【hadoop configuration info start.】===============");
          log.info("【hadoop conf】: size:{}, {}", conf.size(), conf.toString());
          log.info("【fs.defaultFS】: {}", conf.get("fs.defaultFS"));
          log.info("【fs.hdfs.impl】: {}", conf.get("fs.hdfs.impl"));
          FileSystem fs = FileSystem.newInstance(conf);
          log.info("【fileSystem scheme】: {}", fs.getScheme());
          log.info("===============【hadoop configuration info end.】===============");
          return fs;
      }
  }
  

   

•  HadoopTemplate.java

  package com.hahashujia.hadoop.config;
  
  import lombok.extern.slf4j.Slf4j;
  import org.apache.commons.lang3.StringUtils;
  import org.apache.hadoop.fs.FSDataOutputStream;
  import org.apache.hadoop.fs.FileSystem;
  import org.apache.hadoop.fs.Path;
  import org.apache.hadoop.io.IOUtils;
  import org.springframework.beans.factory.annotation.Autowired;
  import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
  import org.springframework.stereotype.Component;
  
  import java.io.IOException;
  import java.io.InputStream;
  
  /**
   * @author yanan.wang
   * @describe
   * @createTime 2019-12-19 15:15
   */
  @Component
  @ConditionalOnBean(FileSystem.class)
  @Slf4j
  public class HadoopTemplate {
  
      @Autowired
      private FileSystem fileSystem;
  
      public void uploadFile(String srcFile, String destPath) {
          copyFileToHDFS(false, true, srcFile, destPath);
      }
  
      public void uploadFile(boolean del, String srcFile, String destPath) {
          copyFileToHDFS(del, true, srcFile, destPath);
      }
  
      public void delDir(String path) {
          rmdir(path, null);
      }
  
      public void download(String fileName, String savePath) {
          getFile(fileName, savePath);
      }
  
  
      /**
       * 创建目录
       *
       * @param filePath
       * @param create
       * @return
       */
      public boolean existDir(String filePath, boolean create) throws IOException {
          boolean flag = false;
          if (StringUtils.isEmpty(filePath)) {
              throw new IllegalArgumentException("filePath不能为空");
          }
          Path path = new Path(filePath);
          if (create) {
              if (!fileSystem.exists(path)) {
                  fileSystem.mkdirs(path);
              }
          }
          if (fileSystem.isDirectory(path)) {
              flag = true;
          }
          return flag;
      }
  
      /**
       * 创建目录
       *
       * @param filePath
       * @return
       */
      public boolean existFile(String filePath) throws IOException {
          if (StringUtils.isEmpty(filePath)) {
              throw new IllegalArgumentException("filePath不能为空");
          }
          Path path = new Path(filePath);
          return fileSystem.exists(path);
      }
  
  
      /**
       * 文件上传至 HDFS
       *
       * @param delSrc    指是否删除源文件，true 为删除，默认为 false
       * @param overwrite
       * @param srcFile   源文件，上传文件路径
       * @param destPath  hdfs的目的路径
       */
      public void copyFileToHDFS(boolean delSrc, boolean overwrite, String srcFile, String destPath) {
          // 源文件路径是Linux下的路径，如果在 windows 下测试，需要改写为Windows下的路径，比如D://hadoop/djt/weibo.txt
          Path srcPath = new Path(srcFile);
          Path dstPath = new Path(destPath);
          // 实现文件上传
          try {
              // 获取FileSystem对象
              fileSystem.copyFromLocalFile(delSrc, overwrite, srcPath, dstPath);
          } catch (IOException e) {
              log.error("", e);
          }
      }
  
      /**
       * 删除文件或者文件目录
       *
       * @param path
       */
      public void rmdir(String path, String fileName) {
          try {
              if (StringUtils.isNotBlank(fileName)) {
                  path = path + "/" + fileName;
              }
              // 删除文件或者文件目录  delete(Path f) 此方法已经弃用
              fileSystem.delete(new Path(path), true);
          } catch (IllegalArgumentException | IOException e) {
              log.error("", e);
          }
      }
  
      /**
       * 从 HDFS 下载文件
       *
       * @param hdfsFile
       * @param destPath 文件下载后,存放地址
       */
      public void getFile(String hdfsFile, String destPath) {
  
          Path hdfsPath = new Path(hdfsFile);
          Path dstPath = new Path(destPath);
          try {
              // 下载hdfs上的文件
              fileSystem.copyToLocalFile(hdfsPath, dstPath);
          } catch (IOException e) {
              log.error("", e);
          }
      }
  
  
      public void writer(String destPath, InputStream in)  {
          try {
              FSDataOutputStream out = fileSystem.create(new Path(destPath));
              IOUtils.copyBytes(in, out, fileSystem.getConf());
          } catch (IOException e) {
              e.printStackTrace();
          }
  
      }
  
  }

   

• 调用（代码片段）

  @Autowired
  private HadoopTemplate hadoopTemplate;
  
  if (hadoopTemplate.existDir(hdfsPath, true)) {
      hadoopTemplate.copyFileToHDFS(false, overwrite, srcFile, hdfsPath);
  } else {
      log.error("==============dir create fail.==============");
  }

  下一篇：是另一种方式调用Hadoop
  Hadoop系列——SpringBoot集成Hadoop、实现hdfs上传下载文件（2）