Java API 读写 hdfs分布式文件系统 （创建目录、读文件、写文件）

                         Java API 读写 hdfs 操作

一、概述

    操作hdfs分布式文件系统的类为FileSystem对象

    1、创建文件夹（递归创建）：fs. mkdirs()

    2、写文件  ： fs.create（new Path()），返回FSDataOutputStream对象

    3、读文件  ： fs.open(new Path())  , 返回FSDataInputStream对象

二、代码

package com.example.hdfs; import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import java.net.URI; import java.util.logging.Logger; public class TestMain { private static final Logger logger = Logger.getLogger("com.example.hdfs.TestMain"); public static void main(String[] args) throws Exception { //HDFS URI} String hdfsuri = "hdfs://localhost:8020/"; String path="/user/hdfs/example/hdfs/"; String fileName="hello.csv"; String fileContent="hello;world"; // ====== Init HDFS File System Object Configuration conf = new Configuration(); // Set FileSystem URI conf.set("fs.defaultFS", hdfsuri); // Because of Maven conf.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName()); conf.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName()); // 设置hadoop应用程序执行用户 System.setProperty("HADOOP_USER_NAME", "hdfs"); System.setProperty("hadoop.home.dir", "/"); //创建 FileSystem - HDFS FileSystem fs = FileSystem.get(URI.create(hdfsuri), conf); //创建文件目录，如果不存在 Path workingDir=fs.getWorkingDirectory(); Path newFolderPath= new Path(path); if(!fs.exists(newFolderPath)) { // Create new Directory fs.mkdirs(newFolderPath); logger.info("Path "+path+" created."); } //==== Write file logger.info("Begin Write file into hdfs"); //Create a path Path hdfswritepath = new Path(newFolderPath + "/" + fileName); //Init output stream FSDataOutputStream outputStream=fs.create(hdfswritepath); //Cassical output stream usage outputStream.writeBytes(fileContent); outputStream.close(); logger.info("End Write file into hdfs"); //==== Read file logger.info("Read file into hdfs"); //Create a path Path hdfsreadpath = new Path(newFolderPath + "/" + fileName); //Init input stream FSDataInputStream inputStream = fs.open(hdfsreadpath); //Classical input stream usage String out= IOUtils.toString(inputStream, "UTF-8"); logger.info(out); inputStream.close(); fs.close(); } }