java实现HDFS增删改查

环境：Hadoop 2.7.3

import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.zookeeper.common.IOUtils;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import java.util.ArrayList;
import java.util.List;

/**
 * Created by LCY on 4/2/2018.
 * 对HDFS执行上传文件、上传目录、查看文件、查看目录、删除文件等操作
 */
public class HDFSUtils {
    /**
     * 在HDFS创建新的目录
     *
     * @param uri HDFS地址，比如： 'hdfs://192.168.12.12'
     * @param dir 路径，比如： '/tmp/testdir'
     * @return boolean true-success, false-failed
     * @exception IOException something wrong happends when operating files
     */
    public  boolean mkdir(String uri,String dir)  {
        try {
            if (StringUtils.isBlank(dir)) {
                return false;
            }
            dir = uri + dir;
            Configuration conf = new Configuration();
            FileSystem fs = FileSystem.get(URI.create(dir), conf);
            if (!fs.exists(new Path(dir))) {
                fs.mkdirs(new Path(dir));
            }
            fs.close();
            return true;
        } catch (IOException e) {
            System.err.println("ERROR:IO");
            return false;
        }
    }

    /**
     * 在HDFS删除目录
     *
     * @param uri HDFS地址，比如： 'hdfs://192.168.12.12'
     * @param dir 文件路径
     * @return boolean true-success, false-failed
     * @exception IOException 如果文件已经打开会跑出异常 FileNotFoundException
     *
     */
    public  boolean deleteDir(String uri,String dir) throws IOException {
        if (StringUtils.isBlank(dir)) {
            return false;
        }
        dir = uri + dir;
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(URI.create(dir), conf);
        fs.delete(new Path(dir), true);
        fs.close();
        return true;
    }

    /**
     * 获取文件列表
     *
     * @param uri HDFS地址，比如： 'hdfs://192.168.12.12'
     * @param dir 目录路径
     * @return List list of file names
     * @throws IOException file io exception
     */
    public  List listAll(String uri,String dir) throws IOException {
        if (StringUtils.isBlank(dir)) {
            return new ArrayList();
        }
        dir = uri + dir;
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(URI.create(dir), conf);
        FileStatus[] stats = fs.listStatus(new Path(dir));
        List names = new ArrayList();
        for (int i = 0; i < stats.length; ++i) {
            if (stats[i].isFile()) {
                // regular file
                names.add(stats[i].getPath().toString());
            } else if (stats[i].isDirectory()) {
                // dir
                names.add(stats[i].getPath().toString());
            } else if (stats[i].isSymlink()) {
                // is s symlink in linux
                names.add(stats[i].getPath().toString());
            }
        }

        fs.close();
        return names;
    }

    /*
     * 上传文件到HDFS
     * 注意：路径为完整路径
     * 如果本地文件不存在有异常 FileNotFoundException
     *
     * @param uri HDFS地址，比如： 'hdfs://192.168.12.12'
     * @param localFile 本地文件路径例如F:/test.txt or /usr/local/test.txt
     *
     * @param hdfsFile hdfs路径例如： /tmp/dir
     * @return boolean true-success, false-failed
     *
     * @throws IOException file io exception
     */
    public  boolean uploadLocalFile2HDFS(String uri,String localFile, String hdfsFile)  {
        if (StringUtils.isBlank(localFile) || StringUtils.isBlank(hdfsFile)) {
            return false;
        }
        try {
            hdfsFile = uri + hdfsFile;
            Configuration config = new Configuration();
            FileSystem hdfs  = FileSystem.get(URI.create(uri), config);
            Path src = new Path(localFile);
            Path dst = new Path(hdfsFile);
            hdfs.copyFromLocalFile(src, dst);
            hdfs.close();
            return true;
        } catch (IOException e) {
            e.printStackTrace();
        } catch (Exception e){
            e.printStackTrace();
        }
        return false;
    }

    /*
     * 在HDFS创建一个新的文件，并写入content数据
     *
     * @param uri HDFS地址，比如： 'hdfs://192.168.12.12'
     * @param newFile 文件路径： '/tmp/test.txt'
     * @param content 写入的内容
     * @return boolean true-success, false-failed
     * @throws IOException file io exception
     */
    public  boolean createNewHDFSFile(String uri,String newFile, String content) throws IOException {
        if (StringUtils.isBlank(newFile) || null == content) {
            return false;
        }
        newFile = uri + newFile;
        Configuration config = new Configuration();
        FileSystem hdfs = FileSystem.get(URI.create(newFile), config);
        FSDataOutputStream fsdos = hdfs.create(new Path(newFile));
        fsdos.write(content.getBytes("UTF-8"));
        fsdos.close();
        hdfs.close();
        return true;
    }
    /**
     * 在HDFS删除文件
     *
     * @param uri HDFS地址，比如： 'hdfs://192.168.12.12'
     * @param hdfsFile 完整的路径 '/tmp/test.txt'
     * @return boolean true-success, false-failed
     * @throws IOException file io exception
     */
    public  boolean deleteHDFSFile(String uri,String hdfsFile) throws IOException {
        if (StringUtils.isBlank(hdfsFile)) {
            return false;
        }
        hdfsFile = uri + hdfsFile;
        Configuration config = new Configuration();
        FileSystem hdfs = FileSystem.get(URI.create(hdfsFile), config);
        Path path = new Path(hdfsFile);
        boolean isDeleted = hdfs.delete(path, true);
        hdfs.close();
        return isDeleted;
    }

    /**
     * 读取文件内容
     *
     * @param uri HDFS地址，比如： 'hdfs://192.168.12.12'
     * @param hdfsFile 完整的文件路径 '/tmp/test.txt'
     * @return 文件的byte[]
     * @throws IOException file io exception
     */
    public  byte[] readHDFSFile(String uri,String hdfsFile) throws Exception {
        if (StringUtils.isBlank(hdfsFile)) {
            return null;
        }
        hdfsFile = uri + hdfsFile;
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(URI.create(hdfsFile), conf);
        // check if the file exists
        Path path = new Path(hdfsFile);
        if (fs.exists(path)) {
            FSDataInputStream is = fs.open(path);
            // get the file info to create the buffer
            FileStatus stat = fs.getFileStatus(path);
            // create the buffer
            byte[] buffer = new byte[Integer.parseInt(String.valueOf(stat.getLen()))];
            is.readFully(0, buffer);
            is.close();
            fs.close();
            return buffer;
        } else {
            throw new Exception("the file is not found .");
        }
    }

    /**
     * 在文件中写入内容
     *
     * @param uri HDFS地址，比如： 'hdfs://192.168.12.12'
     * @param hdfsFile 文件路径'/tmp/test.txt'
     * @param content 写入的内容
     * @return boolean true-success, false-failed
     * @throws Exception something wrong
     */
    public  boolean append(String uri,String hdfsFile, String content) throws Exception {
        if (StringUtils.isBlank(hdfsFile)) {
            return false;
        }
        if(StringUtils.isEmpty(content)){
            return true;
        }

        hdfsFile = uri + hdfsFile;
        Configuration conf = new Configuration();
        // solve the problem when appending at single datanode hadoop env
        conf.set("dfs.client.block.write.replace-datanode-on-failure.policy", "NEVER");
        conf.set("dfs.client.block.write.replace-datanode-on-failure.enable", "true");
        FileSystem fs = FileSystem.get(URI.create(hdfsFile), conf);
        // check if the file exists
        Path path = new Path(hdfsFile);
        if (fs.exists(path)) {
            try {
                InputStream in = new ByteArrayInputStream(content.getBytes());
                OutputStream out = fs.append(new Path(hdfsFile));
                IOUtils.copyBytes(in, out, 4096, true);
                out.close();
                in.close();
                fs.close();
            } catch (Exception ex) {
                fs.close();
                throw ex;
            }
        } else {
            createNewHDFSFile(uri,hdfsFile, content);
        }
        return true;
    }
}

调用：

public class test {
    @Test
    public void update() {
        String uri="hdfs://172.16.0.108";
        HDFSUtils hdfsUtils=new HDFSUtils();
      hdfsUtils.uploadLocalFile2HDFS(uri,"data/ITEM.txt","/tmp/test");
    }

}