HDFS 常用API及HDFS命令

                                             HDFS 常用API及HDFS命令

 

本地环境：

     Centos：Centos7.6

     Java：1.8

     Hadoop：hadoop-2.6.0-cdh5.16.2

 

1.创建一个文件

package com.xk.Hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.net.URI;

public class HdfsApi {

    /**
     * 在HDFS上面创建一个文件夹
     * @throws Exception
     */
    public static void mkdir() throws Exception{
        FileSystem fileSystem = HdfsApi.CreateFileSystem();
        fileSystem.mkdirs(new Path("/hdfsapi"));
        fileSystem.close();
        closeFileSystem(fileSystem);
    }
    /**
     * 创建一个Configuration
     * @return
     */
    public static FileSystem CreateFileSystem() throws Exception{
        Configuration conf = new Configuration();
        conf.set("dfs.replication","1");
        URI uri = new URI("hdfs://hadoop:9000");
        FileSystem fileSystem = FileSystem.get(uri,conf,"hadoop");
        return fileSystem;
    }

    /**
     * 关闭FileSystem
     * @param fileSystem
     * @throws IOException
     */
    public static void closeFileSystem(FileSystem fileSystem) throws IOException {
        fileSystem.close();
    }
}

等效于 hadoop fs -mkdir / hdfs dfs -mkdir

 

2.从本地上传文件

2.1 通过HDFS API 来上传文件

package com.xk.Hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.net.URI;

public class HdfsApi {

    /**
     * 从本地上传文件
     * @throws Exception
     */
    public static void copyFromLocalFile() throws Exception{
        FileSystem fileSystem = HdfsApi.CreateFileSystem();
        fileSystem.copyFromLocalFile(new Path("data/data1.txt"),new Path("/ruozedata/hdfs-works/20211001/3.txt"));
        closeFileSystem(fileSystem);
    }

    
    /**
     * 创建一个Configuration
     * @return
     */
    public static FileSystem CreateFileSystem() throws Exception{
        Configuration conf = new Configuration();
        conf.set("dfs.replication","1");
        URI uri = new URI("hdfs://hadoop:9000");
        FileSystem fileSystem = FileSystem.get(uri,conf,"hadoop");
        return fileSystem;
    }

    /**
     * 关闭FileSystem
     * @param fileSystem
     * @throws IOException
     */
    public static void closeFileSystem(FileSystem fileSystem) throws IOException {
        fileSystem.close();
    }
}

2.2 通过IO流来上传文件

package com.xk.Hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.net.URI;

public class HdfsApi {

   
    /**
     * 通过IO从本地上传文件
     * @throws Exception
     */
    public static void copyFromLocalFileByIo() throws Exception{
        BufferedInputStream in = new BufferedInputStream(new FileInputStream(new File("data/data1.txt")));
        FileSystem fileSystem = HdfsApi.CreateFileSystem();
        FSDataOutputStream out = fileSystem.create(new Path("/out/data1.txt"));
        IOUtils.copyBytes(in,out,4096);
        IOUtils.closeStream(in);
        IOUtils.closeStream(out);
        closeFileSystem(fileSystem);
    }
    /**
     * 创建一个Configuration
     * @return
     */
    public static FileSystem CreateFileSystem() throws Exception{
        Configuration conf = new Configuration();
        conf.set("dfs.replication","1");
        URI uri = new URI("hdfs://hadoop:9000");
        FileSystem fileSystem = FileSystem.get(uri,conf,"hadoop");
        return fileSystem;
    }

    /**
     * 关闭FileSystem
     * @param fileSystem
     * @throws IOException
     */
    public static void closeFileSystem(FileSystem fileSystem) throws IOException {
        fileSystem.close();
    }

}

等效于 hadoop fs -put /hdfs dfs -put

3.从文件系统里面copy文件

package com.xk.Hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.net.URI;

public class HdfsApi {
    /**
     * 从文件系统里面copy文件
     * @throws Exception
     */
    public static void copyToLocalFile() throws Exception{
        FileSystem fileSystem = HdfsApi.CreateFileSystem();
        fileSystem.copyToLocalFile(new Path("/out/data1.txt"),new Path("out/data1.txt"));
        closeFileSystem(fileSystem);
    }

    /**
     * 创建一个Configuration
     * @return
     */
    public static FileSystem CreateFileSystem() throws Exception{
        Configuration conf = new Configuration();
        conf.set("dfs.replication","1");
        URI uri = new URI("hdfs://hadoop:9000");
        FileSystem fileSystem = FileSystem.get(uri,conf,"hadoop");
        return fileSystem;
    }

    /**
     * 关闭FileSystem
     * @param fileSystem
     * @throws IOException
     */
    public static void closeFileSystem(FileSystem fileSystem) throws IOException {
        fileSystem.close();
    }
}

等效于 hadoop fs -get/hdfs dfs -get

4.修改HDFS上面的文件名称

package com.xk.Hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.net.URI;

public class HdfsApi {
    /**
     * 修改HDFS上面的文件名称
     * @throws Exception
     */
    public static void reName() throws Exception{
        FileSystem fileSystem = HdfsApi.CreateFileSystem();
        boolean isReName = fileSystem.rename(new Path("/out/data1.txt"), new Path("/out/data1.txt2"));
        System.out.println(isReName);
        closeFileSystem(fileSystem);
    }

    /**
     * 创建一个Configuration
     * @return
     */
    public static FileSystem CreateFileSystem() throws Exception{
        Configuration conf = new Configuration();
        conf.set("dfs.replication","1");
        URI uri = new URI("hdfs://hadoop:9000");
        FileSystem fileSystem = FileSystem.get(uri,conf,"hadoop");
        return fileSystem;
    }

    /**
     * 关闭FileSystem
     * @param fileSystem
     * @throws IOException
     */
    public static void closeFileSystem(FileSystem fileSystem) throws IOException {
        fileSystem.close();
    }
}

等效于 hadoop fs -mv / hdfs dfs -mv

5.列出文件列表

package com.xk.Hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.net.URI;

public class HdfsApi {
    /**
     * 列出文件列表
     * @throws Exception
     */
    public static void listFiles() throws Exception{
        FileSystem fileSystem = HdfsApi.CreateFileSystem();
        RemoteIterator files = fileSystem.listFiles(new Path("/out"), true);
        while (files.hasNext()){
            LocatedFileStatus file = files.next();
            String path = file.getPath().toString();
            String isdic =  file.isDirectory() ? "文件夹":"文件";
            String owner = file.getOwner();
            System.out.println(isdic + "\t" + path + "\t" +owner);
            // 得到该文件的副本地址
            BlockLocation[] blockLocations = file.getBlockLocations();
            for (BlockLocation blockLocation : blockLocations){
                // 得到该副本的存储地址
                String[] hosts = blockLocation.getHosts();
                for (String host : hosts){
                    System.out.println(host);
                }
            }
        }
        closeFileSystem(fileSystem);
    }

    /**
     * 创建一个Configuration
     * @return
     */
    public static FileSystem CreateFileSystem() throws Exception{
        Configuration conf = new Configuration();
        conf.set("dfs.replication","1");
        URI uri = new URI("hdfs://hadoop:9000");
        FileSystem fileSystem = FileSystem.get(uri,conf,"hadoop");
        return fileSystem;
    }

    /**
     * 关闭FileSystem
     * @param fileSystem
     * @throws IOException
     */
    public static void closeFileSystem(FileSystem fileSystem) throws IOException {
        fileSystem.close();
    }
}

等效于 hadoop fs -ls / hdfs dfs -ls

6.删除文件

package com.xk.Hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.net.URI;

public class HdfsApi {
    /**
     * 删除文件
     * @throws Exception
     */
    public static void delete() throws Exception{
        FileSystem fileSystem = HdfsApi.CreateFileSystem();
        fileSystem.delete(new Path("/hdfsapi/hadoop-2.6.0-cdh5.16.2.tar.gz"),true);
        closeFileSystem(fileSystem);
    }

    /**
     * 创建一个Configuration
     * @return
     */
    public static FileSystem CreateFileSystem() throws Exception{
        Configuration conf = new Configuration();
        conf.set("dfs.replication","1");
        URI uri = new URI("hdfs://hadoop:9000");
        FileSystem fileSystem = FileSystem.get(uri,conf,"hadoop");
        return fileSystem;
    }

    /**
     * 关闭FileSystem
     * @param fileSystem
     * @throws IOException
     */
    public static void closeFileSystem(FileSystem fileSystem) throws IOException {
        fileSystem.close();
    }
}

等效于 hadoop fs -rm / hdfs dfs -rm