Hadoop Java API

获取HDFS客户端对象

Configuration  configuration = new Configuration();
//构造一个访问指定HDFS的客户端对象
FileSystem fileSystem = FileSystem.get(new URI("hdfs://hadoop000:8020"),configuration,"hadoop");

创建文件夹

Path path = new Path("/hdfsapi/test");
//mkdirs(path)创建路径
Boolean result = fileSystem.mkdirs(path);

读取HDFS文件

FSDataInputStream fsDataInputStream =fileSystem.open(new Path("/README.txt"));
IOUtils.copyBytes(fsDataInputStream,System.out,1024);

创建文件、写文件

FSDataOutputStream fsDataOutputStream = fileSystem.create(new Path("/hdfsapi/test/hello.txt"));
fsDataOutputStream.writeUTF("hello world  hello\n  world hello  hello ");
fsDataOutputStream.flush();
fsDataOutputStream.close();

重命名文件

Path oldPath = new Path("/hdfsapi/test/b.txt");
Path newPAth = new Path("/hdfsapi/test/c.txt");
boolean result =  fileSystem.rename(oldPath,newPAth);
System.out.println(result);

从本地复制文件到hdfs

Path localPath = new Path("/developer/jdk-8u211-linux-x64.tar.gz");
Path newPath = new Path("/hdfsapi/test/linux.rar");
fileSystem.copyFromLocalFile(localPath,newPath);

复制文件到本地

Path srcPath = new Path("/hdfsapi/test/a.txt");
Path localPAth = new Path("/developer/a.txt");
fileSystem.copyToLocalFile(srcPath,localPAth);

删除文件

fileSystem.delete(new Path("/hdfsapi/test/jdk.jar"),true);

查看文件块信息

FileStatus fileStatus= fileSystem.getFileStatus(new Path("/hdfsapi/test/jdk.jar"));
BlockLocation[] blockLocations = fileSystem.getFileBlockLocations(fileStatus,0,fileStatus.getLen());
for (BlockLocation block:blockLocations){
      for (String name:block.getNames()){
                System.out.println(name+":"+ block.getOffset()+":"+block.getLength());
            }

        }

列出所有文件 文件夹

 FileStatus[]  fileStatuses = fileSystem.listStatus(new Path("/"));
 for (FileStatus file:fileStatuses){
            String isDir = file.isDirectory() ? "文件夹" : "文件";
            String permission = file.getPermission().toString();
            short replication = file.getReplication();
            long length = file.getLen();
            String path = file.getPath().toString();

            System.out.println(isDir + "\t" + permission + "\t" + replication + "\t" + length + "\t" + path);

        }

递归列出所有文件

RemoteIterator  files = fileSystem.listFiles(new Path("/"),true);
while (files.hasNext()){
            LocatedFileStatus file = files.next();
            String isDir = file.isDirectory() ? "文件夹" : "文件";
            String permission = file.getPermission().toString();
            short replication = file.getReplication();
            long length = file.getLen();
            String path = file.getPath().toString();

            System.out.println(isDir + "\t" + permission + "\t" + replication + "\t" + length + "\t" + path);

        }