hadoop教程-hdfs客户端开发

hadoop环境

hadoop版本为

hadoop version
Hadoop 2.10.1
Subversion https://github.com/apache/hadoop -r 1827467c9a56f133025f28557bfc2c562d78e816
Compiled by centos on 2020-09-14T13:17Z
Compiled with protoc 2.5.0
From source with checksum 3114edef868f1f3824e7d0f68be03650

客户端开发

  • 引入依赖(使用maven)

    org.apache.hadoop
    hadoop-client
    2.10.1



    org.apache.hadoop
    hadoop-hdfs
    2.10.1



    org.apache.hadoop
    hadoop-common
    2.10.1
  • 编写代码
package com.definesys.hadoop;

import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DistributedFileSystem;

import java.io.FileInputStream;
import java.io.IOException;


/**
 * @Description:
 * @author: jianfeng.zheng
 * @since: 2020/12/14 12:36 上午
 * @history: 1.2020/12/14 created by jianfeng.zheng
 */
public class HDFS {

    public static void main(String[] cmd) throws IOException {
        Configuration conf = new Configuration();
        conf.set("fs.defaultFS", "hdfs://master:9000/");
//        conf.set("fs.hdfs.impl", DistributedFileSystem.class.getName());
        System.setProperty("HADOOP_USER_NAME", "hadoop");
        FileSystem fs = FileSystem.get(conf);
        Path dst = new Path("hdfs://master:9000/demo/hello.txt");
        FSDataOutputStream os = fs.create(dst);
        FileInputStream is = new FileInputStream("/root/hello.txt");
        IOUtils.copy(is, os);
        is.close();
        os.close();
        fs.close();
    }
}
  • 打包

如果是web应用,一般会打包为war或者ear,不管是哪种,这两种包格式都会把依赖包打进去,因此不用做特殊处理,如果需要本地运行,那么需要借助两个插件,把以下配置信息复制到pom.xml中


    
        
            org.apache.maven.plugins
            maven-jar-plugin
            2.6
            
                
                    
                        true
                        lib/
                        com.definesys.hadoop.HDFS
                    
                
            
        

        
            org.apache.maven.plugins
            maven-dependency-plugin
            
                
                    copy-dependencies
                    package
                    
                        copy-dependencies
                    
                    
                        ${project.build.directory}/lib
                        false
                        false
                        true
                    
                
            
        
    

maven-jar-plugin会根据配置生成MANIFEST.MF文件,MANIFEST.MF文件记录运行类信息,依赖信息,类似以下这样

Manifest-Version: 1.0
Archiver-Version: Plexus Archiver
Built-By: asan
Class-Path: lib/hadoop-client-2.10.1.jar ....
Created-By: Apache Maven 3.6.3
Build-Jdk: 1.8.0_161
Main-Class: com.definesys.hadoop.HDFS

classpathPrefix指定了依赖jar包所在的路径为lib,maven-dependency-plugin插件负责将依赖包全部copy到指定路径下,这里指定了${project.build.directory}/lib目录,和classpathPrefix对应,打包完成后执行以下命令即可

java -jar hadoop-hdfs-1.0.jar

#或者手动指定运行类

java -cp hadoop-hdfs-1.0.jar com.definesys.hadoop.HDFS
打包还有一个插件maven-assembly-plugin,不建议使用这个插件进行打包,原因是这个插件会将所有依赖解压放到一个jar包里,hadoop有些机制是通过spi实现,解压后会造成配置文件覆盖的情况

一个简单的HDFS操作类

package com.definesys.hadoop;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

import java.io.FileInputStream;
import java.io.IOException;


/**
 * @Description:
 * @author: jianfeng.zheng
 * @since: 2020/12/14 12:36 上午
 * @history: 1.2020/12/14 created by jianfeng.zheng
 */
public class HDFS {

    public static void main(String[] cmd) throws IOException {
        HDFS hdfs = new HDFS();
        hdfs.mkdir("/hdfsDemo");
        hdfs.putFile("/root/hello.txt", "/hdfsDemo");
        hdfs.dowloadFile("/hdfsDemo/hello.txt", "/root/hello-hdfs.txt");
        hdfs.deleteFile("/hdfsDemo");
    }

    public boolean mkdir(String path) throws IOException {
        FileSystem fs = this.getHDFSFileSystem();
        return fs.mkdirs(new Path(path));
    }

    public void putFile(String localPath, String hdfsPath) throws IOException {
        this.getHDFSFileSystem().copyFromLocalFile(new Path(localPath), new Path(hdfsPath));
    }

    public void deleteFile(String path) throws IOException {
        this.getHDFSFileSystem().delete(new Path(path), true);
    }

    public void dowloadFile(String hdfsPath, String localPath) throws IOException {
        this.getHDFSFileSystem().copyToLocalFile(new Path(hdfsPath), new Path(localPath));
    }

    private FileSystem getHDFSFileSystem() {
        Configuration conf = new Configuration();
        conf.set("fs.defaultFS", "hdfs://master:9000/");
        System.setProperty("HADOOP_USER_NAME", "hadoop");
        try {
            FileSystem fs = FileSystem.get(conf);
            return fs;
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }
}

问题

权限问题

Exception in thread "main" org.apache.hadoop.security.AccessControlException: Permission denied: user=root, access=WRITE, inode="/":hadoop:supergroup:drwxr-xr-x
        at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.check(FSPermissionChecker.java:350)
        at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkPermission(FSPermissionChecker.java:251)
        at org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkPermission(FSPermissionChecker.java:189)

HDFS文件系统权限和Linux类似,不同的用户对文件操作权限不一样,如果代码中没有指定用户名,那么就用执行程序的操作系统作为用户名,在这里是root,我们可以看下hdfs的文件权限

$ hadoop fs -ls /
Found 5 items
drwxr-xr-x   - asan   supergroup          0 2020-12-16 10:07 /001
drwx-w----   - hadoop supergroup          0 2020-12-07 10:54 /tmp
drwxr-xr-x   - hadoop supergroup          0 2020-12-07 11:05 /user

# 根路径权限

$ hadoop fs -ls -d /
drwxr-xr-x   - hadoop supergroup          0 2020-12-18 00:42 /

有几个解决方案

  • 修改根路径权限或者其他文件夹权限为777
$ hadoop fs -chmod 777 /demo

$ hadoop fs -ls -d /demo
drwxrwxrwx   - hadoop supergroup          0 2020-12-18 00:46 /demo
  • 取消权限验证

在master节点加入以下配置


    dfs.permissions.enabled
    false
  • 在代码中加入用户名配置(推荐)
System.setProperty("HADOOP_USER_NAME", "hadoop");
代码需在执行hdfs操作之前加入

你可能感兴趣的:(hadoop,hdfs)