02_在window环境开发hadoop_HDFS

1. 在window环境开发HDFS（文件）

1.1. 导入jar包

i. 解压hadoop安装包
ii. 导入/share/hadoop/common/Hadoop-common-2.8.5.jar 以及依赖包 lib下全部jar
iii. 导入/share/haddop/hdfs/hadoop-hdfs-client-2.8.5-tests.jar 以及依赖包lib下全部jar
链接: https://pan.baidu.com/s/1pgyCp17E8IKJ46fM87jZrA 提取码: 3251

1.2. 开始写java代码

先构造Configuration对象
构造时，会先加载jar包中的默认配置xxx-default.xml
再加载用户写的xml 放在src下面 xxx-site.xml 会覆盖默认配置
在代码中可以在此conf.set(name,value);, 会再次覆盖配置

public class HdfsClient {
    public static void main(String[] args) throws URISyntaxException, IOException, InterruptedException {
        //设置HDFS的配置
        //会从项目的classpath中加载core-default.xml hdfs-default.xml core-site.xml hdfs-site.xml等配置文件
        Configuration conf = new Configuration();
        //设置副本数量为2
        conf.set("dfs.replication","2");
        //指定切块文件大小为64M
        conf.set("dfs.blocksize","64M");

        //构造一个访问指定HDFS系统的客户端对象，参数1.HDFS系统的URL，参数2,客户端指定的参数，参数3.客户端身份
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://vm01:9000/"),conf,"root");

    }
}

1.2.1. 上传文件，删文件

//上传一个文件到hdfs中

fileSystem.copyFromLocalFile(
         new Path("C:\\Users\\81022\\Desktop\\1.txt"),new Path("/javaTest/1.txt"));
 //fileSystem.delete(new Path("/video.zip"),true);
 fileSystem.close();

1.2.2. 下载文件，（需要在windows中配置hadoop环境）

在Windows下配置环境变量 HADOOP_HOME
Window版本
https://drive.google.com/open?id=1JSVGVmxTtTehgzcUenwZXxjjrSBFL6dZ

fileSystem.copyToLocalFile(new Path("/javaTest/1.txt"),new Path("C:\\Users\\81022\\Desktop\\"));

1.2.3. 移动文件，改名

fileSystem.rename(new Path("/1.txt"),new Path("/javaTest/2.txt"));

1.2.4. 创建文件夹

fileSystem.mkdirs(new Path("/xx/yy/zz"));

1.2.5. 读文件

RemoteIterator iter = fileSystem.listFiles(new Path("/"), true);
while (iter.hasNext()){
    LocatedFileStatus next = iter.next();
    System.out.println("==============================");
    System.out.println("文件位置+"+next.getPath());
    System.out.println("块大小："+next.getBlockSize());
    System.out.println("文件副本数量："+next.getReplication());
}

原始代码

package com.looc.test;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.junit.Before;
import org.junit.Test;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

/**
 * @author chenPeng
 * @version 1.0.0
 * @ClassName HdfsClient.java
 * @Description TODO
 * @createTime 2019年01月18日 16:17:00
 */
public class HdfsClientTest {


    FileSystem fileSystem = null;

    /**
     * 测试查询
     * @Author chenpeng
     * @Description //TODO
     * @Date 19:28
     * @Param []
     * @return void
     **/
    @Test
    public void fondTest() throws IOException {
        RemoteIterator iter = fileSystem.listFiles(new Path("/"), true);
        while (iter.hasNext()){
            LocatedFileStatus next = iter.next();
            System.out.println("==============================");
            System.out.println("文件位置+"+next.getPath());
            System.out.println("块大小："+next.getBlockSize());
            System.out.println("文件副本数量："+next.getReplication());
        }
        fileSystem.close();
    }

    /**
     * 测试目录及文件查询
     * @Author chenpeng
     * @Description //TODO
     * @Date 19:28
     * @Param []
     * @return void
     **/
    @Test
    public void fondAllTest() throws IOException {
        FileStatus[] fileStatuses = fileSystem.listStatus(new Path("/"));
        for (FileStatus status:fileStatuses) {
            System.out.println("==============================");
            System.out.println("文件位置+"+status.getPath());
            System.out.println(status.isDirectory()? "这是文件夹":"这是文件");
        }
        fileSystem.close();
    }

    /**
     * 初始化
     * @Author chenpeng
     * @Description //TODO
     * @Date 18:57
     * @Param []
     * @return void
     **/
    @Before
    public void init() throws URISyntaxException, IOException, InterruptedException {
        Configuration conf = new Configuration();
        conf.set("dfs.replication","2");
        conf.set("dfs.blocksize","64M");
        fileSystem = FileSystem.get(new URI("hdfs://vm01:9000/"),conf,"root");
    }

    /**
     * 下载文件
     * @Author chenpeng
     * @Description //TODO
     * @Date 18:57
     * @Param []
     * @return void
     **/
    @Test
    public void testGet() throws IOException {
        fileSystem.copyToLocalFile(
                new Path("/javaTest/1.txt"),new Path("C:\\Users\\81022\\Desktop\\"));
        fileSystem.close();
    }
    
    /**
     * 移动文件
     * @Author chenpeng
     * @Description //TODO
     * @Date 19:03 
     * @Param []
     * @return void
     **/
    @Test
    public void renameTest() throws IOException {
        fileSystem.rename(new Path("/1.txt"),new Path("/javaTest/1.txt"));
        fileSystem.close();
    }

    /**
     * 创建目录
     * @Author chenpeng
     * @Description //TODO
     * @Date 19:24
     * @Param []
     * @return void
     **/
    @Test
    public void mkdirTest() throws IOException {
        fileSystem.mkdirs(new Path("/xx/yy/zz"));
        fileSystem.close();
    }

    /**
     * 测试删除文件
     * @Author chenpeng
     * @Description //TODO
     * @Date 18:08
     * @Param [fileSystem]
     * @return void
     **/
    @Test
    public void testDelete() throws IOException {
        fileSystem.delete(new Path("/video.zip"),true);
        fileSystem.close();
    }


    /**
     * 上传文件测试
     * @Author chenpeng
     * @Description //TODO
     * @Date 18:07
     * @Param [fileSystem]
     * @return void
     **/
    @Test
    public void testPut() throws IOException {
        fileSystem.copyFromLocalFile(
                new Path("C:\\Users\\81022\\Desktop\\1.txt"),new Path("/xx/yy/zz/1.txt"));
        fileSystem.close();
    }
}

读取文件中的内容，以及向文件中写内容

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.*;
import java.net.URI;
import java.net.URISyntaxException;

/**
 * @author chenPeng
 * @version 1.0.0
 * @ClassName HdfsReanAndWriteDemo.java
 * @Description TODO
 * @createTime 2019年01月28日 22:37:00
 */
public class HdfsReanAndWriteDemo {
    FileSystem fileSystem;
    private String url = "hdfs://vm01:9000/";
    @Before
    public void init() throws URISyntaxException, IOException, InterruptedException {
        fileSystem = FileSystem.get(new URI(url),new Configuration(),"root");
    }

    /**
     * 读取文件
     * @Author chenpeng
     * @Description //TODO
     * @Date 22:54
     * @Param []
     * @return void
     **/
    @Test
    public void readerHdfsDemo() throws IOException {
        FilterInputStream in = fileSystem.open(new Path("/javaTest/2.txt"));

        BufferedReader br = new BufferedReader(new InputStreamReader(in,"utf-8"));

        String line = null;
        while ((line = br.readLine())!=null){
            System.out.println(line);
        }

        br.close();
        in.close();
    }

    /**
     * 读取文件 指定位置开始
     * @Author chenpeng
     * @Description //TODO
     * @Date 22:54
     * @Param []
     * @return void
     **/
    @Test
    public void readerRandomHdfsDemo() throws IOException {
        FSDataInputStream in = fileSystem.open(new Path("/javaTest/1.txt"));

        //指定从第三个开始读
        in.seek(4);

        byte[] bs = new byte[4];
        int len = 0;
        while ((len = in.read(bs))!=-1){
            System.out.print(new String(bs,0,len,"gbk"));
        }


        in.close();
    }


    @Test
    public void writeHDFSDemo() throws IOException {
        fileSystem.delete(new Path("/javaTest/2.txt"),true);
        FSDataOutputStream out = fileSystem.create(new Path("/javaTest/2.txt"),true);

        out.write("测试第二行".getBytes());
        out.close();
    }

    @After
    public void last() throws IOException {
        fileSystem.close();
    }
}