HDFS流式上传文件

借助工具类

import org.apache.hadoop.io.IOUtils;

实现直接从一个输入流往HDFS的输出流中写数据

 

public static void main(String[] args) throws Exception{
		System.setProperty("hadoop.home.dir", "D:\\hadoop\\hadoop-2.7.3");
		Configuration conf  = new Configuration();
		conf.set("dfs.replication","3");
		FileSystem hdfs = FileSystem.get(new URI("hdfs://node1:9000"), conf, "root");

		//读取本地文件
		InputStream in = new FileInputStream("D:/1.pdf");
		//在Hdfs上创建一个文件，返回输出流
		OutputStream out = hdfs.create(new Path("/1.pdf"));
		//输入 ---》  输出
		IOUtils.copyBytes(in, out, 4096, true);
}

其实现底层原理是借助一个字节数组

 public static void copyBytes(InputStream in, OutputStream out, int buffSize) throws IOException {
        PrintStream ps = out instanceof PrintStream ? (PrintStream)out : null;
        byte[] buf = new byte[buffSize];

        for(int bytesRead = in.read(buf); bytesRead >= 0; bytesRead = in.read(buf)) {
            out.write(buf, 0, bytesRead);
            if (ps != null && ps.checkError()) {
                throw new IOException("Unable to write to output stream.");
            }
        }

    }