hadoop 读流程和写流程

hadoop HDFD读流程
hadoop HDFD写流程

package com.lhj.hadoop;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;



/**
* 创建路径并上传文件，hdfs://node1:9000/test/input/aaa.txt
* 导出jar文件到node1
* 执行jar包：hadoop jar HdfsUtils.jar com.lhj.hadoop.HdfsUtils
* @author Administrator
*
* 读的流程：
* 1.客户端通过FileSystem的open方法打开一个输入流。FileSystem具体的实现是DistributedFileSystem这个类。
* 2.从namenode获取block的位置信息
* 3.从datanode读取数据，（通过流的方式FSDataInputStream）
* 4.close关闭流
*
* write的流程：
* 1.客户端通过FileSystem的create方法，创建一个输出流
* 2.往namenode写元数据
* 3.往其中一个datanode写数据，由datanode产生线程，复制到其他datanode上
* 4.close关闭流
*/
public class HdfsUtils {
        public static void main(String[] args) {
                HdfsUtils hdfsUtils = new HdfsUtils();
                hdfsUtils.read();
                hdfsUtils.write();
        }
       
        private void read() {
                Path path = new Path("hdfs://node1:9000/test/input/aaa.txt");
                Configuration conf = new Configuration();
                BufferedReader reader=null;
               
                try {
                        FSDataInputStream in = FileSystem.get(conf).open(path);//关键代码，FS打开路径
                        reader = new BufferedReader(new InputStreamReader(in));
                       
                        String line=null;
                        while ((line=reader.readLine()) != null) {
                                System.out.println(line);                       
                        }
                } catch (IOException e) {
                        e.printStackTrace();
                } finally{
                        if (reader!=null){
                                try {
                                        reader.close();
                                } catch (IOException e) {
                                        e.printStackTrace();
                                }
                        }
                }
               
        }


        private void write(){
                Path path = new Path("hdfs://node1:9000/test/input/bbb.txt");
                Configuration conf = new Configuration();
                FSDataOutputStream output = null;
               
                String[] contents=new String[]{
                                "aaaaaaaaaaa\n",
                                "bbbbbbbbbbb\n",
                                "ccccccccccc\n",
                                "\n"
                };
               
                try {
                        FileSystem fs = path.getFileSystem(conf);
                        output = fs.create(path);
                       
                        for (String line:contents){
                                output.write(line.getBytes("UTF-8"));
                                output.flush();
                        }

                } catch (IOException e) {
                        e.printStackTrace();
                } finally{
                        try {
                                output.close();
                        } catch (IOException e) {
                                e.printStackTrace();
                        }
                }
        }

}